I want to create a dataset downloader in python, it needs to have a base data downloader…
- User
I want to create a dataset downloader in python, it needs to have a base data downloader class with the shared functionality to be extended from for use with any website, the goal is to download images based on a set criteria(tags, keywords etc for a filter, with a blacklist), it should also be able to obtain a "caption" from the website or api to create a .txt file along side the image file, using the same base name as the image file, which describes the contents of the image. To start with, we need the base class DataDownloader, and we need to extend this into another class for booru style websites, which sets up the core framework for working with booru style websites with a class BooruDownloader, then we extend it further to support ATFBooru with ATFBooruDownloader. And for now we will just put it in the one python script file.
- Assistant
Sure! Here's an example of how you can create the base class
DataDownloaderand extend it for working with booru style websites:import os import requests class DataDownloader: def __init__(self, output_folder): self.output_folder = output_folder def download_data(self, url, filename): response = requests.get(url, stream=True) filepath = os.path.join(self.output_folder, filename) with open(filepath, 'wb') as file: for chunk in response.iter_content(chunk_size=1024): if chunk: file.write(chunk) def download_image_with_caption(self, image_url, caption): filename = os.path.basename(image_url) caption_filename = os.path.splitext(filename)[0] + '.txt' caption_filepath = os.path.join(self.output_folder, caption_filename) self.download_data(image_url, filename) with open(caption_filepath, 'w') as file: file.write(caption) class BooruDownloader(DataDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder) self.booru_url = booru_url def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images based on tags from the booru website pass class ATFBooruDownloader(BooruDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder, booru_url) def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images from the ATFBooru website, using tags and blacklist pass # Example usage output_folder = 'path/to/save/data' booru_url = 'https://example.com/booru' atf_booru_downloader = ATFBooruDownloader(output_folder, booru_url) tags = ['cat', 'kitten'] blacklist = ['nsfw'] atf_booru_downloader.download_images_by_tags(tags, blacklist)Note: You'll need to implement the logic inside the
download_images_by_tags()methods in theBooruDownloaderandATFBooruDownloaderclasses to fetch the images and captions from the respective booru websites. - User
Please put it in a code block so I can easily copy and paste it
- Assistant
Certainly! Here's the code in a code block:
import os import requests class DataDownloader: def __init__(self, output_folder): self.output_folder = output_folder def download_data(self, url, filename): response = requests.get(url, stream=True) filepath = os.path.join(self.output_folder, filename) with open(filepath, 'wb') as file: for chunk in response.iter_content(chunk_size=1024): if chunk: file.write(chunk) def download_image_with_caption(self, image_url, caption): filename = os.path.basename(image_url) caption_filename = os.path.splitext(filename)[0] + '.txt' caption_filepath = os.path.join(self.output_folder, caption_filename) self.download_data(image_url, filename) with open(caption_filepath, 'w') as file: file.write(caption) class BooruDownloader(DataDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder) self.booru_url = booru_url def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images based on tags from the booru website pass class ATFBooruDownloader(BooruDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder, booru_url) def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images from the ATFBooru website, using tags and blacklist pass # Example usage output_folder = 'path/to/save/data' booru_url = 'https://example.com/booru' atf_booru_downloader = ATFBooruDownloader(output_folder, booru_url) tags = ['cat', 'kitten'] blacklist = ['nsfw'] atf_booru_downloader.download_images_by_tags(tags, blacklist)You can copy and paste this code into your Python script file and modify it according to your needs.
- User
We need extra error checking, especially around the requests.get, to ensure we get a valid request, that it isn't 404 etc, and that we actually get a valid page
- Assistant
Certainly! Here's an updated version of the code with additional error checking for the
requests.get()method:import os import requests class DataDownloader: def __init__(self, output_folder): self.output_folder = output_folder def download_data(self, url, filename): response = requests.get(url, stream=True) if response.status_code == 200: filepath = os.path.join(self.output_folder, filename) with open(filepath, 'wb') as file: for chunk in response.iter_content(chunk_size=1024): if chunk: file.write(chunk) return True else: return False def download_image_with_caption(self, image_url, caption): filename = os.path.basename(image_url) caption_filename = os.path.splitext(filename)[0] + '.txt' caption_filepath = os.path.join(self.output_folder, caption_filename) if self.download_data(image_url, filename): with open(caption_filepath, 'w') as file: file.write(caption) return True else: return False class BooruDownloader(DataDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder) self.booru_url = booru_url def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images based on tags from the booru website pass class ATFBooruDownloader(BooruDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder, booru_url) def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images from the ATFBooru website, using tags and blacklist pass # Example usage output_folder = 'path/to/save/data' booru_url = 'https://example.com/booru' atf_booru_downloader = ATFBooruDownloader(output_folder, booru_url) tags = ['cat', 'kitten'] blacklist = ['nsfw'] atf_booru_downloader.download_images_by_tags(tags, blacklist)Now, the
download_data()method checks the status code of the response from therequests.get()call. If it's200, indicating a successful request, the data is downloaded. Otherwise, the method returnsFalse. Similarly, thedownload_image_with_caption()method checks for a successful download of the image, and if it fails, it returnsFalse. You can handle theseFalsereturn values appropriately in your own code. - User
Adapt support in the DataDownloader for a generic REST API.
- Assistant
Certainly! Here's an updated version of the code that adds support for consuming a generic REST API with the
DataDownloaderclass:import os import requests class DataDownloader: def __init__(self, output_folder): self.output_folder = output_folder def download_data(self, url, filename): response = requests.get(url, stream=True) if response.status_code == 200: filepath = os.path.join(self.output_folder, filename) with open(filepath, 'wb') as file: for chunk in response.iter_content(chunk_size=1024): if chunk: file.write(chunk) return True else: return False def download_image_with_caption(self, image_url, caption): filename = os.path.basename(image_url) caption_filename = os.path.splitext(filename)[0] + '.txt' caption_filepath = os.path.join(self.output_folder, caption_filename) if self.download_data(image_url, filename): with open(caption_filepath, 'w') as file: file.write(caption) return True else: return False def fetch_data(self, url): response = requests.get(url) if response.status_code == 200: return response.json() else: return None class BooruDownloader(DataDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder) self.booru_url = booru_url def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images based on tags from the booru website pass class ATFBooruDownloader(BooruDownloader): def __init__(self, output_folder, booru_url): super().__init__(output_folder, booru_url) def download_images_by_tags(self, tags, blacklist=None): # Implement the logic to download images from the ATFBooru website, using tags and blacklist pass class RestApiDataDownloader(DataDownloader): def __init__(self, output_folder, api_url): super().__init__(output_folder) self.api_url = api_url def download_images_by_criteria(self, criteria): url = self.api_url + '/images?criteria=' + criteria data = self.fetch_data(url) if data: for image_data in data['images']: image_url = self.api_url + '/images/' + image_data['id'] caption = image_data['caption'] self.download_image_with_caption(image_url, caption) # Example usage output_folder = 'path/to/save/data' api_url = 'https://api.example.com' api_data_downloader = RestApiDataDownloader(output_folder, api_url) criteria = 'cats' api_data_downloader.download_images_by_criteria(criteria)In this updated code, a new class
RestApiDataDownloaderis introduced, which extends theDataDownloaderclass. It adds a methoddownload_images_by_criteria()that fetches image data from a generic REST API based on the provided criteria and downloads the images along with their captions. Thefetch_data()method in theDataDownloaderclass is used to retrieve the JSON response from the API.You can modify the
download_images_by_criteria()method to suit the specific endpoints and data structure of your REST API.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.