Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· OpenAIAi2 WildChat-4.8M (AI2)

make sure it use the check for ads and also don't save svgs: import os import requests…

  1. User

    make sure it use the check for ads and also don't save svgs:

    import os import requests from bs4 import BeautifulSoup, SoupStrainer from concurrent.futures import ThreadPoolExecutor from urllib.parse import urljoin, urlparse

    Function to determine if the image is likely an ad or icon

    def is_likely_ad_or_icon(image_url, minimum_size_kb=10): # Define common ad and icon indicator keywords common_ad_indicators = ["ad", "banner", "sponsor"] common_icon_indicators = ["favicon", "logo", "icon"]

    # Check for ad indicators in the URL
    if any(indicator in image_url.lower() for indicator in common_ad_indicators):
        return True
    
    # Try to fetch the image content size
    try:
        response = requests.head(image_url, timeout=5)
        content_length = response.headers.get("content-length", None)
        if content_length and int(content_length) < minimum_size_kb * 1024:
            return True
    except Exception as e:
        print(f"Could not fetch head for {image_url}: {e}")
    
    return any(indicator in image_url.lower() for indicator in common_icon_indicators)
    

    def download_image(image_url, destination_folder, session): if not is_likely_ad_or_icon(image_url): try: with session.get(image_url, stream=True) as r: r.raise_for_status() img_name = os.path.basename(urlparse(image_url).path) img_path = os.path.join(destination_folder, img_name) with open(img_path, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) print(f"Downloaded {img_name} to {destination_folder}") except Exception as e: print(f"Could not download {image_url}: {e}")

    Function to download images using multithreading

    def download_images(url, destination_folder, session): # Create the folder path if it doesn't exist if not os.path.isdir(destination_folder): os.makedirs(destination_folder)

    response = session.get(url)
    soup = BeautifulSoup(response.text, "html.parser", parse_only=SoupStrainer("img"))
    images = [
        urljoin(url, img.get("src") or img.get("data-src"))
        for img in soup
        if img.get("src") or img.get("data-src")
    ]
    
    # Use ThreadPoolExecutor to download images in parallel
    with ThreadPoolExecutor(max_workers=10) as executor:
        for image_url in images:
            executor.submit(download_image, image_url, destination_folder, session)
    

    def remove_files(): for files in os.listdir("downloaded_images"): os.remove(os.path.join("downloaded_images", files))

    def get_images(links): fixed_urls = [BeautifulSoup(tag, "html.parser").a["href"] for tag in links if tag] with requests.Session() as session: # Now using the context manager for the session destination_dir = "downloaded_images" with ThreadPoolExecutor(max_workers=10) as executor: for link in fixed_urls: executor.submit(download_images, link, destination_dir, session)

  2. Assistant

    Certainly! To ensure that the provided Python script does not save SVG files and checks for ads while keeping images, we can modify the download_image function to include an additional check for SVG file extensions and the provided criteria you've already implemented for ad/image size/content. We can also refactor the is_likely_ad_or_icon function to incorporate the ad and icon checks cleanly.

    Here is your updated Python script following these requirements:

    import os
    import requests
    from bs4 import BeautifulSoup, SoupStrainer
    from concurrent.futures import ThreadPoolExecutor
    from urllib.parse import urljoin, urlparse
    
    def is_likely_ad_or_icon(image_url, image_headers, minimum_size_kb=10):
        common_ad_indicators = ["ad", "banner", "sponsor"]
        common_icon_indicators = ["favicon", "logo", "icon"]
    
        # Check if image URL contains ad or icon indicators
        if any(indicator in image_url.lower() for indicator in common_ad_indicators + common_icon_indicators):
            return True
    
        # Fetch image content size from the headers and check if it is smaller than the minimum size
        content_length = image_headers.get("content-length", None)
        if content_length and int(content_length) < minimum_size_kb * 1024:
            return True
    
        return False
        
    def is_svg(image_url):
        return image_url.lower().endswith('.svg')
    
    def download_image(image_url, destination_folder, session):
        try:
            response = session.head(image_url, timeout=5)
            if not is_likely_ad_or_icon(image_url, response.headers) and not is_svg(image_url):
                with session.get(image_url, stream=True) as r:
                    r.raise_for_status()
                    img_name = os.path.basename(urlparse(image_url).path)
                    img_path = os.path.join(destination_folder, img_name)
                    with open(img_path, "wb") as f:
                        for chunk in r.iter_content(chunk_size=8192):
                            f.write(chunk)
                print(f"Downloaded {img_name} to {destination_folder}")
        except Exception as e:
            print(f"Could not download {image_url}: {e}")
    
    def download_images(url, destination_folder, session):
        if not os.path.isdir(destination_folder):
            os.makedirs(destination_folder)
    
        response = session.get(url)
        soup = BeautifulSoup(response.text, "html.parser", parse_only=SoupStrainer("img"))
        images = [
            urljoin(url, img.get("src") or img.get("data-src"))
            for img in soup
            if img.get("src") or img.get("data-src")
        ]
    
        with ThreadPoolExecutor(max_workers=10) as executor:
            for image_url in images:
                executor.submit(download_image, image_url, destination_folder, session)
    
    def remove_files(directory):
        for file_name in os.listdir(directory):
            file_path = os.path.join(directory, file_name)
            if os.path.isfile(file_path):
                os.remove(file_path)
        
    def get_images(links):
        fixed_urls = [BeautifulSoup(tag, "html.parser").a["href"] for tag in links if tag]
        with requests.Session() as session:
            destination_dir = "downloaded_images"
            remove_files(destination_dir)  # Ensure directory is empty before downloading new images
            with ThreadPoolExecutor(max_workers=10) as executor:
                for link in fixed_urls:
                    executor.submit(download_images, link, destination_dir, session)
    

    In this modified script, we have:

    1. Refactored the is_likely_ad_or_icon function to also take image headers as a parameter.
    2. Added a new function is_svg to check whether the image URL ends with the '.svg' extension.
    3. Incorporated these two checks into the download_image function to ensure ad images and SVGs are filtered out.
    4. Added a call to remove_files at the start of get_images to clean the directory of previous content before starting a new download session.

    Remember to apply any additional criteria specific to your task or project as needed.

  3. User

    also make sure it ends with something and don't download files with no ending like .jpg/jpeg/png and so on

  4. Assistant

    Certainly! To ensure that the script only downloads images with certain file extensions, we can add another function to check if the URL ends with an allowed image extension. Here's an updated version of the function that implements this requirement:

    import os
    import requests
    from bs4 import BeautifulSoup, SoupStrainer
    from concurrent.futures import ThreadPoolExecutor
    from urllib.parse import urljoin, urlparse
    
    # Allowed image extensions
    ALLOWED_EXTENSIONS = {'jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff'}
    
    def has_allowed_extension(image_url):
        return any(image_url.lower().endswith('.' + ext) for ext in ALLOWED_EXTENSIONS)
    
    def is_likely_ad_or_icon(image_url, image_headers, minimum_size_kb=10):
        common_ad_indicators = ["ad", "banner", "sponsor"]
        common_icon_indicators = ["favicon", "logo", "icon"]
    
        # Check if image URL contains ad or icon indicators
        if any(indicator in image_url.lower() for indicator in common_ad_indicators + common_icon_indicators):
            return True
    
        # Fetch image content size from the headers and check if it is smaller than the minimum size
        content_length = image_headers.get("content-length", None)
        if content_length and int(content_length) < minimum_size_kb * 1024:
            return True
    
        return False
        
    def download_image(image_url, destination_folder, session):
        try:
            image_ext = os.path.splitext(urlparse(image_url).path)[1][1:].strip().lower()
            if image_ext not in ALLOWED_EXTENSIONS:
                print(f"Skipping {image_url}: Unsupported extension {image_ext}")
                return
    
            response = session.head(image_url, timeout=5)
            if not is_likely_ad_or_icon(image_url, response.headers) and has_allowed_extension(image_url):
                with session.get(image_url, stream=True) as r:
                    r.raise_for_status()
                    img_name = os.path.basename(urlparse(image_url).path)
                    img_path = os.path.join(destination_folder, img_name)
                    with open(img_path, "wb") as f:
                        for chunk in r.iter_content(chunk_size=8192):
                            f.write(chunk)
                print(f"Downloaded {img_name} to {destination_folder}")
        except Exception as e:
            print(f"Could not download {image_url}: {e}")
    
    def download_images(url, destination_folder, session):
        if not os.path.isdir(destination_folder):
            os.makedirs(destination_folder)
    
        response = session.get(url)
        soup = BeautifulSoup(response.text, "html.parser", parse_only=SoupStrainer("img"))
        images = [
            urljoin(url, img.get("src") or img.get("data-src"))
            for img in soup
            if img.get("src") or img.get("data-src")
        ]
    
        with ThreadPoolExecutor(max_workers=10) as executor:
            for image_url in images:
                executor.submit(download_image, image_url, destination_folder, session)
    
    def remove_files(directory):
        for file_name in os.listdir(directory):
            file_path = os.path.join(directory, file_name)
            if os.path.isfile(file_path):
                os.remove(file_path)
        
    def get_images(links):
        fixed_urls = [BeautifulSoup(tag, "html.parser").a["href"] for tag in links if tag]
        with requests.Session() as session:
            destination_dir = "downloaded_images"
            remove_files(destination_dir)  # Ensure directory is empty before downloading new images
            with ThreadPoolExecutor(max_workers=10) as executor:
                for link in fixed_urls:
                    executor.submit(download_images, link, destination_dir, session)
    

    In this updated script, we have:

    1. Defined a set of allowed image extensions, ALLOWED_EXTENSIONS.
    2. Added the function has_allowed_extension to check if the URL ends with one of the allowed file extensions.
    3. Modified the download_image function to skip downloading an image if it doesn't have an allowed extension by using the has_allowed_extension function.

    Now, the script is set up to ignore any image URLs that do not end with one of the allowed file extensions such as .jpg, .jpeg, .png, etc. It will download only the files with the extensions specified in the ALLOWED_EXTENSIONS set.