Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· OpenAIAi2 WildChat-4.8M (AI2)

instead of a random name use a hash so it doesnt download it twice import requests from…

  1. User

    instead of a random name use a hash so it doesnt download it twice import requests from bs4 import BeautifulSoup import os import random import string

    def generate_random_filename(extension): letters = string.ascii_lowercase filename = "".join(random.choice(letters) for i in range(10)) + "." + extension return filename

    def scrape_content(urls, save=False): for url in urls: print(f"Scraping URL: {url}") try: response = requests.get(url) response.raise_for_status()

            # Parse the content using BeautifulSoup
            soup = BeautifulSoup(response.content, "html.parser")
            
            # Find all divs with data-type="dot_art"
            dot_art_divs = soup.find_all("div", {"data-type": "dot_art"})
            
            for i, dot_art_div in enumerate(dot_art_divs, start=1):
                content = dot_art_div.get_text(separator="\n", strip=True)
                print(f"Content of dot_art #{i}:")
                print(content)
                
                if save:
                    filename = "a/"+generate_random_filename("txt")
                    with open(filename, "w", encoding="utf-8") as file:
                        file.write(content)
                    print(f"Content of dot_art #{i} saved to {filename}")
                    
            if not dot_art_divs:
                print("No dot_art found on the page.")
                
        except requests.HTTPError as e:
            print(f"Error fetching {url}: {e}")
        except requests.RequestException as e:
            # Handle other requests exceptions
            print(f"Request failed for {url}: {e}")
    

    urls_to_scrape = [ "https://emojicombos.com/nsfw", "https://emojicombos.com/pussy", "https://emojicombos.com/rule34", ]

    scrape_content(urls_to_scrape,save=True)

  2. Assistant

    To avoid downloading the content twice, we can use a hash of the content itself to generate a unique filename for each downloaded piece of content. We can use the hashlib module available in Python to easily generate a hash for the contents.

    Below, I have modified the generate_random_filename function to a new generate_hashed_filename function that accepts the content and returns a hash-based filename. Also, I noticed you were hardcoding the directory "a/" in your original script – it's essential to ensure that this directory exists or create it before writing files to it. Here's an updated version of the script with these changes implemented:

    import requests
    from bs4 import BeautifulSoup
    import hashlib
    import os
    
    def generate_hashed_filename(content, extension):
        # Use the SHA256 hash function
        hash_object = hashlib.sha256(content.encode('utf-8'))
        # Get the hexadecimal representation of the hash
        hex_dig = hash_object.hexdigest()
        # Return the hashed filename
        return hex_dig + "." + extension
    
    def scrape_content(urls, save=False):
        folder = "a"  # Change it accordingly if you want to use a different folder
        
        if save and not os.path.exists(folder):
            os.makedirs(folder)  # Create the folder if it doesn't exist
            
        for url in urls:
            print(f"Scraping URL: {url}")
            try:
                response = requests.get(url)
                response.raise_for_status()
                
                # Parse the content using BeautifulSoup
                soup = BeautifulSoup(response.content, "html.parser")
                
                # Find all divs with data-type="dot_art"
                dot_art_divs = soup.find_all("div", {"data-type": "dot_art"})
                
                for i, dot_art_div in enumerate(dot_art_divs, start=1):
                    content = dot_art_div.get_text(separator="\n", strip=True)
                    print(f"Content of dot_art #{i}:")
                    print(content)
                    
                    if save:
                        filename = generate_hashed_filename(content, "txt")
                        filepath = os.path.join(folder, filename)
                        if not os.path.isfile(filepath):  # Check if file already exists
                            with open(filepath, "w", encoding="utf-8") as file:
                                file.write(content)
                            print(f"Content of dot_art #{i} saved to {filepath}")
                        else:
                            print(f"File {filepath} already exists, skipping.")
                        
                if not dot_art_divs:
                    print("No dot_art found on the page.")
                    
            except requests.HTTPError as e:
                print(f"Error fetching {url}: {e}")
            except requests.RequestException as e:
                # Handle other requests exceptions
                print(f"Request failed for {url}: {e}")
    
    urls_to_scrape = [
        "https://emojicombos.com/nsfw",
        "https://emojicombos.com/pussy",
        "https://emojicombos.com/rule34",
    ]
    
    scrape_content(urls_to_scrape, save=True)
    

    This script now generates a unique hash for each piece of content and uses that hash as the filename, ensuring that the same content is not downloaded more than once. It also checks if the hashed filename already exists in the specified directory before saving to prevent duplicate downloads. Remember to ensure the URLs you're scraping do not contain content that violates any website's terms of service or local laws.

    Additionally, if the folder "a" is intended to be one level up from your script's directory, you may want to adjust the folder = "a" line appropriately, such as using folder = "../a" instead. Make sure you have the necessary permissions to create and write to this directory.