Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

import os import re import requests from bs4 import BeautifulSoup from PIL import Image…

  1. User

    import os import re import requests from bs4 import BeautifulSoup from PIL import Image from io import BytesIO from urllib.parse import urljoin from tqdm import tqdm import zipfile import time import random

    Constants

    BASE_URL = "https://hdri-haven.com" HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+")

    Directories

    DOWNLOAD_DIR = "downloads" WEBP_DIR = "webp_images"

    def create_directories(): os.makedirs(DOWNLOAD_DIR, exist_ok=True) os.makedirs(WEBP_DIR, exist_ok=True)

    def get_all_hdri_pages(): """ Scans the BASE_URL to find all HDRI page URLs matching the pattern. """ response = requests.get(BASE_URL) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser')

    links = set()
    for a_tag in soup.find_all('a', href=True):
        href = urljoin(BASE_URL, a_tag['href'])
        if HDRI_URL_PATTERN.match(href):
            links.add(href)
    return list(links)
    

    def get_2k_image_link(hdri_page_url): """ Scrapes the 2K image download link from an HDRI page. """ response = requests.get(hdri_page_url) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser')

    link_2k = soup.find('a', class_='downloadable link-2k', href=True)
    if link_2k:
        return link_2k['href']
    return None
    

    def download_image(image_url): """ Downloads an image from the given URL and saves it to the DOWNLOAD_DIR. Returns the file path of the downloaded image. """ local_filename = os.path.join(DOWNLOAD_DIR, os.path.basename(image_url)) with requests.get(image_url, stream=True) as r: r.raise_for_status() with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) return local_filename

    def convert_to_webp(image_path): """ Converts the given image to WebP format and saves it in the WEBP_DIR. """ base_name = os.path.splitext(os.path.basename(image_path))[0] webp_path = os.path.join(WEBP_DIR, f"{base_name}.webp")

    with Image.open(image_path) as img:
        img.save(webp_path, 'WEBP')
    return webp_path
    

    def zip_webp_images(zip_name="webp_images.zip"): """ Creates a zip file containing all WebP images in the WEBP_DIR. """ with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf: for root, _, files in os.walk(WEBP_DIR): for file in files: file_path = os.path.join(root, file) zipf.write(file_path, arcname=file) print(f"Zip file '{zip_name}' created successfully.")

    def main(): create_directories()

    print("Scanning website for HDRI pages...")
    hdri_pages = get_all_hdri_pages()
    print(f"Found {len(hdri_pages)} HDRI pages.")
    
    downloaded_images = []
    for page in tqdm(hdri_pages, desc="Processing HDRI pages"):
        image_link = get_2k_image_link(page)
        if image_link:
            try:
                image_path = download_image(image_link)
                downloaded_images.append(image_path)
                # Random delay between 2 to 3 seconds
                time.sleep(random.uniform(2, 3))
            except Exception as e:
                print(f"Failed to download {image_link}: {e}")
    
    print("Converting images to WebP format...")
    for image_path in tqdm(downloaded_images, desc="Converting images"):
        try:
            convert_to_webp(image_path)
        except Exception as e:
            print(f"Failed to convert {image_path}: {e}")
    
    print("Creating zip file of WebP images...")
    zip_webp_images()
    
    print("Process completed successfully.")
    

    if name == "main": main()

    this code is working but it only can find 48 hdr 2k images. i need more. think very carefully. the all pages are the same structure. we need to find more somehow.

  2. Assistant

    It looks like the issue lies in how your get_all_hdri_pages() function is fetching the HDRI URLs. Currently, it's only scanning the home page (BASE_URL) for HDRI links. The home page might only contain a limited number of HDRI links (in your case, 48), which is why you're not finding all the available HDRI images on the site.

    To find more HDRI images, you need to scan the HDRI index page which lists all the HDRI assets. On the HDRI Haven website, this page is usually located at https://hdri-haven.com/hdris/. By scraping this page instead, you can retrieve the full list of HDRI image pages.

    Here's how you can modify your code:

    1. Update the BASE_URL to point to the HDRI index page.

      Change:

      BASE_URL = "https://hdri-haven.com"
      

      To:

      BASE_URL = "https://hdri-haven.com/hdris/"
      
    2. Adjust the HDRI_URL_PATTERN to match the new URL structure if necessary.

      Your current pattern should still match, but ensure it's correct:

      HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+")
      
    3. Modify the get_all_hdri_pages() function to use the new BASE_URL.

      Update the function like so:

      def get_all_hdri_pages():
          """
          Scans the HDRI index page to find all HDRI page URLs matching the pattern.
          """
          response = requests.get(BASE_URL)
          response.raise_for_status()
          soup = BeautifulSoup(response.text, 'html.parser')
      
          links = set()
          for a_tag in soup.find_all('a', href=True):
              href = urljoin(BASE_URL, a_tag['href'])
              if HDRI_URL_PATTERN.match(href):
                  links.add(href)
          return list(links)
      
    4. Handle pagination if the HDRI index page has multiple pages.

      If the index page uses pagination, you'll need to loop through all pages to collect all the HDRI links. Here's how you can modify get_all_hdri_pages() to handle this:

      def get_all_hdri_pages():
          """
          Scans all pages of the HDRI index to find all HDRI page URLs.
          """
          page_number = 1
          links = set()
          while True:
              index_url = f"{BASE_URL}?page={page_number}"
              response = requests.get(index_url)
              response.raise_for_status()
              soup = BeautifulSoup(response.text, 'html.parser')
      
              page_links = set()
              for a_tag in soup.find_all('a', href=True):
                  href = urljoin(BASE_URL, a_tag['href'])
                  if HDRI_URL_PATTERN.match(href):
                      page_links.add(href)
      
              if not page_links:
                  break  # No more pages
              links.update(page_links)
              page_number += 1
          return list(links)
      
    5. Test the updated code.

      After making these changes, run your script again. It should now find all the HDRI pages available on the site.

      Note: Be mindful of the website's terms of service and robots.txt file to ensure you're allowed to scrape this data. Additionally, consider adding delays between requests to respect the website's bandwidth.

    Here's the revised get_all_hdri_pages() function with pagination handling:

    def get_all_hdri_pages():
        """
        Scans all pages of the HDRI index to find all HDRI page URLs.
        """
        links = set()
        response = requests.get(BASE_URL)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
    
        # Find the total number of pages
        pagination = soup.find('div', class_='pagination')
        if pagination:
            pages = pagination.find_all('a', href=True)
            total_pages = max([int(a.get_text()) for a in pages if a.get_text().isdigit()])
        else:
            total_pages = 1
    
        for page_number in range(1, total_pages + 1):
            index_url = f"{BASE_URL}?page={page_number}"
            response = requests.get(index_url)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, 'html.parser')
    
            for a_tag in soup.find_all('a', href=True):
                href = urljoin(BASE_URL, a_tag['href'])
                if HDRI_URL_PATTERN.match(href):
                    links.add(href)
        return list(links)
    

    Additional Tips:

    • Use the Sitemap: Many websites provide a sitemap (e.g., https://hdri-haven.com/sitemap.xml) that lists all available URLs. You can parse the sitemap to get all HDRI page URLs.

    • Update for Redirects: Since https://hdri-haven.com now redirects to https://polyhaven.com, you may need to update the script to work with the new site structure. Adjust the BASE_URL and patterns accordingly.

    • Respect Robots.txt: Always check https://hdri-haven.com/robots.txt to see if the site allows scraping.

    Revised Main Script:

    Make sure to update the BASE_URL and any other URLs or patterns to match the current site structure if the site has changed.

    import os
    import re
    import requests
    from bs4 import BeautifulSoup
    from PIL import Image
    from urllib.parse import urljoin
    from tqdm import tqdm
    import zipfile
    import time
    import random
    
    # Constants
    BASE_URL = "https://hdri-haven.com/hdris/"
    HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+")
    
    # Directories
    DOWNLOAD_DIR = "downloads"
    WEBP_DIR = "webp_images"
    
    def create_directories():
        os.makedirs(DOWNLOAD_DIR, exist_ok=True)
        os.makedirs(WEBP_DIR, exist_ok=True)
    
    def get_all_hdri_pages():
        """
        Scans all pages of the HDRI index to find all HDRI page URLs.
        """
        links = set()
        response = requests.get(BASE_URL)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
    
        # Find the total number of pages
        pagination = soup.find('div', class_='pagination')
        if pagination:
            pages = pagination.find_all('a', href=True)
            total_pages = max([int(a.get_text()) for a in pages if a.get_text().isdigit()])
        else:
            total_pages = 1
    
        for page_number in range(1, total_pages + 1):
            index_url = f"{BASE_URL}?page={page_number}"
            response = requests.get(index_url)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, 'html.parser')
    
            for a_tag in soup.find_all('a', href=True):
                href = urljoin(BASE_URL, a_tag['href'])
                if HDRI_URL_PATTERN.match(href):
                    links.add(href)
        return list(links)
    
    # Rest of your code remains unchanged
    
    if __name__ == "__main__":
        main()
    

    By making these changes, your script should now find and process all available HDRI images on the site.