import os import re import requests from bs4 import BeautifulSoup from PIL import Image…
- User
import os import re import requests from bs4 import BeautifulSoup from PIL import Image from io import BytesIO from urllib.parse import urljoin from tqdm import tqdm import zipfile import time import random
Constants
BASE_URL = "https://hdri-haven.com" HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+")
Directories
DOWNLOAD_DIR = "downloads" WEBP_DIR = "webp_images"
def create_directories(): os.makedirs(DOWNLOAD_DIR, exist_ok=True) os.makedirs(WEBP_DIR, exist_ok=True)
def get_all_hdri_pages(): """ Scans the BASE_URL to find all HDRI page URLs matching the pattern. """ response = requests.get(BASE_URL) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser')
links = set() for a_tag in soup.find_all('a', href=True): href = urljoin(BASE_URL, a_tag['href']) if HDRI_URL_PATTERN.match(href): links.add(href) return list(links)def get_2k_image_link(hdri_page_url): """ Scrapes the 2K image download link from an HDRI page. """ response = requests.get(hdri_page_url) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser')
link_2k = soup.find('a', class_='downloadable link-2k', href=True) if link_2k: return link_2k['href'] return Nonedef download_image(image_url): """ Downloads an image from the given URL and saves it to the DOWNLOAD_DIR. Returns the file path of the downloaded image. """ local_filename = os.path.join(DOWNLOAD_DIR, os.path.basename(image_url)) with requests.get(image_url, stream=True) as r: r.raise_for_status() with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) return local_filename
def convert_to_webp(image_path): """ Converts the given image to WebP format and saves it in the WEBP_DIR. """ base_name = os.path.splitext(os.path.basename(image_path))[0] webp_path = os.path.join(WEBP_DIR, f"{base_name}.webp")
with Image.open(image_path) as img: img.save(webp_path, 'WEBP') return webp_pathdef zip_webp_images(zip_name="webp_images.zip"): """ Creates a zip file containing all WebP images in the WEBP_DIR. """ with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf: for root, _, files in os.walk(WEBP_DIR): for file in files: file_path = os.path.join(root, file) zipf.write(file_path, arcname=file) print(f"Zip file '{zip_name}' created successfully.")
def main(): create_directories()
print("Scanning website for HDRI pages...") hdri_pages = get_all_hdri_pages() print(f"Found {len(hdri_pages)} HDRI pages.") downloaded_images = [] for page in tqdm(hdri_pages, desc="Processing HDRI pages"): image_link = get_2k_image_link(page) if image_link: try: image_path = download_image(image_link) downloaded_images.append(image_path) # Random delay between 2 to 3 seconds time.sleep(random.uniform(2, 3)) except Exception as e: print(f"Failed to download {image_link}: {e}") print("Converting images to WebP format...") for image_path in tqdm(downloaded_images, desc="Converting images"): try: convert_to_webp(image_path) except Exception as e: print(f"Failed to convert {image_path}: {e}") print("Creating zip file of WebP images...") zip_webp_images() print("Process completed successfully.")if name == "main": main()
this code is working but it only can find 48 hdr 2k images. i need more. think very carefully. the all pages are the same structure. we need to find more somehow.
- Assistant
It looks like the issue lies in how your
get_all_hdri_pages()function is fetching the HDRI URLs. Currently, it's only scanning the home page (BASE_URL) for HDRI links. The home page might only contain a limited number of HDRI links (in your case, 48), which is why you're not finding all the available HDRI images on the site.To find more HDRI images, you need to scan the HDRI index page which lists all the HDRI assets. On the HDRI Haven website, this page is usually located at
https://hdri-haven.com/hdris/. By scraping this page instead, you can retrieve the full list of HDRI image pages.Here's how you can modify your code:
-
Update the
BASE_URLto point to the HDRI index page.Change:
BASE_URL = "https://hdri-haven.com"To:
BASE_URL = "https://hdri-haven.com/hdris/" -
Adjust the
HDRI_URL_PATTERNto match the new URL structure if necessary.Your current pattern should still match, but ensure it's correct:
HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+") -
Modify the
get_all_hdri_pages()function to use the newBASE_URL.Update the function like so:
def get_all_hdri_pages(): """ Scans the HDRI index page to find all HDRI page URLs matching the pattern. """ response = requests.get(BASE_URL) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') links = set() for a_tag in soup.find_all('a', href=True): href = urljoin(BASE_URL, a_tag['href']) if HDRI_URL_PATTERN.match(href): links.add(href) return list(links) -
Handle pagination if the HDRI index page has multiple pages.
If the index page uses pagination, you'll need to loop through all pages to collect all the HDRI links. Here's how you can modify
get_all_hdri_pages()to handle this:def get_all_hdri_pages(): """ Scans all pages of the HDRI index to find all HDRI page URLs. """ page_number = 1 links = set() while True: index_url = f"{BASE_URL}?page={page_number}" response = requests.get(index_url) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') page_links = set() for a_tag in soup.find_all('a', href=True): href = urljoin(BASE_URL, a_tag['href']) if HDRI_URL_PATTERN.match(href): page_links.add(href) if not page_links: break # No more pages links.update(page_links) page_number += 1 return list(links) -
Test the updated code.
After making these changes, run your script again. It should now find all the HDRI pages available on the site.
Note: Be mindful of the website's terms of service and robots.txt file to ensure you're allowed to scrape this data. Additionally, consider adding delays between requests to respect the website's bandwidth.
Here's the revised
get_all_hdri_pages()function with pagination handling:def get_all_hdri_pages(): """ Scans all pages of the HDRI index to find all HDRI page URLs. """ links = set() response = requests.get(BASE_URL) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Find the total number of pages pagination = soup.find('div', class_='pagination') if pagination: pages = pagination.find_all('a', href=True) total_pages = max([int(a.get_text()) for a in pages if a.get_text().isdigit()]) else: total_pages = 1 for page_number in range(1, total_pages + 1): index_url = f"{BASE_URL}?page={page_number}" response = requests.get(index_url) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') for a_tag in soup.find_all('a', href=True): href = urljoin(BASE_URL, a_tag['href']) if HDRI_URL_PATTERN.match(href): links.add(href) return list(links)Additional Tips:
-
Use the Sitemap: Many websites provide a sitemap (e.g.,
https://hdri-haven.com/sitemap.xml) that lists all available URLs. You can parse the sitemap to get all HDRI page URLs. -
Update for Redirects: Since
https://hdri-haven.comnow redirects tohttps://polyhaven.com, you may need to update the script to work with the new site structure. Adjust theBASE_URLand patterns accordingly. -
Respect Robots.txt: Always check
https://hdri-haven.com/robots.txtto see if the site allows scraping.
Revised Main Script:
Make sure to update the
BASE_URLand any other URLs or patterns to match the current site structure if the site has changed.import os import re import requests from bs4 import BeautifulSoup from PIL import Image from urllib.parse import urljoin from tqdm import tqdm import zipfile import time import random # Constants BASE_URL = "https://hdri-haven.com/hdris/" HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+") # Directories DOWNLOAD_DIR = "downloads" WEBP_DIR = "webp_images" def create_directories(): os.makedirs(DOWNLOAD_DIR, exist_ok=True) os.makedirs(WEBP_DIR, exist_ok=True) def get_all_hdri_pages(): """ Scans all pages of the HDRI index to find all HDRI page URLs. """ links = set() response = requests.get(BASE_URL) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Find the total number of pages pagination = soup.find('div', class_='pagination') if pagination: pages = pagination.find_all('a', href=True) total_pages = max([int(a.get_text()) for a in pages if a.get_text().isdigit()]) else: total_pages = 1 for page_number in range(1, total_pages + 1): index_url = f"{BASE_URL}?page={page_number}" response = requests.get(index_url) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') for a_tag in soup.find_all('a', href=True): href = urljoin(BASE_URL, a_tag['href']) if HDRI_URL_PATTERN.match(href): links.add(href) return list(links) # Rest of your code remains unchanged if __name__ == "__main__": main()By making these changes, your script should now find and process all available HDRI images on the site.
-