Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· openaiHugging Face Suzhen/CodeChat

import tkinter as tk from tkinter import messagebox, filedialog, scrolledtext import os…

  1. User

    import tkinter as tk from tkinter import messagebox, filedialog, scrolledtext import os import re import time import requests from io import BytesIO from PIL import Image, UnidentifiedImageError from selenium import webdriver from msedge.selenium_tools import EdgeOptions from concurrent.futures import ThreadPoolExecutor from urllib.parse import urljoin import threading

    def print_to_text_widget(text): def _print_to_text_widget(): output_text.config(state="normal") output_text.insert(tk.END, text + "\n") output_text.see(tk.END) output_text.config(state="disabled")

    root.after(0, _print_to_text_widget)
    

    def download_images(): # 获取网址输入框的内容 url = url_entry.get() num = num_entry.get()

    int_num = int(num)
    
    # 获取文件夹路径输入框的内容
    folder_path = folder_path_entry.get()
    
    if not url or not folder_path or not num or int_num < 0:
        messagebox.showerror("错误", "请输入网址和文件夹路径")
        return
    
    # 初始化 Edge WebDriver
    options = EdgeOptions()
    options.use_chromium = True
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    driver = webdriver.Edge(
        executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe"
    )  # 使用你的驱动程序路径
    
    print_to_text_widget("打开网页...")
    
    # 创建用于存储图片的目录
    driver.get(url)
    title = driver.title
    # Remove illegal characters
    title = re.sub(r'[\\/:*?"<>|]', "", title)
    
    # Create a new directory with the name of the website
    folder_name = os.path.join(folder_path, title)
    os.makedirs(folder_name, exist_ok=True)
    
    if int_num == 0:
        int_num = 1
    for i in range(1, int_num + 1):
        # 访问网页
        if i == 1:
            urltemp = f"{i}/"
            url = url + urltemp
        if 10 >= i > 1:
            urltemp = f"{i}/"
            url = url[:-2]
            url = url + urltemp
        if 100 >= i > 10:
            urltemp = f"{i}/"
            url = url[:-3]
            url = url + urltemp
        if 1000 >= i > 100:
            urltemp = f"{i}/"
            url = url[:-4]
            url = url + urltemp
        if 10000 >= i > 1000:
            urltemp = f"{i}/"
            url = url[:-5]
            url = url + urltemp
    
        driver.get(url)
    
        if driver.title.count("404 -") > 0:
            break
    
        # 等待网页加载完成
        driver.implicitly_wait(2)
    
        # Create a function to scroll to the bottom of the page
        def scroll_to_bottom(driver):
            old_position = 0
            new_position = None
    
            while new_position != old_position:
                old_position = driver.execute_script("return window.pageYOffset;")
                time.sleep(0.5)
                new_position = driver.execute_script(
                    "window.scrollTo(0, document.body.scrollHeight);"
                    "return window.pageYOffset;"
                )
    
        # Scroll to the bottom of the page to make sure all images are loaded
        scroll_to_bottom(driver)
    
        # 获取所有图片标签
        img_tags = driver.find_elements_by_tag_name("img")
    
        # Define a function for downloading and saving an image
        def download_image(img_tag, img_index):
            # 获取图片 URL
            img_url = img_tag.get_attribute("src")
            if not img_url:
                return
    
            # 将相对 URL 转换为绝对 URL
            img_url = urljoin(url, img_url)
    
            try:
                # 发送 GET 请求并获取图片数据
                response = requests.get(img_url, timeout=5)
            except (requests.exceptions.RequestException, requests.exceptions.Timeout):
                return
    
            # 创建用于存储图片的文件名
            img_filename = os.path.join(folder_name, f"image_{i}")
    
            # 打开图片文件并检查格式
            try:
                img_data = response.content
                img = Image.open(BytesIO(img_data))
                img_ext = img.format.lower()
    
                # Filter out GIFs and images smaller than             # 600x600
                if img_ext == "gif" or img.size < (600, 600):
                    return
            except (OSError, UnidentifiedImageError):
                return
            if img_ext == "jpeg":
                img_ext = "jpg"
            if not img_filename.endswith(img_ext):
                img_filename += f".{img_ext}"
    
            # 保存图片到本地磁盘
            with open(img_filename, "wb") as f:
                f.write(img_data)
    
            print_to_text_widget(f"下载图片 {img_filename} 完成")
    
        # Use ThreadPoolExecutor to download images in parallel
        with ThreadPoolExecutor(max_workers=5) as executor:
            for index, img_tag in enumerate(img_tags):
                executor.submit(download_image, img_tag, index)
    
    print_to_text_widget("所有图片下载完成!")
    
    # 关闭浏览器
    driver.quit()
    

    def download_images_in_new_thread(): # 创建一个新的线程来执行耗时的下载操作 threading.Thread(target=download_images).start()

    def select_folder(): folder_path = filedialog.askdirectory() folder_path_entry.delete(0, tk.END) folder_path_entry.insert(tk.END, folder_path)

    创建主窗口

    root = tk.Tk()

    创建网址输入框和对应的标签

    url_label = tk.Label(root, text="网址") url_label.pack() url_entry = tk.Entry(root, width=50) url_entry.pack()

    图片数量

    num_label = tk.Label(root, text="数量") num_label.pack() num_entry = tk.Entry(root, width=10) num_entry.pack()

    创建文件夹路径输入框和对应的标签

    folder_path_label = tk.Label(root, text="文件夹路径") folder_path_label.pack() folder_path_entry = tk.Entry(root, width=50) folder_path_entry.pack()

    创建选择

    文件夹按钮

    select_folder_button = tk.Button(root, text="选择文件夹", command=select_folder) select_folder_button.pack()

    创建开始下载按钮

    download_button = tk.Button(root, text="开始下载", command=download_images_in_new_thread) download_button.pack()

    创建一个 Text widget 用于显示打印信息

    output_text = scrolledtext.ScrolledText(root, state="disabled", width=70, height=10) output_text.pack()

    启动消息循环

    root.mainloop()

  2. Assistant

    This code is a simple Python script using the tkinter library for creating a basic graphical user interface (GUI) to download images from a given URL. The GUI contains input fields for the user to specify the URL and target folder for the downloaded images.

    The program is using Selenium to fetch the HTML content of the input URL and extract all image (img) elements. It uses Python's Requests library to download image data, and the Python Imaging Library (PIL) to identify, filter, and save the images to the specified folder.

    Here is a brief overview of the code:

    1. Import the necessary libraries.
    2. Create print_to_text_widget function to print the text to the tkinter Text widget.
    3. Create the download_images function to download the images from the specified URL.
    4. Use Selenium to automate a browser and navigate to the given URL.
    5. Scroll down to the bottom of the page to ensure all images have been loaded.
    6. Extract all image elements using Selenium's find_elements_by_tag_name method.
    7. Process and save images using the Requests library, and filter GIFs and small images using the PIL library.
    8. Implement a GUI using tkinter for user input and output.
    9. Create a download_images_in_new_thread function to call the download_images function in a separate thread.
    10. Create a select_folder function to open a folder directory dialog.
    11. Create GUI elements (buttons, input fields) and place them on the window.
    12. Launch the tkinter event loop to run the application.

    To run this program, first, make sure that you have Python 3 installed on your system. Then, install the required libraries (if not already installed) by running the following command:

    pip install selenium msedge-selenium-tools pillow requests tkinter
    

    Then, download the suitable Selenium WebDriver for your browser (in this code it uses Edge Browser WebDriver), and update the executable_path in the following statement with the path where you have placed the msedgedriver.exe file:

    driver = webdriver.Edge(executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe") 
    

    Finally, save the file with a .py extension (e.g., image_downloader.py) and run the script using the command:

    python image_downloader.py
    
  3. User

    按顺序下载每个网页的图太慢了,可以一次打开多个网页同时下载吗?

  4. Assistant

    是的,您可以使用Python的concurrent.futures.ThreadPoolExecutor类同时下载多个页面的图像。以下是根据您的要求修改后的download_images函数。它使用了一个名为download_page_images的新函数,该函数接受一个网址和索引,用于打开Web页面并下载该页面上的图像。

    def download_page_images(url, index, folder_path):
        options = EdgeOptions()
        options.use_chromium = True
        options.add_argument("--headless")
        options.add_argument("--disable-gpu")
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-dev-shm-usage")
        driver = webdriver.Edge(
            executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe"
        )  # 使用你的驱动程序路径
    
        driver.get(url)
    
        if driver.title.count("404 -") > 0:
            driver.quit()
            return
    
        # 等待网页加载完成
        driver.implicitly_wait(2)
    
        # Scroll to the bottom of the page to make sure all images are loaded
        scroll_to_bottom(driver)
    
        # 获取所有图片标签
        img_tags = driver.find_elements_by_tag_name("img")
    
        # Download images as described earlier
        with ThreadPoolExecutor(max_workers=5) as executor:
            for img_index, img_tag in enumerate(img_tags):
                executor.submit(download_image, img_tag, img_index)
    
        driver.quit()
    
    
    def download_images():
        # ...
        num_of_pages = 10  # Set the number of pages to download images simultaneously
    
        with ThreadPoolExecutor(max_workers=num_of_pages) as executor:
            for i in range(1, int(num) + 1):
                temp_url = url + f"{i}/"
                executor.submit(download_page_images, temp_url, i, folder_path)
    
        print_to_text_widget("所有图片下载完成!")
    

    需要注意的是,这种方法可能使服务器对您的IP采取限制措施,因为同时发送的请求数量可能引发服务器的防爬取系统。在实际开发中,可以通过添加计时器和更改请求的User-Agent来减慢请求速度和模拟不同的浏览器,以规避一些基本的防爬措施。

  5. User

    写出修改后完整代码

  6. Assistant

    以下是根据您的要求修改后的完整代码:

    import tkinter as tk
    from tkinter import messagebox, filedialog, scrolledtext
    import os
    import re
    import time
    import requests
    from io import BytesIO
    from PIL import Image, UnidentifiedImageError
    from selenium import webdriver
    from msedge.selenium_tools import EdgeOptions
    from concurrent.futures import ThreadPoolExecutor
    from urllib.parse import urljoin
    import threading
    
    
    def print_to_text_widget(text):
        def _print_to_text_widget():
            output_text.config(state="normal")
            output_text.insert(tk.END, text + "\n")
            output_text.see(tk.END)
            output_text.config(state="disabled")
    
        root.after(0, _print_to_text_widget)
    
    
    def scroll_to_bottom(driver):
        old_position = 0
        new_position = None
    
        while new_position != old_position:
            old_position = driver.execute_script("return window.pageYOffset;")
            time.sleep(0.5)
            new_position = driver.execute_script(
                "window.scrollTo(0, document.body.scrollHeight);"
                "return window.pageYOffset;"
            )
    
    
    def download_image(img_tag, img_index):
        img_url = img_tag.get_attribute("src")
        if not img_url:
            return
    
        img_url = urljoin(url, img_url)
    
        try:
            response = requests.get(img_url, timeout=5)
        except (requests.exceptions.RequestException, requests.exceptions.Timeout):
            return
    
        try:
            img_data = response.content
            img = Image.open(BytesIO(img_data))
            img_ext = img.format.lower()
    
            if img_ext == "gif" or img.size < (600, 600):
                return
        except (OSError, UnidentifiedImageError):
            return
        if img_ext == "jpeg":
            img_ext = "jpg"
        if not img_filename.endswith(img_ext):
            img_filename += f".{img_ext}"
    
        with open(img_filename, "wb") as f:
            f.write(img_data)
    
        print_to_text_widget(f"下载图片 {img_filename} 完成")
    
    
    def download_page_images(url, index, folder_path):
        options = EdgeOptions()
        options.use_chromium = True
        options.add_argument("--headless")
        options.add_argument("--disable-gpu")
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-dev-shm-usage")
        driver = webdriver.Edge(
            executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe"
        )  # 使用你的驱动程序路径
    
        driver.get(url)
    
        if driver.title.count("404 -") > 0:
            driver.quit()
            return
    
        driver.implicitly_wait(2)
    
        scroll_to_bottom(driver)
    
        img_tags = driver.find_elements_by_tag_name("img")
    
        with ThreadPoolExecutor(max_workers=5) as executor:
            for img_index, img_tag in enumerate(img_tags):
                executor.submit(download_image, img_tag, img_index)
    
        driver.quit()
    
    
    def download_images():
        url = url_entry.get()
        num = num_entry.get()
        folder_path = folder_path_entry.get()
    
        if not url or not folder_path or not num or int(num) < 0:
            messagebox.showerror("错误", "请输入网址和文件夹路径")
            return
    
        driver.get(url)
        title = driver.title
        title = re.sub(r'[\\/:*?"<>|]', "", title)
    
        folder_name = os.path.join(folder_path, title)
        os.makedirs(folder_name, exist_ok=True)
    
        num_of_pages = int(num)
    
        with ThreadPoolExecutor(max_workers=num_of_pages) as executor:
            for i in range(1, num_of_pages + 1):
                temp_url = url + f"{i}/"
                executor.submit(download_page_images, temp_url, i, folder_path)
    
        print_to_text_widget("所有图片下载完成!")
    
    
    def download_images_in_new_thread():
        threading.Thread(target=download_images).start()
    
    def select_folder():
        folder_path = filedialog.askdirectory()
        folder_path_entry.delete(0, tk.END)
        folder_path_entry.insert(tk.END, folder_path)
    
    root = tk.Tk()
    
    url_label = tk.Label(root, text="网址")
    url_label.pack()
    url_entry = tk.Entry(root, width=50)
    url_entry.pack()
    
    num_label = tk.Label(root, text="数量")
    num_label.pack()
    num_entry = tk.Entry(root, width=10)
    num_entry.pack()
    
    folder_path_label = tk.Label(root, text="文件夹路径")
    folder_path_label.pack()
    folder_path_entry = tk.Entry(root, width=50)
    folder_path_entry.pack()
    
    select_folder_button = tk.Button(root, text="选择文件夹", command=select_folder)
    select_folder_button.pack()
    
    download_button = tk.Button(root, text="开始下载", command=download_images_in_new_thread)
    download_button.pack()
    
    output_text = scrolledtext.ScrolledText(root, state="disabled", width=70, height=10)
    output_text.pack()
    
    root.mainloop()
    

    请确保已经安装了必要的Python库,并使用您自己的msedgedriver路径。启动应用程序后,提供网址、文件夹路径以及想要下载的同时页面图像数量。

    注意:这个代码片段会打开和处理多个页面,这可能会要求网络和硬件资源,并可能会导致服务器对你的请求施加限制。在实际应用中,可能需要使用代理、改变User-Agent和/或增加计时器等方法以避免引起服务器的注意。

  7. User

    设置webdriver.Edge打开网页时不显示窗口

  8. Assistant

    要在运行代码时不显示浏览器窗口,您可以在EdgeOptions中添加--headless参数。 修改后的download_page_images函数如下所示:

    def download_page_images(url, index, folder_path):
        options = EdgeOptions()
        options.use_chromium = True
        options.add_argument("--headless")  # 添加此行以隐藏浏览器窗口
        options.add_argument("--disable-gpu")
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-dev-shm-usage")
        driver = webdriver.Edge(
            executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe"
        )  # 使用你的驱动程序路径
    
        # ... 保持其余代码不变
    

    确保保留options.add_argument("--headless")行,并确保未删除任何其他选项。在这种配置下运行代码时,将不会看到浏览器窗口。这在运行爬虫程序时特别有用,因为它消除了在桌面上打开和关闭多个浏览器窗口所带来的混乱。

  9. User

    浏览器窗口还是出现了

  10. Assistant

    很抱歉给您带来了麻烦。对于Microsoft Edge浏览器,目前在官方文档中未明确提到headless模式。不过Chrome和Edge浏览器都基于Chromium,因此我们可以尝试使用Chrome的方法。 请将以下语句替换download_page_images函数中的相应部分,并确保已安装了Microsoft Edge WebDriver,已将其添加到系统路径或已将其添加到executable_path。这里是应用了新版本 Edge 的修改后的代码:

    def download_page_images(url, index, folder_path):
        options = EdgeOptions()
        options.use_chromium = True
        options.add_argument("headless")
        options.add_argument("disable-gpu")
        options.add_argument("no-sandbox")
        options.add_argument("disable-dev-shm-usage")
        driver = webdriver.Edge(
            executable_path=r"D:\TOOLS\EdgeWeb\edgedriver_win64\msedgedriver.exe", options=options
        )  # 使用你的驱动程序路径
    
        # ... 保持其余代码不变
    

    确保要使用的 Microsoft Edge 版本是基于 Chromium 的最新版本。如果您仍然遇到问题,请使用 Chrome 浏览器运行此脚本,因为它已经在 Headless 模式下得到了很好的支持。要使用 Chrome,需要安装 ChromeDriver 并下载相应版本的 Chrome 浏览器。 修改后,请将 webdriver.Edge 更改为 webdriver.Chrome。

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.