Skip to content
The Chat Archive
gpt-4-0314· OpenAI WildChat-4.8M (AI2)

用Python3写一个可以跳过拖动式验证码的爬虫脚本。

  1. User

    用Python3写一个可以跳过拖动式验证码的爬虫脚本。

  2. Assistant

    跳过拖动式验证码的难度较大,需要对验证码进行图像识别、特征提取等操作。但可以借助现有的库,如使用selenium操作浏览器模拟人类操作,结合图像识别库opencv进行图片处理和特征提取。下面是一个简单的模板供参考:

    import time
    import cv2
    import numpy as np
    from io import BytesIO
    from PIL import Image
    from selenium import webdriver
    from selenium.webdriver.common.action_chains import ActionChains
    
    def getCaptchaImage(driver):
        # 获取验证码图片
        captcha_image_element = driver.find_element_by_xpath("//xpath/of/captcha/image")
        captcha_image_location = captcha_image_element.location
        captcha_image_size = captcha_image_element.size
        driver.save_screenshot("screenshot.png")
        
        screenshot_image = Image.open("screenshot.png")
    
        left = captcha_image_location["x"]
        top = captcha_image_location["y"]
        right = left + captcha_image_size["width"]
        bottom = top + captcha_image_size["height"]
    
        captcha_image = screenshot_image.crop((left, top, right, bottom))
        captcha_image.save("captcha_image.png")
        return "captcha_image.png"
    
    def processCaptchaImage(captcha_image_path):
        # 根据验证码图片进行特征提取,并计算拖动距离
        captcha_image = cv2.imread(captcha_image_path)
        # 示例:处理验证码图片,提取特征,计算拖动距离
        # 使用合适的图像处理算法(如Canny边缘检测等)
        # ...
        distance = 0
        # 返回拖动距离
        return distance
    
    def dragSlider(driver, distance):
        # 拖动滑块
        slider = driver.find_element_by_xpath("//xpath/of/slider")
        action_chains = ActionChains(driver)
    
        action_chains.click_and_hold(slider).perform()
        time.sleep(0.5)
        action_chains.move_by_offset(distance, 0).perform()
        action_chains.release().perform()
    
    def main():
        url = "https://www.example.com/your-url-with-slider-captcha"
    
        options = webdriver.ChromeOptions()
        options.add_argument("--start-maximized")
        driver = webdriver.Chrome(options=options)
        driver.get(url)
    
        captcha_image_path = getCaptchaImage(driver)
        distance = processCaptchaImage(captcha_image_path)
        dragSlider(driver, distance)
    
        # 可以在此处添加代码以获取后续网页源码或其他操作
    
        driver.quit()
    
    if __name__ == "__main__":
        main()
    

    注意:由于拖动式验证码的难度较高,上面的代码只是一个大致的框架。你需要根据实际网站对验证码图片进行特征提取并计算拖动距离。另外,由于不同网站实现的验证码不尽相同,代码中的XPath选择器可能也需要进行适当调整。

  3. User

    用Python3写一个爬取 http://q.10jqka.com.cn/gn/ 这个网页中,所有概念以及成分股的爬虫

  4. Assistant

    可以使用requests库发送请求和BeautifulSoup库解析HTML。以下是一个简单的示范脚本:

    import requests
    from bs4 import BeautifulSoup
    
    def fetch_concepts_list(url):
        response = requests.get(url)
        soup = BeautifulSoup(response.content, "html.parser")
        concepts_list = []
    
        # 天天基金数据源的 CSS 选择器
        links = soup.select("div[class='secMenu hoverChild'] > a[href*='http://q.10jqka.com.cn/gn/']")
    
        # 提取链接值中的概念并保存
        for link in links:
            concepts_list.append(link.string)
    
        return concepts_list
    
    def fetch_stocks_for_concept(url):
        response = requests.get(url)
        soup = BeautifulSoup(response.content, "html.parser")
        stocks = []
    
        # 获取包含股票代码和股票名称的表格数据
        table_rows = soup.select("table[class='m-table'] > tbody > tr")
    
        for row in table_rows:
            row_data = row.find_all("td")
            stock_code = row_data[0].text.strip()
            stock_name = row_data[1].text.strip()
            stocks.append({
                "stock_code": stock_code,
                "stock_name": stock_name
            })
    
        return stocks
    
    def main():
        base_url = "http://q.10jqka.com.cn/gn/"
        concepts_stocks = {}
    
        concepts_list = fetch_concepts_list(base_url)
    
        # 生成详细的概念分类链接
        for concept in concepts_list:
            concept_stocks_url = f"{base_url}detail/code/{concept}/"
            stocks_list = fetch_stocks_for_concept(concept_stocks_url)
            concepts_stocks[concept] = stocks_list
    
        # 打印结果
        for concept, stocks in concepts_stocks.items():
            print(f"{concept}:")
            for stock in stocks:
                print(f"{stock['stock_code']}, {stock['stock_name']}")
    
    if __name__ == "__main__":
        main()
    

    这个脚本将爬取 http://q.10jqka.com.cn/gn/ 的所有概念以及成分股,并以基金代码和名称的形式输出。请注意,这个脚本没有考虑分页和请求间隔等问题,而且网页结构可能会随时更改,如果运行时出现问题,请检查相应的网页结构或添加分页处理功能。