def searchsimilarproducts(productinfo, numresults=5): similarproducts = [] searchquery =…
- User
def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ]
headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Поиск товаров на Wildberries if 'wildberries.ru' in url: products = soup.find_all('div', class_=['product-card', 'j-card-item']) logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['goods-name', 'product-card__name']) price = product.find(['span', 'ins'], class_=['price-current', 'lower-price']) link = product.find('a', class_=['product-card__main', 'j-open-full-product-card']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Поиск товаров на Ozon elif 'ozon.ru' in url: products = soup.find_all('div', class_=['tile-root', 'x5-a']) logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['tsBody500Medium', 'tile-hover-target']) price = product.find(['span', 'div'], class_=['tsHeadline500Medium', 'c2h5']) link = product.find('a', class_=['tile-hover-target', 'tile-link']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Поиск товаро на Яндекс.Маркет elif 'market.yandex.ru' in url: products = soup.find_all('div', class_=['_2im8-', '_2S9MU', '_2jvJ4']) logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products[:num_results]: title = product.find(['h3', 'div'], class_=['_2UHry', '_3t7Is']) price = product.find(['span', 'div'], class_=['_3NaXx', '_33ZFz', '_2m5MZ']) link = product.find('a', class_=['_2f75n', '_3Lwc_']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") return similar_products[:num_results] исправь ошибки - Assistant
The code has a few issues that need fixing to make it functional and error-free. Here’s an updated version of the
search_similar_productsfunction with corrections and comments explaining changes:import requests from bs4 import BeautifulSoup from urllib.parse import quote_plus, urljoin import logging # Configure logger logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ] headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Searching for products on Wildberries if 'wildberries.ru' in url: products = soup.find_all('div', class_=['product-card', 'j-card-item']) logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['goods-name', 'product-card__name']) price = product.find(['span', 'ins'], class_=['price-current', 'lower-price']) link = product.find('a', class_=['product-card__main', 'j-open-full-product-card']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Searching for products on Ozon elif 'ozon.ru' in url: products = soup.find_all('div', class_=['tile-root', 'x5-a']) logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['tsBody500Medium', 'tile-hover-target']) price = product.find(['span', 'div'], class_=['tsHeadline500Medium', 'c2h5']) link = product.find('a', class_=['tile-hover-target', 'tile-link']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Searching for products on Yandex Market elif 'market.yandex.ru' in url: products = soup.find_all('div', class_=['_2im8-', '_2S9MU', '_2jvJ4']) logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products[:num_results]: title = product.find(['h3', 'div'], class_=['_2UHry', '_3t7Is']) price = product.find(['span', 'div'], class_=['_3NaXx', '_33ZFz', '_2m5MZ']) link = product.find('a', class_=['_2f75n', '_3Lwc_']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") # Return only the requested number of results return similar_products[:num_results] # Sample usage product_info = {'title': 'laptop'} results = search_similar_products(product_info) print(results)Key Changes:
-
Import Statements: Ensure
requests,BeautifulSoup,urljoin,quote_plusfromurllib.parse, andloggingmodules are imported. -
Logging Configuration: Added basic logging configuration for logging messages.
-
Check the HTML Parsing: Make sure that the class names for titles, prices, and links match the actual ones from the websites being scraped. HTML structure can change, so these parts might need updates over time.
-
Initialize and Return: Ensure that
similar_productsis initialized as an empty list and that the function returns only up tonum_results. -
Exception Handling: Improved with relevant messages to log any request failures or unexpected errors.
Remember to respect the terms of service of each website when scraping and consider potential frequency limits or usage restrictions.
-
- User
def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ]
headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Searching for products on Wildberries if 'wildberries.ru' in url: products = soup.find_all('div', class_=['product-card', 'j-card-item']) logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['goods-name', 'product-card__name']) price = product.find(['span', 'ins'], class_=['price-current', 'lower-price']) link = product.find('a', class_=['product-card__main', 'j-open-full-product-card']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Searching for products on Ozon elif 'ozon.ru' in url: products = soup.find_all('div', class_=['tile-root', 'x5-a']) logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products[:num_results]: title = product.find(['span', 'div'], class_=['tsBody500Medium', 'tile-hover-target']) price = product.find(['span', 'div'], class_=['tsHeadline500Medium', 'c2h5']) link = product.find('a', class_=['tile-hover-target', 'tile-link']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) # Searching for products on Yandex Market elif 'market.yandex.ru' in url: products = soup.find_all('div', class_=['_2im8-', '_2S9MU', '_2jvJ4']) logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products[:num_results]: title = product.find(['h3', 'div'], class_=['_2UHry', '_3t7Is']) price = product.find(['span', 'div'], class_=['_3NaXx', '_33ZFz', '_2m5MZ']) link = product.find('a', class_=['_2f75n', '_3Lwc_']) if title and price and link: similar_products.append({ 'title': title.text.strip(), 'price': price.text.strip(), 'url': urljoin(url, link['href']) }) logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") # Return only the requested number of results return similar_products[:num_results] сделай так что бы похожие товары действительно парсились а то похожих товаров 0 - Assistant
To ensure that the function effectively parses similar products and doesn't return zero results, you may need to address several issues. However, it's important to note that successful parsing heavily depends on the current HTML structure of the target websites, which can change over time.
Below are steps to refine the function, with some troubleshooting tips:
-
Verify Class Names and Tags: Make sure the HTML elements and classes you are targeting still exist on the websites. This usually requires manually inspecting the page source or using developer tools.
-
Network Requests: Ensure requests are not being blocked. Check if the website allows web scraping or consider implementing delays or using proxies to avoid being throttled or blocked.
-
BeautifulSoup Parsing: Verify that BeautifulSoup is correctly interpreting the document structure.
-
Logging and Testing: Use detailed logging to understand which part of the parsing might be failing.
Here's an updated version of the function with emphasis on checking class names:
import requests from bs4 import BeautifulSoup from urllib.parse import quote_plus, urljoin import logging import time # Configure logger logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ] headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') if 'wildberries.ru' in url: products = soup.find_all('div', class_='product-card__wrapper') logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products: title_tag = product.find('span', class_='goods-name') price_tag = product.find('ins', class_='price-current') link_tag = product.find('a', class_='product-card__link') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break elif 'ozon.ru' in url: products = soup.find_all('div', class_='tile-hover-target') logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products: title_tag = product.find('span', class_='tsBody500Medium') price_tag = product.find('span', class_='c2h5') link_tag = product.find('a', class_='tile-link') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break elif 'market.yandex.ru' in url: products = soup.find_all('div', class_='_2jvJ4') logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products: title_tag = product.find('h3', class_='_2UHry') price_tag = product.find('span', class_='_3NaXx') link_tag = product.find('a', class_='_2f75n') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") # To prevent being blocked for scraping time.sleep(1) # Return only the requested number of results return similar_products[:num_results] # Sample usage product_info = {'title': 'laptop'} results = search_similar_products(product_info) print(results)Additional Tips:
-
Regular Updates: Regularly check the websites for updates to the HTML structure. If you notice parsing failures, verify and update the class selectors in your code.
-
Explore API Solutions: If available, using official APIs for these services might be a more stable and reliable approach.
-
Respect Terms of Service: Always ensure your scraping activities comply with the terms of service of the websites you are targeting.
-
- User
def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ]
headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') if 'wildberries.ru' in url: products = soup.find_all('div', class_='product-card__wrapper') logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products: title_tag = product.find('span', class_='goods-name') price_tag = product.find('ins', class_='price-current') link_tag = product.find('a', class_='product-card__link') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break elif 'ozon.ru' in url: products = soup.find_all('div', class_='tile-hover-target') logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products: title_tag = product.find('span', class_='tsBody500Medium') price_tag = product.find('span', class_='c2h5') link_tag = product.find('a', class_='tile-link') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break elif 'market.yandex.ru' in url: products = soup.find_all('div', class_='_2jvJ4') logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products: title_tag = product.find('h3', class_='_2UHry') price_tag = product.find('span', class_='_3NaXx') link_tag = product.find('a', class_='_2f75n') if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) if len(similar_products) >= num_results: break logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") # To prevent being blocked for scraping time.sleep(1) # Return only the requested number of results return similar_products[:num_results] хорошо сделай так что бы находились просто анологичные товары - Assistant
To increase the likelihood of finding relevant or similar products using web scraping, it’s important to ensure you're using correct and accurate HTML selectors, as well as handling scenarios where data might not be structured uniformly across different platforms.
The goals are:
- Ensure that selectors match the current HTML structure of the target website.
- Implement fallback mechanisms in case selectors change or required elements are not found.
- Paginate the results if necessary, depending on how many results you need and the structure of the site.
Key Tips to Optimize Search for Similar Products:
- Verify HTML Structure Regularly: Check the HTML of the websites you're scraping regularly, as they can change.
- Optimize Queries: Refine the queries and headers used in requests to avoid bot detection.
- Use More Dynamic Selection: If class names change frequently, try using more resilient selection techniques like regular expressions or parent-child traversal based on other stable tags.
- Log the Intermediate Steps: For better troubleshooting, log what's being found at each parsing step.
Here's your improved function with some optimizations:
import requests from bs4 import BeautifulSoup from urllib.parse import quote_plus, urljoin import logging import time import re # Configure logger logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def search_similar_products(product_info, num_results=5): similar_products = [] search_query = quote_plus(product_info['title']) search_urls = [ f"https://www.wildberries.ru/catalog/0/search.aspx?search={search_query}", f"https://www.ozon.ru/search/?text={search_query}", f"https://market.yandex.ru/search?text={search_query}" ] headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } for url in search_urls: logger.info(f"Поиск на: {url}") try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') # Handling Wildberries parsing if 'wildberries.ru' in url: products = soup.find_all('div', class_=re.compile(r'product-card')) logger.info(f"Найдено продуктов на Wildberries: {len(products)}") for product in products[:num_results]: # Limit to num_results directly title_tag = product.find('span', class_=re.compile(r'goods-name')) price_tag = product.find('ins', class_=re.compile(r'price-current')) link_tag = product.find('a', class_=re.compile(r'product-card__link')) if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) # Handling Ozon parsing elif 'ozon.ru' in url: products = soup.find_all('div', class_=re.compile(r'tile-root')) logger.info(f"Найдено продуктов на Ozon: {len(products)}") for product in products[:num_results]: title_tag = product.find('span', class_=re.compile(r'tsBody500Medium')) price_tag = product.find('span', class_=re.compile(r'c2h5')) link_tag = product.find('a', class_=re.compile(r'tile-link')) if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) # Handling Yandex Market parsing elif 'market.yandex.ru' in url: products = soup.find_all('div', class_=re.compile(r'_2jvJ4')) logger.info(f"Найдено продуктов на Яндекс.Маркет: {len(products)}") for product in products[:num_results]: title_tag = product.find('h3', class_=re.compile(r'_2UHry')) price_tag = product.find('span', class_=re.compile(r'_3NaXx')) link_tag = product.find('a', class_=re.compile(r'_2f75n')) if title_tag and price_tag and link_tag: similar_products.append({ 'title': title_tag.text.strip(), 'price': price_tag.text.strip(), 'url': urljoin(url, link_tag['href']) }) logger.info(f"Найдено похожих товаров: {len(similar_products)}") except requests.RequestException as e: logger.warning(f"Ошибка при запросе к {url}: {str(e)}") except Exception as e: logger.error(f"Неожиданная ошибка при поиске похожих товаров на {url}: {str(e)}") # Respectful delay for scraping time.sleep(1) return similar_products[:num_results] # Sample usage product_info = {'title': 'laptop'} results = search_similar_products(product_info) print(results)Final Considerations:
- Selector Accuracy: Regular expressions within
re.compile()are used to allow some flexibility in matching class names. - Output Validation: Before attempting to access text or attributes, validate the presence of tags.
- Dynamic Adjustments: Regularly check and adjust class selectors as the site structures evolve.
- Respect Web Policies: Implement delays and check for robots.txt for compliance with the site’s scraping policies.