Si entra al principio en un link que tiene ficheros, el self.tovisit se quedará vació y…
- User
Si entra al principio en un link que tiene ficheros, el self.to_visit se quedará vació y el programa terminará cuando aun faltan muchos links por visitar. Ves el error?
"import requests from bs4 import BeautifulSoup import tldextract import time import random import sys from urllib.parse import urljoin, urlparse from colorama import Fore, Style, init from difflib import SequenceMatcher import requests_cache import textstat from collections import deque import re import threading import platform import os import logging from urllib.robotparser import RobotFileParser
Configurar el nivel de logging
logging.basicConfig(level=logging.INFO)
Para manejo de teclas en Windows y Unix
if platform.system() == "Windows": import msvcrt else: import termios import tty import select
FILE_EXTENSIONS = { '.pdf', '.docx', '.xlsx', '.pptx', '.jpg', '.jpeg', '.png', '.gif', '.zip', '.rar', '.7z', '.txt', '.csv', '.md', '.mp3', '.wav', '.mp4', '.avi', '.cs' }
init(autoreset=True)
requests_cache.install_cache('web_cache', expire_after=180, allowable_codes=(200,))
class WebNavigatorGame: def init(self, start_url, duration_seconds=60, keywords=None, max_omits=3, download_dir='downloaded_files'): self.start_url = self.normalize_url(start_url) self.domain = self.get_domain(self.start_url) self.duration = duration_seconds self.visited = set() self.saved = set() self.to_visit = deque() self.texts = [] self.cache = {} self.headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/112.0.0.0 Safari/537.36" ), "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate", "Connection": "keep-alive", } self.base_url = self.start_url.rstrip('/') self.robot_parser = self.init_robot_parser() self.keywords = keywords if keywords else []
self.stop_event = threading.Event() self.elapsed_time_event = threading.Event() self.omitted_links = set() self.omit_counts = {} self.max_omits = max_omits self.download_dir = download_dir os.makedirs(self.download_dir, exist_ok=True) if not os.access(self.download_dir, os.W_OK): print(Fore.RED + f"No hay permisos de escritura en el directorio de descarga: {self.download_dir}") sys.exit(1) self.downloaded_files = set() self.blocked_by_robots = set() def normalize_url(self, url): """Normalizar la URL eliminando fragmentos y parámetros.""" url = url.strip() parsed = urlparse(url) scheme = parsed.scheme or 'http' netloc = parsed.netloc path = parsed.path normalized_url = f"{scheme}://{netloc}{path}" return normalized_url def get_domain(self, url): extracted = tldextract.extract(url) domain = f"{extracted.domain}.{extracted.suffix}" return domain.lower() def init_robot_parser(self): robots_url = urljoin(self.start_url, "/robots.txt") rp = RobotFileParser() try: rp.set_url(robots_url) rp.read() print(Fore.GREEN + f"robots.txt cargado desde {robots_url}") except Exception as e: print(Fore.RED + f"Error al cargar robots.txt: {e}") rp = None return rp def can_fetch(self, url): """Ignorar robots.txt (solo para pruebas con permiso).""" return True # Ignorar robots.txt para pruebas. Restaurar verificación en producción. def get_page(self, url): if not self.can_fetch(url): return None for attempt in range(3): try: response = requests.get(url, headers=self.headers, timeout=10) response.raise_for_status() return response.text except requests.exceptions.Timeout: print(Fore.RED + f"Timeout al acceder a {url}. Reintentando...") except requests.RequestException as e: print(Fore.RED + f"Error al obtener {url}: {e}") break return None def extract_text(self, html_content): soup = BeautifulSoup(html_content, 'lxml') for script_or_style in soup(['script', 'style']): script_or_style.decompose() text = soup.get_text(separator=' ', strip=True) return text def is_valid_url(self, url): if self.get_domain(url) != self.domain: return False normalized_url = self.normalize_url(url).rstrip('/') base_url = self.base_url.rstrip('/') return normalized_url.startswith(base_url) def extract_center_links(self, html_content, base_url): soup = BeautifulSoup(html_content, 'html.parser') main_content = soup.find(['main', 'article']) if main_content is None: main_content = soup.find('div', {'id': 'content'}) if main_content is None: main_content = soup links = set() file_links = set() for link in main_content.find_all('a', href=True): href = link['href'] full_url = urljoin(base_url, href) normalized_url = self.normalize_url(full_url) parsed = urlparse(normalized_url) path = parsed.path _, ext = os.path.splitext(path) ext = ext.lower() if ext in FILE_EXTENSIONS: file_links.add(normalized_url) else: if self.is_valid_url(normalized_url): clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" links.add(clean_url) else: print(Fore.YELLOW + f"URL descartada por is_valid_url: {normalized_url}") return links, file_links # ... (resto del código permanece igual)Resto del código sin cambios, exceptuando ajustes en methods que llamen a can_fetch o is_valid_url.
def prioritize_links(self, links): """Priorizar enlaces que contengan las palabras clave especificadas.""" if not self.keywords: return list(links) prioritized = [] others = [] keyword_pattern = re.compile("|".join(self.keywords), re.IGNORECASE) for link in links: if keyword_pattern.search(link): prioritized.append(link) else: others.append(link) return prioritized + others def get_extracted_data(self, url): """Obtener datos extraídos de la caché o extraerlos si no están en caché.""" if url in self.cache: cached_data = self.cache[url] if 'file_downloaded' in cached_data: return None, set(), set([url]), False else: return cached_data.get('text'), cached_data.get('links'), cached_data.get('file_links'), False else: parsed = urlparse(url) path = parsed.path _, ext = os.path.splitext(path) ext = ext.lower() if ext in FILE_EXTENSIONS: success = self.download_file(url) if success: self.cache[url] = {'file_downloaded': True} return None, set(), set([url]), True else: return None, set(), set(), True else: html_content = self.get_page(url) if html_content is None: self.cache[url] = {'text': None, 'links': set(), 'file_links': set()} return None, set(), set(), True text = self.extract_text(html_content) links, file_links = self.extract_center_links(html_content, url) prioritized_links = self.prioritize_links(links) self.cache[url] = {'text': text, 'links': set(prioritized_links), 'file_links': set(file_links)} return text, set(prioritized_links), set(file_links), True def download_file(self, url): """Descargar el fichero desde la URL y guardarlo.""" if url in self.downloaded_files: print(Fore.BLUE + f"Fichero ya descargado: {url}") return False if not self.can_fetch(url): return False print(Fore.MAGENTA + f"Descargando fichero: {url}") try: response = requests.get(url, headers=self.headers, stream=True, timeout=20) response.raise_for_status() parsed = urlparse(url) filename = os.path.basename(parsed.path) if not filename: filename = f"downloaded_file_{int(time.time())}" file_path = os.path.join(self.download_dir, filename) with open(file_path, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): if chunk: f.write(chunk) print(Fore.GREEN + f"Fichero descargado y guardado en: {file_path}") self.downloaded_files.add(url) return True except requests.RequestException as e: print(Fore.RED + f"Error al descargar el fichero {url}: {e}") return False def is_text_unique(self, new_text): """Verificar unicidad del texto.""" for old_text in self.texts: similarity = SequenceMatcher(None, new_text, old_text).ratio() if similarity > 0.5: print(Fore.YELLOW + f"El texto es similar a uno existente en un {similarity * 100:.2f}%.") return False return True def is_text_readable(self, text, threshold=60): """Verificar legibilidad del texto.""" if not text: return False score = textstat.flesch_reading_ease(text) print(f"Flesch Reading Ease score: {score}") if score >= threshold: return True readable_chars_pattern = r'[a-zA-Z0-9\s+=\-*/%<>;:{}()\[\].,"\']' total_chars = len(text) readable_chars = len(re.findall(readable_chars_pattern, text)) readable_percentage = (readable_chars / total_chars) * 100 if total_chars > 0 else 0 print(f"Porcentaje de caracteres legibles: {readable_percentage:.2f}%") return readable_percentage >= 90 def run(self): """Ejecutar el crawler.""" start_time = time.time() last_print_time = start_time current_url = self.start_url self.to_visit.append(current_url) while not self.stop_event.is_set() and (time.time() - start_time < self.duration) and self.to_visit: current_time = time.time() elapsed = current_time - start_time if current_time - last_print_time >= 10: print(Fore.YELLOW + f"Tiempo transcurrido: {int(elapsed)} segundos") last_print_time = current_time current_url = self.to_visit.popleft() if current_url in self.visited: continue print(Fore.CYAN + f"\nVisitando: {current_url}") text, links, file_links, request_made = self.get_extracted_data(current_url) if file_links: for file_url in file_links: if file_url not in self.visited: print(Fore.BLUE + f"Procesando enlace a fichero: {file_url}") self.visited.add(file_url) self.omitted_links.add(file_url) self.visited.add(current_url) continue if text is None and not file_links: self.visited.add(current_url) continue if not self.is_text_readable(text) and current_url != self.start_url: print(Fore.YELLOW + "El texto no es legible. Extrayendo solo enlaces de esta URL.") new_links = links - self.visited self.add_to_visit(new_links) self.omitted_links.add(current_url) self.omit_counts[current_url] = self.omit_counts.get(current_url, 0) + 1 if self.omit_counts[current_url] < self.max_omits: self.to_visit.append(current_url) print(Fore.BLUE + f"Reintentando {current_url} (Intento {self.omit_counts[current_url]})") else: print(Fore.RED + f"Máximo de omisiones alcanzado para {current_url}. No se procesará nuevamente.") self.visited.add(current_url) continue if request_made: if self.is_text_unique(text): self.visited.add(current_url) self.saved.add(current_url) self.texts.append(text) print(Fore.GREEN + f"Texto extraído (primeros 200 caracteres):\n{text[:200]}...\n") else: print(Fore.YELLOW + "El texto no es lo suficientemente único. Omitiendo extracción de texto para esta URL.") new_links = links - self.visited self.add_to_visit(new_links) self.omitted_links.add(current_url) self.omit_counts[current_url] = self.omit_counts.get(current_url, 0) + 1 if self.omit_counts[current_url] < self.max_omits: self.to_visit.append(current_url) print(Fore.BLUE + f"Reintentando {current_url} (Intento {self.omit_counts[current_url]})") else: print(Fore.RED + f"Máximo de omisiones alcanzado para {current_url}. No se procesará nuevamente.") self.visited.add(current_url) continue else: print(Fore.BLUE + f"Revisitado {current_url}. Usando datos en caché.\n") new_links = links - self.visited self.add_to_visit(new_links) if request_made: print(Fore.MAGENTA + "Esperando 1 segundo antes de la siguiente solicitud...\n") time.sleep(1) # Finalizar la búsqueda print(Fore.MAGENTA + f"\nBúsqueda finalizada! Textos guardados de {len(self.saved)} páginas únicas.") self.save_texts() self.save_omitted_links() self.save_downloaded_files() self.save_blocked_by_robots() def add_to_visit(self, new_links): """Agregar nuevos enlaces a visitar.""" filtered_links = set() for link in new_links: if link not in self.visited and link not in self.to_visit and self.is_valid_url(link): filtered_links.add(link) if filtered_links: self.to_visit.extend(filtered_links) random.shuffle(self.to_visit) else: print(Fore.YELLOW + "No se encontraron nuevos enlaces en esta página.") def save_texts(self, filename="extracted_texts.txt"): """Guardar todos los textos extraídos en un archivo.""" try: with open(filename, 'w', encoding='utf-8') as f: for idx, text in enumerate(self.texts, 1): f.write(f"--- Página {idx} ---\n") f.write(text + "\n\n") print(Fore.BLUE + f"Todos los textos han sido guardados en {filename}.") except Exception as e: print(Fore.RED + f"Error al guardar los textos: {e}") def save_omitted_links(self, filename="omitted_links.txt"): """Guardar los enlaces omitidos.""" try: with open(filename, 'w', encoding='utf-8') as f: for idx, url in enumerate(self.omitted_links, 1): f.write(f"{idx}. {url}\n") print(Fore.BLUE + f"Enlaces omitidos han sido guardados en {filename}.") except Exception as e: print(Fore.RED + f"Error al guardar los enlaces omitidos: {e}") def save_downloaded_files(self, filename="downloaded_files.txt"): """Guardar los enlaces de los ficheros descargados.""" try: with open(filename, 'w', encoding='utf-8') as f: for idx, url in enumerate(self.downloaded_files, 1): f.write(f"{idx}. {url}\n") print(Fore.BLUE + f"Enlaces de ficheros descargados han sido guardados en {filename}.") except Exception as e: print(Fore.RED + f"Error al guardar los enlaces de ficheros descargados: {e}") def save_blocked_by_robots(self, filename="blocked_by_robots.txt"): """Guardar las URLs bloqueadas por robots.txt.""" try: with open(filename, 'w', encoding='utf-8') as f: for idx, url in enumerate(self.blocked_by_robots, 1): f.write(f"{idx}. {url}\n") print(Fore.BLUE + f"URLs bloqueadas por robots.txt han sido guardadas en {filename}.") except Exception as e: print(Fore.RED + f"Error al guardar las URLs bloqueadas por robots.txt: {e}") def listen_for_keypress(self): """Escuchar la pulsación de la tecla 'z' para detener la búsqueda.""" if platform.system() == "Windows": while not self.stop_event.is_set(): if msvcrt.kbhit(): key = msvcrt.getch() if key.lower() == b'z': print(Fore.RED + "\nTecla 'z' presionada. Finalizando la búsqueda...") self.stop_event.set() break time.sleep(0.1) else: def is_data(): return select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], []) old_settings = termios.tcgetattr(sys.stdin) try: tty.setcbreak(sys.stdin.fileno()) while not self.stop_event.is_set(): if is_data(): key = sys.stdin.read(1) if key.lower() == 'z': print(Fore.RED + "\nTecla 'z' presionada. Finalizando la búsqueda...") self.stop_event.set() break time.sleep(0.1) finally: termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)if name == "main": if len(sys.argv) < 2: start_url = "https://github.com/adityaiiitv/Breakout/" duration = 3000 #sys.exit(1)
keywords = [] """"" start_url = sys.argv[1] try: duration = int(sys.argv[2]) except ValueError: print("La duración debe ser un entero que representa segundos.") sys.exit(1) # Obtener palabras clave opcionales keywords = [] if len(sys.argv) > 3: keywords = sys.argv[3].split(',') """"" game = WebNavigatorGame(start_url, duration, keywords) # Iniciar el hilo que escucha las pulsaciones de teclas listener_thread = threading.Thread(target=game.listen_for_keypress, daemon=True) listener_thread.start() # Ejecutar el crawler game.run() " - Assistant
Yes, I see the issue you're pointing out. When the initial URL provided is a direct link to a file (e.g., a PDF, image, or any resource with an extension listed in
FILE_EXTENSIONS), the crawler processes this file, adds it to thevisitedset, and then exits becauseself.to_visitbecomes empty. This happens even though there might be many more links on the same domain that could be visited. The crawler doesn't add any new URLs toself.to_visitin this scenario, leading to premature termination.The root cause of the problem:
- File URLs don't yield new links: When the crawler processes a file URL, it downloads the file but doesn't extract any new links to add to
self.to_visitsince files (like PDFs or images) typically don't contain HTML links. - Starting with a file URL leaves
self.to_visitempty: If the starting URL is a file,self.to_visitonly contains this URL. After processing it,self.to_visitbecomes empty, and the crawler has no further URLs to visit, causing it to exit.
How to fix the issue:
To address this, we need to ensure that the crawler has a way to discover additional URLs even when starting with a file link. Here are some possible solutions:
-
Modify the crawler to retrieve and add the parent page or base domain links when starting from a file URL.
-
Approach:
- When the crawler detects that the starting URL is a file, it should determine the parent URL (e.g., the directory containing the file or the base domain).
- Fetch the parent URL's page content and extract links from it.
- Add these links to
self.to_visitso that the crawler has more URLs to process.
-
Implementation:
Add a function to get the parent URL:
def get_parent_url(self, url): parsed = urlparse(url) if parsed.path in ['', '/']: return None # No parent URL available path = os.path.dirname(parsed.path) parent_url = f"{parsed.scheme}://{parsed.netloc}{path}" return parent_urlModify the
runmethod to handle the initial file URL case:def run(self): """Execute the crawler.""" start_time = time.time() last_print_time = start_time # Check if the start URL is a file parsed_start_url = urlparse(self.start_url) _, ext = os.path.splitext(parsed_start_url.path) ext = ext.lower() if ext in FILE_EXTENSIONS: print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.") parent_url = self.get_parent_url(self.start_url) if parent_url: print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.") self.to_visit.append(parent_url) else: print(Fore.RED + "No parent URL found. Exiting crawler.") return else: self.to_visit.append(self.start_url) # Rest of the run method remains the same
-
-
Enforce that the starting URL must be a webpage, not a file.
-
Approach:
- Before running the crawler, check if the starting URL ends with a file extension listed in
FILE_EXTENSIONS. - If it does, prompt the user or raise an error to provide a valid starting webpage URL.
- Before running the crawler, check if the starting URL ends with a file extension listed in
-
Implementation:
In your
__init__method or before starting the crawler, add:def __init__(self, start_url, ...): # existing initialization code # Check if the start URL is a file parsed_start_url = urlparse(start_url) _, ext = os.path.splitext(parsed_start_url.path) ext = ext.lower() if ext in FILE_EXTENSIONS: raise ValueError("The starting URL cannot be a direct link to a file. Please provide a webpage URL.")
-
-
Allow the crawler to proceed to the base domain when starting from a file URL.
-
Approach:
- Adjust the crawler to recognize when it's starting from a file URL and automatically set the base domain URL as the next URL to visit.
-
Implementation:
In the
runmethod:def run(self): """Execute the crawler.""" start_time = time.time() last_print_time = start_time # Check if the start URL is a file parsed_start_url = urlparse(self.start_url) _, ext = os.path.splitext(parsed_start_url.path) ext = ext.lower() if ext in FILE_EXTENSIONS: base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/" print(Fore.YELLOW + f"Start URL is a file. Switching to base domain: {base_domain_url}") self.to_visit.append(base_domain_url) else: self.to_visit.append(self.start_url) # Rest of the run method remains the same
-
Recommendation:
Option 1 is a more flexible solution because it allows the crawler to move up one level and attempt to find links on the parent page of the file. If the parent page is not available or doesn't exist, the crawler can then decide to exit gracefully or prompt the user.
Here's how you can implement option 1 in your code:
def get_parent_url(self, url): parsed = urlparse(url) if parsed.path in ['', '/']: return None # No parent URL available # Remove the last segment of the path path = os.path.dirname(parsed.path.rstrip('/')) parent_url = f"{parsed.scheme}://{parsed.netloc}{path}" return parent_url def run(self): """Execute the crawler.""" start_time = time.time() last_print_time = start_time # Check if the start URL is a file parsed_start_url = urlparse(self.start_url) _, ext = os.path.splitext(parsed_start_url.path) ext = ext.lower() if ext in FILE_EXTENSIONS: print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.") parent_url = self.get_parent_url(self.start_url) if parent_url and parent_url != self.start_url: print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.") self.to_visit.append(parent_url) else: # If no parent URL, use the base domain base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/" print(Fore.YELLOW + f"No parent URL found. Using base domain: {base_domain_url} as starting point.") self.to_visit.append(base_domain_url) else: self.to_visit.append(self.start_url) # Rest of the run method remains the sameWith this implementation:
- If the start URL is a file, the crawler attempts to find the parent directory URL.
- If a parent URL is found, it's added to
self.to_visit. - If not, the crawler defaults to using the base domain URL.
- This ensures that
self.to_visitis not empty, and the crawler has URLs to process after handling the initial file URL.
Testing the solution:
To test this, you can run the crawler with a starting URL that points directly to a file, such as a PDF or image. For example:
python crawler.py https://example.com/path/to/file.pdfEnsure that the crawler outputs messages indicating that it recognized the start URL as a file and is attempting to retrieve the parent URL or defaulting to the base domain.
Additional considerations:
- Error handling: Ensure that the crawler handles cases where the parent URL or base domain may not be accessible or may not have any links to follow.
- Circular references: Be cautious of potential infinite loops if the crawler keeps adding parent URLs that have already been visited. Make sure you check if the parent URL is already in
self.visited. - Politeness and robots.txt: Since you're adding new URLs to crawl, ensure that you're respecting the
robots.txtrules for those URLs, especially if the base domain is being used.
Updated code snippet with the fix:
Here's the modified
runmethod with comments:def run(self): """Execute the crawler.""" start_time = time.time() last_print_time = start_time # Initialization of current URL current_url = self.start_url # Check if the start URL is a file parsed_start_url = urlparse(self.start_url) _, ext = os.path.splitext(parsed_start_url.path) ext = ext.lower() if ext in FILE_EXTENSIONS: print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.") parent_url = self.get_parent_url(self.start_url) if parent_url and parent_url != self.start_url: print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.") self.to_visit.append(parent_url) else: # If no parent URL, use the base domain base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/" print(Fore.YELLOW + f"No parent URL found. Using base domain: {base_domain_url} as starting point.") self.to_visit.append(base_domain_url) else: self.to_visit.append(self.start_url) while not self.stop_event.is_set() and (time.time() - start_time < self.duration) and self.to_visit: current_time = time.time() elapsed = current_time - start_time if current_time - last_print_time >= 10: print(Fore.YELLOW + f"Elapsed time: {int(elapsed)} seconds") last_print_time = current_time current_url = self.to_visit.popleft() if current_url in self.visited: continue print(Fore.CYAN + f"\nVisiting: {current_url}") text, links, file_links, request_made = self.get_extracted_data(current_url) if file_links: for file_url in file_links: if file_url not in self.visited: print(Fore.BLUE + f"Processing file link: {file_url}") self.visited.add(file_url) self.omitted_links.add(file_url) self.visited.add(current_url) continue if text is None and not file_links: self.visited.add(current_url) continue # Rest of your existing code...Summary:
- The issue arises when the crawler starts with a file URL, leading to
self.to_visitbecoming empty after processing the file. - To fix this, adjust the crawler to fetch and add the parent URL or base domain links when starting from a file URL.
- Implementing this change ensures that the crawler has additional URLs to visit and doesn't terminate prematurely.
- File URLs don't yield new links: When the crawler processes a file URL, it downloads the file but doesn't extract any new links to add to