Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

Si entra al principio en un link que tiene ficheros, el self.tovisit se quedará vació y…

  1. User

    Si entra al principio en un link que tiene ficheros, el self.to_visit se quedará vació y el programa terminará cuando aun faltan muchos links por visitar. Ves el error?

    "import requests from bs4 import BeautifulSoup import tldextract import time import random import sys from urllib.parse import urljoin, urlparse from colorama import Fore, Style, init from difflib import SequenceMatcher import requests_cache import textstat from collections import deque import re import threading import platform import os import logging from urllib.robotparser import RobotFileParser

    Configurar el nivel de logging

    logging.basicConfig(level=logging.INFO)

    Para manejo de teclas en Windows y Unix

    if platform.system() == "Windows": import msvcrt else: import termios import tty import select

    FILE_EXTENSIONS = { '.pdf', '.docx', '.xlsx', '.pptx', '.jpg', '.jpeg', '.png', '.gif', '.zip', '.rar', '.7z', '.txt', '.csv', '.md', '.mp3', '.wav', '.mp4', '.avi', '.cs' }

    init(autoreset=True)

    requests_cache.install_cache('web_cache', expire_after=180, allowable_codes=(200,))

    class WebNavigatorGame: def init(self, start_url, duration_seconds=60, keywords=None, max_omits=3, download_dir='downloaded_files'): self.start_url = self.normalize_url(start_url) self.domain = self.get_domain(self.start_url) self.duration = duration_seconds self.visited = set() self.saved = set() self.to_visit = deque() self.texts = [] self.cache = {} self.headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/112.0.0.0 Safari/537.36" ), "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate", "Connection": "keep-alive", } self.base_url = self.start_url.rstrip('/') self.robot_parser = self.init_robot_parser() self.keywords = keywords if keywords else []

        self.stop_event = threading.Event()
        self.elapsed_time_event = threading.Event()
        self.omitted_links = set()
        self.omit_counts = {}
        self.max_omits = max_omits
        self.download_dir = download_dir
        os.makedirs(self.download_dir, exist_ok=True)
    
        if not os.access(self.download_dir, os.W_OK):
            print(Fore.RED + f"No hay permisos de escritura en el directorio de descarga: {self.download_dir}")
            sys.exit(1)
    
        self.downloaded_files = set()
        self.blocked_by_robots = set()
    
    def normalize_url(self, url):
        """Normalizar la URL eliminando fragmentos y parámetros."""
        url = url.strip()
        parsed = urlparse(url)
        scheme = parsed.scheme or 'http'
        netloc = parsed.netloc
        path = parsed.path
        normalized_url = f"{scheme}://{netloc}{path}"
        return normalized_url
    
    def get_domain(self, url):
        extracted = tldextract.extract(url)
        domain = f"{extracted.domain}.{extracted.suffix}"
        return domain.lower()
    
    def init_robot_parser(self):
        robots_url = urljoin(self.start_url, "/robots.txt")
        rp = RobotFileParser()
        try:
            rp.set_url(robots_url)
            rp.read()
            print(Fore.GREEN + f"robots.txt cargado desde {robots_url}")
        except Exception as e:
            print(Fore.RED + f"Error al cargar robots.txt: {e}")
            rp = None
        return rp
    
    def can_fetch(self, url):
        """Ignorar robots.txt (solo para pruebas con permiso)."""
        return True  # Ignorar robots.txt para pruebas. Restaurar verificación en producción.
    
    def get_page(self, url):
        if not self.can_fetch(url):
            return None
    
        for attempt in range(3):
            try:
                response = requests.get(url, headers=self.headers, timeout=10)
                response.raise_for_status()
                return response.text
            except requests.exceptions.Timeout:
                print(Fore.RED + f"Timeout al acceder a {url}. Reintentando...")
            except requests.RequestException as e:
                print(Fore.RED + f"Error al obtener {url}: {e}")
                break
        return None
    
    def extract_text(self, html_content):
        soup = BeautifulSoup(html_content, 'lxml')
        for script_or_style in soup(['script', 'style']):
            script_or_style.decompose()
        text = soup.get_text(separator=' ', strip=True)
        return text
    
    def is_valid_url(self, url):
        if self.get_domain(url) != self.domain:
            return False
        normalized_url = self.normalize_url(url).rstrip('/')
        base_url = self.base_url.rstrip('/')
        return normalized_url.startswith(base_url)
    
    def extract_center_links(self, html_content, base_url):
        soup = BeautifulSoup(html_content, 'html.parser')
    
        main_content = soup.find(['main', 'article'])
        if main_content is None:
            main_content = soup.find('div', {'id': 'content'})
        if main_content is None:
            main_content = soup
    
        links = set()
        file_links = set()
    
        for link in main_content.find_all('a', href=True):
            href = link['href']
            full_url = urljoin(base_url, href)
            normalized_url = self.normalize_url(full_url)
    
            parsed = urlparse(normalized_url)
            path = parsed.path
            _, ext = os.path.splitext(path)
            ext = ext.lower()
    
            if ext in FILE_EXTENSIONS:
                file_links.add(normalized_url)
            else:
                if self.is_valid_url(normalized_url):
                    clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                    links.add(clean_url)
                else:
                    print(Fore.YELLOW + f"URL descartada por is_valid_url: {normalized_url}")
        return links, file_links
    
    # ... (resto del código permanece igual)
    

    Resto del código sin cambios, exceptuando ajustes en methods que llamen a can_fetch o is_valid_url.

    def prioritize_links(self, links):
        """Priorizar enlaces que contengan las palabras clave especificadas."""
        if not self.keywords:
            return list(links)
        prioritized = []
        others = []
        keyword_pattern = re.compile("|".join(self.keywords), re.IGNORECASE)
        for link in links:
            if keyword_pattern.search(link):
                prioritized.append(link)
            else:
                others.append(link)
        return prioritized + others
    
    def get_extracted_data(self, url):
        """Obtener datos extraídos de la caché o extraerlos si no están en caché."""
        if url in self.cache:
            cached_data = self.cache[url]
            if 'file_downloaded' in cached_data:
                return None, set(), set([url]), False
            else:
                return cached_data.get('text'), cached_data.get('links'), cached_data.get('file_links'), False
        else:
            parsed = urlparse(url)
            path = parsed.path
            _, ext = os.path.splitext(path)
            ext = ext.lower()
            if ext in FILE_EXTENSIONS:
                success = self.download_file(url)
                if success:
                    self.cache[url] = {'file_downloaded': True}
                    return None, set(), set([url]), True
                else:
                    return None, set(), set(), True
            else:
                html_content = self.get_page(url)
                if html_content is None:
                    self.cache[url] = {'text': None, 'links': set(), 'file_links': set()}
                    return None, set(), set(), True
                text = self.extract_text(html_content)
                links, file_links = self.extract_center_links(html_content, url)
                prioritized_links = self.prioritize_links(links)
                self.cache[url] = {'text': text, 'links': set(prioritized_links), 'file_links': set(file_links)}
                return text, set(prioritized_links), set(file_links), True
    
    def download_file(self, url):
        """Descargar el fichero desde la URL y guardarlo."""
        if url in self.downloaded_files:
            print(Fore.BLUE + f"Fichero ya descargado: {url}")
            return False
    
        if not self.can_fetch(url):
            return False
    
        print(Fore.MAGENTA + f"Descargando fichero: {url}")
        try:
            response = requests.get(url, headers=self.headers, stream=True, timeout=20)
            response.raise_for_status()
    
            parsed = urlparse(url)
            filename = os.path.basename(parsed.path)
            if not filename:
                filename = f"downloaded_file_{int(time.time())}"
    
            file_path = os.path.join(self.download_dir, filename)
    
            with open(file_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
    
            print(Fore.GREEN + f"Fichero descargado y guardado en: {file_path}")
            self.downloaded_files.add(url)
            return True
        except requests.RequestException as e:
            print(Fore.RED + f"Error al descargar el fichero {url}: {e}")
            return False
    
    def is_text_unique(self, new_text):
        """Verificar unicidad del texto."""
        for old_text in self.texts:
            similarity = SequenceMatcher(None, new_text, old_text).ratio()
            if similarity > 0.5:
                print(Fore.YELLOW + f"El texto es similar a uno existente en un {similarity * 100:.2f}%.")
                return False
        return True
    
    def is_text_readable(self, text, threshold=60):
        """Verificar legibilidad del texto."""
        if not text:
            return False
    
        score = textstat.flesch_reading_ease(text)
        print(f"Flesch Reading Ease score: {score}")
    
        if score >= threshold:
            return True
    
        readable_chars_pattern = r'[a-zA-Z0-9\s+=\-*/%<>;:{}()\[\].,"\']'
    
        total_chars = len(text)
        readable_chars = len(re.findall(readable_chars_pattern, text))
    
        readable_percentage = (readable_chars / total_chars) * 100 if total_chars > 0 else 0
        print(f"Porcentaje de caracteres legibles: {readable_percentage:.2f}%")
    
        return readable_percentage >= 90
    
    def run(self):
        """Ejecutar el crawler."""
        start_time = time.time()
        last_print_time = start_time
        current_url = self.start_url
        self.to_visit.append(current_url)
    
        while not self.stop_event.is_set() and (time.time() - start_time < self.duration) and self.to_visit:
            current_time = time.time()
            elapsed = current_time - start_time
    
            if current_time - last_print_time >= 10:
                print(Fore.YELLOW + f"Tiempo transcurrido: {int(elapsed)} segundos")
                last_print_time = current_time
    
            current_url = self.to_visit.popleft()
            if current_url in self.visited:
                continue
    
            print(Fore.CYAN + f"\nVisitando: {current_url}")
    
            text, links, file_links, request_made = self.get_extracted_data(current_url)
            if file_links:
                for file_url in file_links:
                    if file_url not in self.visited:
                        print(Fore.BLUE + f"Procesando enlace a fichero: {file_url}")
                        self.visited.add(file_url)
                        self.omitted_links.add(file_url)
                self.visited.add(current_url)
                continue
    
            if text is None and not file_links:
                self.visited.add(current_url)
                continue
    
            if not self.is_text_readable(text) and current_url != self.start_url:
                print(Fore.YELLOW + "El texto no es legible. Extrayendo solo enlaces de esta URL.")
                new_links = links - self.visited
                self.add_to_visit(new_links)
                self.omitted_links.add(current_url)
    
                self.omit_counts[current_url] = self.omit_counts.get(current_url, 0) + 1
                if self.omit_counts[current_url] < self.max_omits:
                    self.to_visit.append(current_url)
                    print(Fore.BLUE + f"Reintentando {current_url} (Intento {self.omit_counts[current_url]})")
                else:
                    print(Fore.RED + f"Máximo de omisiones alcanzado para {current_url}. No se procesará nuevamente.")
                    self.visited.add(current_url)
                continue
    
            if request_made:
                if self.is_text_unique(text):
                    self.visited.add(current_url)
                    self.saved.add(current_url)
                    self.texts.append(text)
                    print(Fore.GREEN + f"Texto extraído (primeros 200 caracteres):\n{text[:200]}...\n")
                else:
                    print(Fore.YELLOW + "El texto no es lo suficientemente único. Omitiendo extracción de texto para esta URL.")
    
                    new_links = links - self.visited
                    self.add_to_visit(new_links)
                    self.omitted_links.add(current_url)
    
                    self.omit_counts[current_url] = self.omit_counts.get(current_url, 0) + 1
                    if self.omit_counts[current_url] < self.max_omits:
                        self.to_visit.append(current_url)
                        print(Fore.BLUE + f"Reintentando {current_url} (Intento {self.omit_counts[current_url]})")
                    else:
                        print(Fore.RED + f"Máximo de omisiones alcanzado para {current_url}. No se procesará nuevamente.")
                        self.visited.add(current_url)
                    continue
            else:
                print(Fore.BLUE + f"Revisitado {current_url}. Usando datos en caché.\n")
    
            new_links = links - self.visited
            self.add_to_visit(new_links)
    
            if request_made:
                print(Fore.MAGENTA + "Esperando 1 segundo antes de la siguiente solicitud...\n")
                time.sleep(1)
    
        # Finalizar la búsqueda
        print(Fore.MAGENTA + f"\nBúsqueda finalizada! Textos guardados de {len(self.saved)} páginas únicas.")
        self.save_texts()
        self.save_omitted_links()
        self.save_downloaded_files()
        self.save_blocked_by_robots()
    
    def add_to_visit(self, new_links):
        """Agregar nuevos enlaces a visitar."""
        filtered_links = set()
        for link in new_links:
            if link not in self.visited and link not in self.to_visit and self.is_valid_url(link):
                filtered_links.add(link)
        if filtered_links:
            self.to_visit.extend(filtered_links)
            random.shuffle(self.to_visit)
        else:
            print(Fore.YELLOW + "No se encontraron nuevos enlaces en esta página.")
    
    def save_texts(self, filename="extracted_texts.txt"):
        """Guardar todos los textos extraídos en un archivo."""
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                for idx, text in enumerate(self.texts, 1):
                    f.write(f"--- Página {idx} ---\n")
                    f.write(text + "\n\n")
            print(Fore.BLUE + f"Todos los textos han sido guardados en {filename}.")
        except Exception as e:
            print(Fore.RED + f"Error al guardar los textos: {e}")
    
    def save_omitted_links(self, filename="omitted_links.txt"):
        """Guardar los enlaces omitidos."""
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                for idx, url in enumerate(self.omitted_links, 1):
                    f.write(f"{idx}. {url}\n")
            print(Fore.BLUE + f"Enlaces omitidos han sido guardados en {filename}.")
        except Exception as e:
            print(Fore.RED + f"Error al guardar los enlaces omitidos: {e}")
    
    def save_downloaded_files(self, filename="downloaded_files.txt"):
        """Guardar los enlaces de los ficheros descargados."""
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                for idx, url in enumerate(self.downloaded_files, 1):
                    f.write(f"{idx}. {url}\n")
            print(Fore.BLUE + f"Enlaces de ficheros descargados han sido guardados en {filename}.")
        except Exception as e:
            print(Fore.RED + f"Error al guardar los enlaces de ficheros descargados: {e}")
    
    def save_blocked_by_robots(self, filename="blocked_by_robots.txt"):
        """Guardar las URLs bloqueadas por robots.txt."""
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                for idx, url in enumerate(self.blocked_by_robots, 1):
                    f.write(f"{idx}. {url}\n")
            print(Fore.BLUE + f"URLs bloqueadas por robots.txt han sido guardadas en {filename}.")
        except Exception as e:
            print(Fore.RED + f"Error al guardar las URLs bloqueadas por robots.txt: {e}")
    
    def listen_for_keypress(self):
        """Escuchar la pulsación de la tecla 'z' para detener la búsqueda."""
        if platform.system() == "Windows":
            while not self.stop_event.is_set():
                if msvcrt.kbhit():
                    key = msvcrt.getch()
                    if key.lower() == b'z':
                        print(Fore.RED + "\nTecla 'z' presionada. Finalizando la búsqueda...")
                        self.stop_event.set()
                        break
                time.sleep(0.1)
        else:
            def is_data():
                return select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], [])
    
            old_settings = termios.tcgetattr(sys.stdin)
            try:
                tty.setcbreak(sys.stdin.fileno())
                while not self.stop_event.is_set():
                    if is_data():
                        key = sys.stdin.read(1)
                        if key.lower() == 'z':
                            print(Fore.RED + "\nTecla 'z' presionada. Finalizando la búsqueda...")
                            self.stop_event.set()
                            break
                    time.sleep(0.1)
            finally:
                termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
    

    if name == "main": if len(sys.argv) < 2: start_url = "https://github.com/adityaiiitv/Breakout/" duration = 3000 #sys.exit(1)

    keywords = []        
    
    """""
    start_url = sys.argv[1]
    try:
        duration = int(sys.argv[2])
    except ValueError:
        print("La duración debe ser un entero que representa segundos.")
        sys.exit(1)
    
    # Obtener palabras clave opcionales
    keywords = []
    if len(sys.argv) > 3:
        keywords = sys.argv[3].split(',')
    
    """""
    
    game = WebNavigatorGame(start_url, duration, keywords)
    
    # Iniciar el hilo que escucha las pulsaciones de teclas
    listener_thread = threading.Thread(target=game.listen_for_keypress, daemon=True)
    listener_thread.start()
    
    # Ejecutar el crawler
    game.run()  "
    
  2. Assistant

    Yes, I see the issue you're pointing out. When the initial URL provided is a direct link to a file (e.g., a PDF, image, or any resource with an extension listed in FILE_EXTENSIONS), the crawler processes this file, adds it to the visited set, and then exits because self.to_visit becomes empty. This happens even though there might be many more links on the same domain that could be visited. The crawler doesn't add any new URLs to self.to_visit in this scenario, leading to premature termination.

    The root cause of the problem:

    • File URLs don't yield new links: When the crawler processes a file URL, it downloads the file but doesn't extract any new links to add to self.to_visit since files (like PDFs or images) typically don't contain HTML links.
    • Starting with a file URL leaves self.to_visit empty: If the starting URL is a file, self.to_visit only contains this URL. After processing it, self.to_visit becomes empty, and the crawler has no further URLs to visit, causing it to exit.

    How to fix the issue:

    To address this, we need to ensure that the crawler has a way to discover additional URLs even when starting with a file link. Here are some possible solutions:

    1. Modify the crawler to retrieve and add the parent page or base domain links when starting from a file URL.

      • Approach:

        • When the crawler detects that the starting URL is a file, it should determine the parent URL (e.g., the directory containing the file or the base domain).
        • Fetch the parent URL's page content and extract links from it.
        • Add these links to self.to_visit so that the crawler has more URLs to process.
      • Implementation:

        Add a function to get the parent URL:

        def get_parent_url(self, url):
            parsed = urlparse(url)
            if parsed.path in ['', '/']:
                return None  # No parent URL available
            path = os.path.dirname(parsed.path)
            parent_url = f"{parsed.scheme}://{parsed.netloc}{path}"
            return parent_url
        

        Modify the run method to handle the initial file URL case:

        def run(self):
            """Execute the crawler."""
            start_time = time.time()
            last_print_time = start_time
            
            # Check if the start URL is a file
            parsed_start_url = urlparse(self.start_url)
            _, ext = os.path.splitext(parsed_start_url.path)
            ext = ext.lower()
        
            if ext in FILE_EXTENSIONS:
                print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.")
                parent_url = self.get_parent_url(self.start_url)
                if parent_url:
                    print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.")
                    self.to_visit.append(parent_url)
                else:
                    print(Fore.RED + "No parent URL found. Exiting crawler.")
                    return
            else:
                self.to_visit.append(self.start_url)
            
            # Rest of the run method remains the same
        
    2. Enforce that the starting URL must be a webpage, not a file.

      • Approach:

        • Before running the crawler, check if the starting URL ends with a file extension listed in FILE_EXTENSIONS.
        • If it does, prompt the user or raise an error to provide a valid starting webpage URL.
      • Implementation:

        In your __init__ method or before starting the crawler, add:

        def __init__(self, start_url, ...):
            # existing initialization code
            
            # Check if the start URL is a file
            parsed_start_url = urlparse(start_url)
            _, ext = os.path.splitext(parsed_start_url.path)
            ext = ext.lower()
            if ext in FILE_EXTENSIONS:
                raise ValueError("The starting URL cannot be a direct link to a file. Please provide a webpage URL.")
        
    3. Allow the crawler to proceed to the base domain when starting from a file URL.

      • Approach:

        • Adjust the crawler to recognize when it's starting from a file URL and automatically set the base domain URL as the next URL to visit.
      • Implementation:

        In the run method:

        def run(self):
            """Execute the crawler."""
            start_time = time.time()
            last_print_time = start_time
            
            # Check if the start URL is a file
            parsed_start_url = urlparse(self.start_url)
            _, ext = os.path.splitext(parsed_start_url.path)
            ext = ext.lower()
        
            if ext in FILE_EXTENSIONS:
                base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/"
                print(Fore.YELLOW + f"Start URL is a file. Switching to base domain: {base_domain_url}")
                self.to_visit.append(base_domain_url)
            else:
                self.to_visit.append(self.start_url)
            
            # Rest of the run method remains the same
        

    Recommendation:

    Option 1 is a more flexible solution because it allows the crawler to move up one level and attempt to find links on the parent page of the file. If the parent page is not available or doesn't exist, the crawler can then decide to exit gracefully or prompt the user.

    Here's how you can implement option 1 in your code:

    def get_parent_url(self, url):
        parsed = urlparse(url)
        if parsed.path in ['', '/']:
            return None  # No parent URL available
        # Remove the last segment of the path
        path = os.path.dirname(parsed.path.rstrip('/'))
        parent_url = f"{parsed.scheme}://{parsed.netloc}{path}"
        return parent_url
    
    def run(self):
        """Execute the crawler."""
        start_time = time.time()
        last_print_time = start_time
    
        # Check if the start URL is a file
        parsed_start_url = urlparse(self.start_url)
        _, ext = os.path.splitext(parsed_start_url.path)
        ext = ext.lower()
    
        if ext in FILE_EXTENSIONS:
            print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.")
            parent_url = self.get_parent_url(self.start_url)
            if parent_url and parent_url != self.start_url:
                print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.")
                self.to_visit.append(parent_url)
            else:
                # If no parent URL, use the base domain
                base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/"
                print(Fore.YELLOW + f"No parent URL found. Using base domain: {base_domain_url} as starting point.")
                self.to_visit.append(base_domain_url)
        else:
            self.to_visit.append(self.start_url)
    
        # Rest of the run method remains the same
    

    With this implementation:

    • If the start URL is a file, the crawler attempts to find the parent directory URL.
    • If a parent URL is found, it's added to self.to_visit.
    • If not, the crawler defaults to using the base domain URL.
    • This ensures that self.to_visit is not empty, and the crawler has URLs to process after handling the initial file URL.

    Testing the solution:

    To test this, you can run the crawler with a starting URL that points directly to a file, such as a PDF or image. For example:

    python crawler.py https://example.com/path/to/file.pdf
    

    Ensure that the crawler outputs messages indicating that it recognized the start URL as a file and is attempting to retrieve the parent URL or defaulting to the base domain.

    Additional considerations:

    • Error handling: Ensure that the crawler handles cases where the parent URL or base domain may not be accessible or may not have any links to follow.
    • Circular references: Be cautious of potential infinite loops if the crawler keeps adding parent URLs that have already been visited. Make sure you check if the parent URL is already in self.visited.
    • Politeness and robots.txt: Since you're adding new URLs to crawl, ensure that you're respecting the robots.txt rules for those URLs, especially if the base domain is being used.

    Updated code snippet with the fix:

    Here's the modified run method with comments:

    def run(self):
        """Execute the crawler."""
        start_time = time.time()
        last_print_time = start_time
    
        # Initialization of current URL
        current_url = self.start_url
    
        # Check if the start URL is a file
        parsed_start_url = urlparse(self.start_url)
        _, ext = os.path.splitext(parsed_start_url.path)
        ext = ext.lower()
    
        if ext in FILE_EXTENSIONS:
            print(Fore.YELLOW + "Start URL is a file. Attempting to retrieve parent URL.")
            parent_url = self.get_parent_url(self.start_url)
            if parent_url and parent_url != self.start_url:
                print(Fore.YELLOW + f"Using parent URL: {parent_url} as starting point.")
                self.to_visit.append(parent_url)
            else:
                # If no parent URL, use the base domain
                base_domain_url = f"{parsed_start_url.scheme}://{parsed_start_url.netloc}/"
                print(Fore.YELLOW + f"No parent URL found. Using base domain: {base_domain_url} as starting point.")
                self.to_visit.append(base_domain_url)
        else:
            self.to_visit.append(self.start_url)
    
        while not self.stop_event.is_set() and (time.time() - start_time < self.duration) and self.to_visit:
            current_time = time.time()
            elapsed = current_time - start_time
    
            if current_time - last_print_time >= 10:
                print(Fore.YELLOW + f"Elapsed time: {int(elapsed)} seconds")
                last_print_time = current_time
    
            current_url = self.to_visit.popleft()
            if current_url in self.visited:
                continue
    
            print(Fore.CYAN + f"\nVisiting: {current_url}")
    
            text, links, file_links, request_made = self.get_extracted_data(current_url)
            if file_links:
                for file_url in file_links:
                    if file_url not in self.visited:
                        print(Fore.BLUE + f"Processing file link: {file_url}")
                        self.visited.add(file_url)
                        self.omitted_links.add(file_url)
                self.visited.add(current_url)
                continue
    
            if text is None and not file_links:
                self.visited.add(current_url)
                continue
    
            # Rest of your existing code...
    

    Summary:

    • The issue arises when the crawler starts with a file URL, leading to self.to_visit becoming empty after processing the file.
    • To fix this, adjust the crawler to fetch and add the parent URL or base domain links when starting from a file URL.
    • Implementing this change ensures that the crawler has additional URLs to visit and doesn't terminate prematurely.