try guess programmer level and years of experience according to this script also if it…
- User
try guess programmer level and years of experience according to this script also if it is a bad, average, good or an excellent one
import asyncio import aiohttp from aiohttp import web from aiohttp.client_exceptions import ClientError from bs4 import BeautifulSoup import json from typing import List, Dict, Optional import ssl import logging from datetime import datetime import colorlog import socket import netifaces import argparse import sys import signal
Configuración del script
CONFIG = { 'base_url': 'https://10.0.0.2', 'username': 'misael_campos', 'password': 'd337O8AetR3p4', 'post_endpoint': '/action/pononuopticalinfo.html', 'who': 100, 'onuid': 0, 'pon_range': range(1, 17), 'onu_group_range': range(0, 2), 'request_timeout': aiohttp.ClientTimeout( total=240, # Tiempo de espera total de la operación connect=120, # Tiempo de espera de conexión sock_read=120 # Tiempo de espera de lectura del socket ), 'max_concurrent_requests': 8, # Reducido para prevenir sobrecarga del servidor 'max_retries': 3, # Número máximo de intentos de reintento 'retry_delay': 1, # Retraso entre reintentos en segundos 'batch_size': 8, # Número de solicitudes concurrentes por lote 'host': '0.0.0.0', # Dirección del host 'port': 8080, # Número de puerto 'endpoint': '/data', # Ruta del endpoint añadida 'ignore_row_length_warning': False # Nueva opción de configuración }
Configurar registro
handler = colorlog.StreamHandler() handler.setFormatter(colorlog.ColoredFormatter( '%(log_color)s%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', log_colors={ 'DEBUG': 'light_cyan', 'INFO': 'light_green', 'WARNING': 'light_yellow', 'ERROR': 'light_red', 'CRITICAL': 'light_red,bg_white', } ))
logger = colorlog.getLogger(name) logger.addHandler(handler) logger.setLevel(logging.INFO)
Eliminar cualquier manejador existente para evitar registros duplicados
for hdlr in logger.handlers[:-1]: logger.removeHandler(hdlr)
def get_all_ip_addresses() -> List[str]: """ Obtiene todas las direcciones IPv4 no locales de las interfaces de red de la máquina.
Retorna: List[str]: Una lista de direcciones IPv4. """ ips = [] try: interfaces = netifaces.interfaces() for iface in interfaces: addrs = netifaces.ifaddresses(iface) ipv4 = addrs.get(netifaces.AF_INET) if ipv4: for addr in ipv4: ip = addr.get('addr') if ip and ip != '127.0.0.1': ips.append(ip) except ValueError as ve: logger.error(f"Error al obtener direcciones IP: {ve}") except Exception as e: logger.error(f"Error inesperado al obtener direcciones IP: {e}") return ipsclass AsyncScraper: """ Scraper asíncrono para obtener información óptica de un router.
Atributos: base_url (str): La URL base del router. ssl_context (ssl.SSLContext): Contexto SSL para peticiones HTTPS. session (aiohttp.ClientSession): La sesión HTTP para realizar peticiones. sem (asyncio.Semaphore): Semáforo para limitar peticiones concurrentes. """ def __init__(self, base_url: str): """ Inicializa el AsyncScraper con la URL base proporcionada. Args: base_url (str): La URL base del router. """ self.base_url = base_url.rstrip('/') self.ssl_context = ssl.create_default_context() self.ssl_context.check_hostname = False self.ssl_context.verify_mode = ssl.CERT_NONE self.session: Optional[aiohttp.ClientSession] = None self.sem = asyncio.Semaphore(CONFIG['max_concurrent_requests']) async def __aenter__(self): """ Entra en el gestor de contexto asíncrono, inicializando la sesión HTTP. Retorna: AsyncScraper: La instancia misma con una sesión activa. """ timeout = CONFIG['request_timeout'] self.session = aiohttp.ClientSession( connector=aiohttp.TCPConnector( ssl=self.ssl_context, limit=CONFIG['max_concurrent_requests'] ), timeout=timeout, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): """ Sale del gestor de contexto asíncrono, cerrando la sesión HTTP. Args: exc_type: Tipo de excepción. exc_val: Valor de la excepción. exc_tb: Traza de la excepción. """ if self.session: await self.session.close() async def login(self, username: str, password: str) -> bool: """ Realiza el inicio de sesión en el router. Args: username (str): El nombre de usuario para la autenticación. password (str): La contraseña para la autenticación. Retorna: bool: True si el inicio de sesión es exitoso, False en caso contrario. """ login_url = f"{self.base_url}/action/main.html" login_data = { 'user': username, 'pass': password, 'button': 'Login', 'who': '100', } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(login_url, data=login_data) as response: response.raise_for_status() text = await response.text() if "Sorry, you do not have access" in text: logger.error("Error de inicio de sesión: Acceso denegado") return False elif "Error" in text or "Incorrect password" in text: logger.error("Error de inicio de sesión: Credenciales incorrectas") return False logger.info("¡Inicio de sesión exitoso!") return True except aiohttp.ClientResponseError as e: logger.error(f"Error HTTP durante el intento de inicio de sesión {attempt + 1}: {e}") except aiohttp.ClientConnectorError as e: logger.error(f"Error de conexión durante el intento de inicio de sesión {attempt + 1}: {e}") except asyncio.TimeoutError: logger.error(f"Tiempo de espera agotado durante el intento de inicio de sesión {attempt + 1}") except Exception as e: logger.error(f"Error inesperado durante el intento de inicio de sesión {attempt + 1}: {e}") if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) logger.info(f"Reintentando inicio de sesión (Intento {attempt + 2})...") else: logger.error("Todos los intentos de inicio de sesión fallaron.") return False async def fetch_pon_onu_optical_info(self, pon: int, onu_group: int) -> Optional[str]: """ Obtiene información óptica PON ONU del router. Args: pon (int): Identificador PON. onu_group (int): Identificador del grupo ONU. Retorna: Optional[str]: El contenido HTML si es exitoso, None en caso contrario. """ async with self.sem: post_url = f"{self.base_url}{CONFIG['post_endpoint']}" payload = { 'pon': pon, 'onu_group': onu_group, 'who': CONFIG['who'], 'onuid': CONFIG['onuid'] } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(post_url, data=payload) as response: response.raise_for_status() content = await response.text() logger.info( f"Datos obtenidos exitosamente para PON {pon}, Grupo ONU {onu_group} " f"(Intento {attempt + 1})" ) return content except aiohttp.ClientResponseError as e: logger.warning( f"Error HTTP {e.status} para PON {pon}, Grupo ONU {onu_group} " f"(Intento {attempt + 1}): {e.message}" ) except aiohttp.ClientConnectorError as e: logger.warning( f"Error de conexión para PON {pon}, Grupo ONU {onu_group} " f"(Intento {attempt + 1}): {e}" ) except asyncio.TimeoutError: logger.warning( f"Tiempo de espera agotado al obtener PON {pon}, Grupo ONU {onu_group} " f"(Intento {attempt + 1})" ) except Exception as e: logger.warning( f"Error inesperado al obtener PON {pon}, Grupo ONU {onu_group} " f"(Intento {attempt + 1}): {e}" ) if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) logger.info(f"Reintentando PON {pon}, Grupo ONU {onu_group}...") else: logger.error( f"Todos los intentos fallaron para PON {pon}, Grupo ONU {onu_group}" ) return None @staticmethod def parse_html_table(html_content: str) -> List[Dict[str, Optional[str]]]: """ Analiza una tabla HTML y extrae datos en una lista de diccionarios. Args: html_content (str): El contenido HTML que contiene la tabla. Retorna: List[Dict[str, Optional[str]]]: Datos de la tabla analizados. """ if not html_content: return [] soup = BeautifulSoup(html_content, 'html.parser') table = soup.find('table', border="1", cellpadding="4", cellspacing="0") if not table: logger.warning("No se encontró tabla en el contenido HTML") return [] headers = [] header_row = table.find('tr') if not header_row: logger.warning("No se encontró fila de encabezados en la tabla") return [] for th in header_row.find_all(['td', 'th']): header_text = th.get_text(strip=True) headers.append(header_text if header_text else f"Columna_{len(headers) + 1}") data = [] for row in table.find_all('tr')[1:]: cells = row.find_all('td') if len(cells) != len(headers): if not CONFIG['ignore_row_length_warning']: logger.warning("La fila no coincide con la longitud del encabezado. Omitiendo fila") continue row_data = { header: cell.get_text(strip=True) or None for header, cell in zip(headers, cells) } data.append(row_data) return dataasync def fetch_and_parse(scraper: AsyncScraper, pon: int, onu_group: int) -> Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: """ Obtiene y analiza información óptica para un PON y grupo ONU específico.
Args: scraper (AsyncScraper): La instancia del scraper. pon (int): Identificador PON. onu_group (int): Identificador del grupo ONU. Retorna: Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: Datos analizados estructurados por PON y grupo ONU. """ html_content = await scraper.fetch_pon_onu_optical_info(pon, onu_group) parsed_data = AsyncScraper.parse_html_table(html_content) if html_content else [] if not parsed_data: logger.warning(f"No se analizaron datos para PON {pon}, Grupo ONU {onu_group}") return {pon: {onu_group: parsed_data}}async def process_batch(scraper: AsyncScraper, batch: List[tuple]) -> List[Dict]: """ Procesa un lote de combinaciones de PON y grupo ONU.
Args: scraper (AsyncScraper): La instancia del scraper. batch (List[tuple]): Una lista de tuplas (pon, onu_group). Retorna: List[Dict]: Una lista de diccionarios de datos analizados. """ tasks = [fetch_and_parse(scraper, pon, onu_group) for pon, onu_group in batch] return await asyncio.gather(*tasks, return_exceptions=True)async def collect_data(scraper: AsyncScraper) -> Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: """ Recopila datos obteniendo y analizando información para todas las combinaciones de PON y grupo ONU.
Args: scraper (AsyncScraper): La instancia del scraper. Retorna: Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: Datos analizados agregados. """ # Crear todas las combinaciones de PON y grupos ONU all_combinations = [ (pon, onu_group) for pon in CONFIG['pon_range'] for onu_group in CONFIG['onu_group_range'] ] # Procesar en lotes aggregated_data: Dict[int, Dict[int, List[Dict[str, Optional[str]]]]] = {} for i in range(0, len(all_combinations), CONFIG['batch_size']): batch = all_combinations[i:i + CONFIG['batch_size']] logger.info(f"Procesando lote {i // CONFIG['batch_size'] + 1}") results = await process_batch(scraper, batch) for result in results: if isinstance(result, Exception): logger.error(f"La tarea del lote falló con excepción: {result}") continue for pon, onu_data in result.items(): if pon not in aggregated_data: aggregated_data[pon] = {} aggregated_data[pon].update(onu_data) return aggregated_dataasync def handle_data(request: web.Request) -> web.StreamResponse: """ Manejador HTTP para el endpoint configurado. Activa la recopilación de datos y transmite los datos JSON agregados de forma incremental usando SSE.
Args: request (web.Request): La petición HTTP entrante. Retorna: web.StreamResponse: La respuesta SSE transmitiendo los datos. """ start_time = datetime.now() logger.info(f"Petición recibida en {CONFIG['endpoint']} a las {start_time}") # Inicializar la respuesta SSE resp = web.StreamResponse( status=200, reason='OK', headers={ 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', } ) await resp.prepare(request) try: async with AsyncScraper(CONFIG['base_url']) as scraper: if not await scraper.login(CONFIG['username'], CONFIG['password']): error_data = json.dumps({"error": "Error de inicio de sesión. Por favor, verifique las credenciales."}) await resp.write(f"event: error\ndata: {error_data}\n\n".encode('utf-8')) await resp.write_eof() return resp aggregated_data = await collect_data(scraper) end_time = datetime.now() duration = end_time - start_time logger.info(f"Recopilación de datos completa. Duración: {duration}") # Enviar los datos agregados como un evento SSE json_data = json.dumps(aggregated_data) sse_event = f"event: complete\ndata: {json_data}\n\n" await resp.write(sse_event.encode('utf-8')) await resp.write_eof() except ClientError as e: logger.error(f"Error de cliente en handle_data: {e}") error_event = f"event: error\ndata: {{\"error\": \"Error de cliente: {e}\"}}\n\n" await resp.write(error_event.encode('utf-8')) await resp.write_eof() except asyncio.CancelledError: logger.warning("La petición fue cancelada.") await resp.write_eof() except Exception as e: logger.error(f"Error inesperado en handle_data: {e}") error_event = f"event: error\ndata: {{\"error\": \"Error inesperado: {e}\"}}\n\n" await resp.write(error_event.encode('utf-8')) await resp.write_eof() return respasync def init_app() -> web.Application: """ Inicializa la aplicación web aiohttp.
Retorna: web.Application: La aplicación web aiohttp configurada. """ app = web.Application() app.router.add_get(CONFIG['endpoint'], handle_data) # Usar endpoint configurable return appdef parse_args() -> argparse.Namespace: """ Analiza los argumentos de línea de comandos.
Retorna: argparse.Namespace: Los argumentos de línea de comandos analizados. """ parser = argparse.ArgumentParser(description="Servidor Scraper Asíncrono") parser.add_argument( '--ignore-row-length-warnings', action='store_true', help='Ignorar advertencias sobre desajustes en la longitud de filas en tablas HTML.' ) # Puedes agregar más parámetros CLI aquí si es necesario. return parser.parse_args()async def main(): """ Punto de entrada principal para ejecutar el servidor web con apagado controlado. """ # Analizar argumentos CLI args = parse_args()
# Actualizar CONFIG basado en argumentos CLI CONFIG['ignore_row_length_warning'] = args.ignore_row_length_warnings # Obtener todas las direcciones IP local_ips = get_all_ip_addresses() if not local_ips: logger.warning("No se encontraron direcciones IPv4 no locales. Usando 127.0.0.1 como respaldo") local_ips = [CONFIG['host']] endpoint = CONFIG.get('endpoint', '/data') logger.info(f"Iniciando servidor en el puerto {CONFIG['port']}") logger.info("Accesible en las siguientes URLs:") for ip in local_ips: logger.info(f"http://{ip}:{CONFIG['port']}{endpoint}") app = await init_app() runner = web.AppRunner(app) await runner.setup() site = web.TCPSite(runner, host=CONFIG['host'], port=CONFIG['port']) await site.start() logger.info("El servidor está ahora en funcionamiento") # Crear un evento para esperar el apagado shutdown_event = asyncio.Event() # Definir manejador de señales def shutdown_signal_handler(): logger.info("Señal de apagado recibida") shutdown_event.set() loop = asyncio.get_running_loop() for sig in (signal.SIGINT, signal.SIGTERM): try: loop.add_signal_handler(sig, shutdown_signal_handler) except NotImplementedError: # Los manejadores de señales no están implementados en algunas plataformas (ej., Windows) logger.warning(f"Manejadores de señales no implementados para {sig}") # Esperar hasta que se reciba una señal de apagado await shutdown_event.wait() logger.info("Apagando servidor...") # Realizar pasos de apagado await runner.shutdown() await runner.cleanup() logger.info("Servidor apagado exitosamente.")if name == "main": try: asyncio.run(main()) except Exception as e: logger.critical(f"Excepción no manejada: {e}") sys.exit(1)
- Assistant
Based on the provided Python script, here's an assessment of the programmer's skill level, years of experience, and the quality of the script:
Programmer Level and Experience
Level: Advanced
Years of Experience: 5+ years
Rationale:
-
Asynchronous Programming:
- Proficient use of
asyncioandaiohttpfor asynchronous HTTP requests and server handling. - Implementation of asynchronous context managers (
__aenter__and__aexit__) shows a deep understanding of asynchronous resource management.
- Proficient use of
-
Error Handling and Retries:
- Comprehensive error handling with retries and backoff strategies demonstrates experience in building resilient applications.
- Handling specific exceptions like
ClientResponseError,ClientConnectorError, andTimeoutErrorindicates familiarity withaiohttp's exception hierarchy.
-
Concurrency Management:
- Use of
asyncio.Semaphoreto limit concurrent requests showcases knowledge of managing concurrency to prevent overloading resources.
- Use of
-
Web Scraping and Parsing:
- Integration of
BeautifulSoupfor HTML parsing suggests experience with web scraping tasks. - Parsing specific HTML structures and handling inconsistencies in the data reflects practical scraping challenges.
- Integration of
-
Logging and Monitoring:
- Advanced logging setup using
colorlogfor colored output enhances readability and debugging. - Structured and meaningful log messages indicate thoughtful monitoring and maintenance considerations.
- Advanced logging setup using
-
Configuration Management:
- Centralized configuration using a dictionary allows for easy adjustments and scalability.
- Use of
argparsefor command-line arguments demonstrates the ability to create flexible and user-friendly scripts.
-
Networking and System Interaction:
- Leveraging
netifacesto retrieve network interface information shows proficiency in interacting with system-level details. - Handling SSL contexts and understanding security implications indicate a solid grasp of networking principles.
- Leveraging
-
Code Structure and Documentation:
- Well-organized code with clear separation of concerns (e.g., scraper class, web handler, utility functions).
- Comprehensive docstrings and comments (albeit in Spanish) facilitate understanding and maintenance.
Script Quality Assessment
Quality Rating: Good
Strengths:
-
Modularity: The script is well-structured with distinct classes and functions, making it maintainable and scalable.
-
Asynchronous Efficiency: Effective use of asynchronous programming paradigms ensures non-blocking operations, which is crucial for I/O-bound tasks like web scraping.
-
Robust Error Handling: The implementation of retries and specific exception handling enhances the script's reliability.
-
Logging: Advanced logging provides clear insights into the script's operations, aiding in debugging and monitoring.
-
Configuration Flexibility: Centralized and adjustable configuration settings make the script adaptable to different environments and requirements.
Areas for Improvement:
-
Security Practices:
- Hard-Coded Credentials: Storing sensitive information like
usernameandpassworddirectly in the script is a security risk. It's advisable to use environment variables or secure configuration files to manage such data. - SSL Verification: Disabling SSL hostname checking and certificate verification (
ssl_context.verify_mode = ssl.CERT_NONE) can expose the application to security vulnerabilities. Enabling proper SSL verification is recommended, especially for production environments.
- Hard-Coded Credentials: Storing sensitive information like
-
Error Reporting:
- While the script handles exceptions gracefully, providing more granular error messages or integrating with monitoring tools could enhance observability.
-
Testing and Documentation:
- Incorporating unit tests and more extensive documentation (in multiple languages, if targeting a diverse audience) would improve the script's reliability and accessibility.
-
Configuration Management:
- Moving the configuration to external files (e.g., YAML, JSON) or utilizing configuration management libraries can enhance flexibility and security.
-
Code Optimization:
- While the current structure is solid, further optimizations or refactoring (e.g., simplifying nested loops, enhancing parsing logic) could improve performance and readability.
Conclusion
The script demonstrates a high level of proficiency in Python programming, particularly in asynchronous operations, web scraping, and robust application design. With over five years of experience, the programmer has effectively utilized advanced libraries and best practices to build a functional and maintainable script. Addressing the highlighted areas for improvement would elevate the script from "Good" to "Excellent," ensuring better security, maintainability, and scalability.
-