change the following script, i dont want to save a .json file I want to be able to…
- User
change the following script, i dont want to save a .json file I want to be able to access a URL for the json that this script generates. that the script will start running and will wait the URL request to give the json just like a REST API
import asyncio import aiohttp from bs4 import BeautifulSoup import json from typing import List, Dict, Optional import ssl import logging from datetime import datetime import colorlog
Configuration settings
CONFIG = { 'base_url': 'https://10.0.0.2', 'username': 'misael_campos', 'password': 'd337O8AetR3p4', 'output_json': 'aggregated_pon_onu_data.json', 'post_endpoint': '/action/pononuopticalinfo.html', 'who': 100, 'onuid': 0, 'pon_range': range(1, 17), 'onu_group_range': range(0, 2), 'request_timeout': aiohttp.ClientTimeout( total=240, # Total operation timeout connect=120, # Connection timeout sock_read=120 # Socket read timeout ), 'max_concurrent_requests': 8, # Reduced to prevent server overload 'max_retries': 3, # Maximum number of retry attempts 'retry_delay': 1, # Delay between retries in seconds 'batch_size': 8 # Number of concurrent requests per batch }
Configure logging
handler = colorlog.StreamHandler() handler.setFormatter(colorlog.ColoredFormatter( '%(log_color)s%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', log_colors={ 'DEBUG': 'light_cyan', 'INFO': 'light_green', 'WARNING': 'light_yellow', 'ERROR': 'light_red', 'CRITICAL': 'light_red,bg_white', } ))
logger = colorlog.getLogger(name) logger.addHandler(handler) logger.setLevel(logging.INFO)
Remove any existing handlers to avoid duplicate logs
for hdlr in logger.handlers[:-1]: logger.removeHandler(hdlr)
class AsyncScraper: def init(self, base_url: str): self.base_url = base_url.rstrip('/') self.ssl_context = ssl.create_default_context() self.ssl_context.check_hostname = False self.ssl_context.verify_mode = ssl.CERT_NONE self.session = None self.sem = asyncio.Semaphore(CONFIG['max_concurrent_requests'])
async def __aenter__(self): timeout = CONFIG['request_timeout'] self.session = aiohttp.ClientSession( connector=aiohttp.TCPConnector( ssl=self.ssl_context, limit=CONFIG['max_concurrent_requests'] ), timeout=timeout, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): if self.session: await self.session.close() async def login(self, username: str, password: str) -> bool: login_url = f"{self.base_url}/action/main.html" login_data = { 'user': username, 'pass': password, 'button': 'Login', 'who': '100', } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(login_url, data=login_data) as response: text = await response.text() if "Sorry, you do not have access" in text: logger.error("Login failed: Access denied") return False elif "Error" in text or "Incorrect password" in text: logger.error("Login failed: Incorrect credentials") return False logger.info("Login successful!") return True except Exception as e: logger.error(f"Login attempt {attempt + 1} failed: {e}") if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) else: return False async def fetch_pon_onu_optical_info(self, pon: int, onu_group: int) -> Optional[str]: async with self.sem: post_url = f"{self.base_url}{CONFIG['post_endpoint']}" payload = { 'pon': pon, 'onu_group': onu_group, 'who': CONFIG['who'], 'onuid': CONFIG['onuid'] } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(post_url, data=payload) as response: if response.status == 200: content = await response.text() logger.info(f"Successfully fetched data for PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") return content else: logger.warning(f"Error {response.status} for PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") except asyncio.TimeoutError: logger.warning(f"Timeout fetching PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") except Exception as e: logger.warning(f"Error fetching PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1}): {e}") if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) logger.info(f"Retrying PON {pon}, ONU Group {onu_group}...") else: logger.error(f"All attempts failed for PON {pon}, ONU Group {onu_group}") return Nonedef parse_html_table(html_content: str) -> List[Dict[str, Optional[str]]]: if not html_content: return []
soup = BeautifulSoup(html_content, 'html.parser') table = soup.find('table', border="1", cellpadding="4", cellspacing="0") if not table: logger.warning("No table found in the HTML content.") return [] headers = [] header_row = table.find('tr') if not header_row: logger.warning("No header row found in the table.") return [] for th in header_row.find_all(['td', 'th']): header_text = th.get_text(strip=True) headers.append(header_text if header_text else f"Column_{len(headers)+1}") data = [] for row in table.find_all('tr')[1:]: cells = row.find_all('td') if len(cells) != len(headers): logger.warning("Row does not match header length. Skipping row.") continue row_data = { header: cell.get_text(strip=True) or None for header, cell in zip(headers, cells) } data.append(row_data) return dataasync def fetch_and_parse(scraper: AsyncScraper, pon: int, onu_group: int) -> Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: html_content = await scraper.fetch_pon_onu_optical_info(pon, onu_group) parsed_data = parse_html_table(html_content) if html_content else [] if not parsed_data: logger.warning(f"No data parsed for PON {pon}, ONU Group {onu_group}.") return {pon: {onu_group: parsed_data}}
async def process_batch(scraper: AsyncScraper, batch: List[tuple]) -> List[Dict]: tasks = [fetch_and_parse(scraper, pon, onu_group) for pon, onu_group in batch] return await asyncio.gather(*tasks, return_exceptions=True)
async def main(): start_time = datetime.now() logger.info(f"Starting data collection at {start_time}")
async with AsyncScraper(CONFIG['base_url']) as scraper: if not await scraper.login(CONFIG['username'], CONFIG['password']): return # Create all combinations of PON and ONU groups all_combinations = [ (pon, onu_group) for pon in CONFIG['pon_range'] for onu_group in CONFIG['onu_group_range'] ] # Process in batches aggregated_data: Dict[int, Dict[int, List[Dict[str, Optional[str]]]]] = {} for i in range(0, len(all_combinations), CONFIG['batch_size']): batch = all_combinations[i:i + CONFIG['batch_size']] logger.info(f"Processing batch {i//CONFIG['batch_size'] + 1}") results = await process_batch(scraper, batch) for result in results: if isinstance(result, Exception): logger.error(f"Batch task failed with exception: {result}") continue for pon, onu_data in result.items(): if pon not in aggregated_data: aggregated_data[pon] = {} aggregated_data[pon].update(onu_data) # Save results with open(CONFIG['output_json'], 'w', encoding='utf-8') as f: json.dump(aggregated_data, f, indent=4) end_time = datetime.now() duration = end_time - start_time logger.info(f"\nData fetching complete. Duration: {duration}")if name == "main": asyncio.run(main())
- Assistant
Certainly! To transform your existing asynchronous scraper script into a REST API that serves the generated JSON data via a URL, you can utilize the
aiohttplibrary's web server capabilities. Below is the modified script that accomplishes this:-
Key Changes:
- Removed File Saving: The script no longer writes the aggregated data to a
.jsonfile. - Added REST API Endpoint: Introduced an
/dataendpoint that, when accessed via a GET request, triggers the data scraping process and returns the JSON response. - Integrated
aiohttpWeb Server: Utilizedaiohttp.webto create and run the web server. - Enhanced Structure: Organized the scraper initialization and cleanup within the web server's lifecycle events to ensure efficient resource management.
- Removed File Saving: The script no longer writes the aggregated data to a
-
Dependencies:
- Ensure you have
aiohttpandbeautifulsoup4installed. You can install them using:pip install aiohttp beautifulsoup4 colorlog
- Ensure you have
-
Complete Modified Script:
import asyncio import aiohttp from aiohttp import web from bs4 import BeautifulSoup import json from typing import List, Dict, Optional import ssl import logging from datetime import datetime import colorlog # Configuration settings CONFIG = { 'base_url': 'https://10.0.0.2', 'username': 'misael_campos', 'password': 'd337O8AetR3p4', 'post_endpoint': '/action/pononuopticalinfo.html', 'who': 100, 'onuid': 0, 'pon_range': range(1, 17), 'onu_group_range': range(0, 2), 'request_timeout': aiohttp.ClientTimeout( total=240, # Total operation timeout connect=120, # Connection timeout sock_read=120 # Socket read timeout ), 'max_concurrent_requests': 8, # Reduced to prevent server overload 'max_retries': 3, # Maximum number of retry attempts 'retry_delay': 1, # Delay between retries in seconds 'batch_size': 8 # Number of concurrent requests per batch } # Configure logging handler = colorlog.StreamHandler() handler.setFormatter(colorlog.ColoredFormatter( '%(log_color)s%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', log_colors={ 'DEBUG': 'light_cyan', 'INFO': 'light_green', 'WARNING': 'light_yellow', 'ERROR': 'light_red', 'CRITICAL': 'light_red,bg_white', } )) logger = colorlog.getLogger(__name__) logger.addHandler(handler) logger.setLevel(logging.INFO) # Remove any existing handlers to avoid duplicate logs for hdlr in logger.handlers[:-1]: logger.removeHandler(hdlr) class AsyncScraper: def __init__(self, base_url: str): self.base_url = base_url.rstrip('/') self.ssl_context = ssl.create_default_context() self.ssl_context.check_hostname = False self.ssl_context.verify_mode = ssl.CERT_NONE self.session = None self.sem = asyncio.Semaphore(CONFIG['max_concurrent_requests']) async def __aenter__(self): timeout = CONFIG['request_timeout'] self.session = aiohttp.ClientSession( connector=aiohttp.TCPConnector( ssl=self.ssl_context, limit=CONFIG['max_concurrent_requests'] ), timeout=timeout, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): if self.session: await self.session.close() async def login(self, username: str, password: str) -> bool: login_url = f"{self.base_url}/action/main.html" login_data = { 'user': username, 'pass': password, 'button': 'Login', 'who': '100', } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(login_url, data=login_data) as response: text = await response.text() if "Sorry, you do not have access" in text: logger.error("Login failed: Access denied") return False elif "Error" in text or "Incorrect password" in text: logger.error("Login failed: Incorrect credentials") return False logger.info("Login successful!") return True except Exception as e: logger.error(f"Login attempt {attempt + 1} failed: {e}") if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) else: return False async def fetch_pon_onu_optical_info(self, pon: int, onu_group: int) -> Optional[str]: async with self.sem: post_url = f"{self.base_url}{CONFIG['post_endpoint']}" payload = { 'pon': pon, 'onu_group': onu_group, 'who': CONFIG['who'], 'onuid': CONFIG['onuid'] } for attempt in range(CONFIG['max_retries']): try: async with self.session.post(post_url, data=payload) as response: if response.status == 200: content = await response.text() logger.info(f"Successfully fetched data for PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") return content else: logger.warning(f"Error {response.status} for PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") except asyncio.TimeoutError: logger.warning(f"Timeout fetching PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1})") except Exception as e: logger.warning(f"Error fetching PON {pon}, ONU Group {onu_group} (Attempt {attempt + 1}): {e}") if attempt < CONFIG['max_retries'] - 1: await asyncio.sleep(CONFIG['retry_delay']) logger.info(f"Retrying PON {pon}, ONU Group {onu_group}...") else: logger.error(f"All attempts failed for PON {pon}, ONU Group {onu_group}") return None def parse_html_table(html_content: str) -> List[Dict[str, Optional[str]]]: if not html_content: return [] soup = BeautifulSoup(html_content, 'html.parser') table = soup.find('table', border="1", cellpadding="4", cellspacing="0") if not table: logger.warning("No table found in the HTML content.") return [] headers = [] header_row = table.find('tr') if not header_row: logger.warning("No header row found in the table.") return [] for th in header_row.find_all(['td', 'th']): header_text = th.get_text(strip=True) headers.append(header_text if header_text else f"Column_{len(headers)+1}") data = [] for row in table.find_all('tr')[1:]: cells = row.find_all('td') if len(cells) != len(headers): logger.warning("Row does not match header length. Skipping row.") continue row_data = { header: cell.get_text(strip=True) or None for header, cell in zip(headers, cells) } data.append(row_data) return data async def fetch_and_parse(scraper: AsyncScraper, pon: int, onu_group: int) -> Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: html_content = await scraper.fetch_pon_onu_optical_info(pon, onu_group) parsed_data = parse_html_table(html_content) if html_content else [] if not parsed_data: logger.warning(f"No data parsed for PON {pon}, ONU Group {onu_group}.") return {pon: {onu_group: parsed_data}} async def process_batch(scraper: AsyncScraper, batch: List[tuple]) -> List[Dict]: tasks = [fetch_and_parse(scraper, pon, onu_group) for pon, onu_group in batch] return await asyncio.gather(*tasks, return_exceptions=True) async def collect_data(scraper: AsyncScraper) -> Dict[int, Dict[int, List[Dict[str, Optional[str]]]]]: # Create all combinations of PON and ONU groups all_combinations = [ (pon, onu_group) for pon in CONFIG['pon_range'] for onu_group in CONFIG['onu_group_range'] ] # Process in batches aggregated_data: Dict[int, Dict[int, List[Dict[str, Optional[str]]]]] = {} for i in range(0, len(all_combinations), CONFIG['batch_size']): batch = all_combinations[i:i + CONFIG['batch_size']] logger.info(f"Processing batch {i//CONFIG['batch_size'] + 1}") results = await process_batch(scraper, batch) for result in results: if isinstance(result, Exception): logger.error(f"Batch task failed with exception: {result}") continue for pon, onu_data in result.items(): if pon not in aggregated_data: aggregated_data[pon] = {} aggregated_data[pon].update(onu_data) return aggregated_data async def handle_data(request): """ HTTP handler for the /data endpoint. Triggers data collection and returns the aggregated JSON. """ start_time = datetime.now() logger.info(f"Received /data request at {start_time}") async with AsyncScraper(CONFIG['base_url']) as scraper: if not await scraper.login(CONFIG['username'], CONFIG['password']): return web.Response(status=500, text="Login failed. Please check credentials.") aggregated_data = await collect_data(scraper) end_time = datetime.now() duration = end_time - start_time logger.info(f"Data collection complete. Duration: {duration}") return web.json_response(aggregated_data) async def init_app(): """ Initializes the aiohttp web application. """ app = web.Application() app.router.add_get('/data', handle_data) return app def main(): """ Main entry point to run the web server. """ app = asyncio.run(init_app()) web.run_app(app, host='0.0.0.0', port=8080) if __name__ == "__main__": main()-
Explanation of Modifications:
-
Import Statements:
- Added
from aiohttp import webto utilizeaiohttp's web server capabilities.
- Added
-
Removed File Saving:
- Eliminated the part of the original
main()function that was writing the aggregated data to a.jsonfile.
- Eliminated the part of the original
-
Web Server Handler (
handle_data):- Created an asynchronous handler
handle_datathat:- Logs the receipt of a
/datarequest. - Initializes the
AsyncScraperand performs the login. - If login is successful, it triggers the data collection process.
- Returns the aggregated data as a JSON response.
- Logs the duration of the data collection process.
- Logs the receipt of a
- Created an asynchronous handler
-
Data Collection Function (
collect_data):- Refactored the data aggregation logic into a separate
collect_datafunction, making it reusable within the web handler.
- Refactored the data aggregation logic into a separate
-
Web Application Initialization (
init_app):- Defined an
init_appfunction to set up theaiohttpweb application and route.
- Defined an
-
Main Function (
main):- Modified the
mainfunction to initialize and run the web server on0.0.0.0:8080. You can change the port as needed.
- Modified the
-
-
Running the API:
- Save the modified script to a file, for example,
scraper_api.py. - Run the script using:
python scraper_api.py - The server will start and listen on
http://0.0.0.0:8080.
- Save the modified script to a file, for example,
-
Accessing the JSON Data:
- To retrieve the aggregated JSON data, send a GET request to
http://<server_ip>:8080/data. - For example, using
curl:curl http://localhost:8080/data - The server will initiate the scraping process upon each request and return the resulting JSON data.
- To retrieve the aggregated JSON data, send a GET request to
-
Notes & Recommendations:
-
Performance Consideration:
- Since the data scraping process can be time-consuming, each request to
/datawill trigger a fresh scraping session. Depending on your use case and server capabilities, you might want to implement caching mechanisms or limit the frequency of requests to prevent server overload.
- Since the data scraping process can be time-consuming, each request to
-
Security Considerations:
- Ensure that access to the
/dataendpoint is secured, especially if deploying in a production environment. You might want to add authentication or restrict access to certain IP addresses.
- Ensure that access to the
-
Error Handling:
- The current implementation returns a 500 status code if login fails. You can enhance error handling by providing more detailed responses or handling other potential errors during data scraping.
-
Scalability:
- For higher scalability and better performance, consider deploying the API with a production-ready server like
gunicornpaired withaiohttp.
- For higher scalability and better performance, consider deploying the API with a production-ready server like
-
Logging:
- The script uses
colorlogfor colorful logging in the console. Ensure that the deployment environment supports ANSI color codes or adjust the logging configuration accordingly.
- The script uses
-
By following the above modifications, your script will function as a REST API, allowing clients to access the scraped JSON data via a simple HTTP GET request without the need to manage
.jsonfiles manually. -