Grok 4.1 Fast· xai
Frontier Reasoning Traces
Implement a Python script to scrape and translate website content.
- User
Implement a Python script to scrape and translate website content.
- Assistant
#!/usr/bin/env python3 """ Sherlock's Web Scraper & Translator =================================== A Python script to scrape text content from a website and translate it to a target language. Requirements: - pip install requests beautifulsoup4 googletrans==4.0.0-rc1 Usage: python scraper_translator.py <URL> [--from-lang <lang>] [--to-lang <lang>] [--output <file.html>] Examples: python scraper_translator.py https://example.com --to-lang es --output translated.html python scraper_translator.py https://example.com --from-lang fr --to-lang en Notes: - Uses googletrans (unofficial Google Translate API wrapper). - Extracts main text content, ignoring scripts, styles, nav, etc. - Translates in chunks to avoid rate limits. - Always respect robots.txt and website terms of service. - Output is a simple HTML file with translated content or printed to stdout. """ import argparse import requests from bs4 import BeautifulSoup from googletrans import Translator, LANGUAGES import sys import re def clean_text(text): """Clean and normalize extracted text.""" # Remove extra whitespace and newlines text = re.sub(r'\s+', ' ', text).strip() return text def scrape_website(url): """Scrape main text content from a website.""" headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } try: response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() except requests.RequestException as e: print(f"Error fetching {url}: {e}", file=sys.stderr) sys.exit(1) soup = BeautifulSoup(response.content, 'html.parser') # Remove unwanted elements for element in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']): element.decompose() # Extract text from main content areas main_content = soup.find('main') or soup.find('article') or soup.find('body') if main_content: text = main_content.get_text() else: text = soup.get_text() return clean_text(text) def translate_text(text, from_lang='auto', to_lang='en', chunk_size=5000): """Translate text in chunks to avoid API limits.""" translator = Translator() translated_parts = [] # Split into chunks chunks = [text[i:i+chunk_size] for i in range(0, len(text), chunk_size)] for i, chunk in enumerate(chunks): try: result = translator.translate(chunk, src=from_lang, dest=to_lang) translated_parts.append(result.text) print(f"Translated chunk {i+1}/{len(chunks)}...", file=sys.stderr) except Exception as e: print(f"Translation error for chunk {i+1}: {e}", file=sys.stderr) translated_parts.append(chunk) # Fallback to original return ' '.join(translated_parts) def create_output_html(original_url, original_text, translated_text, to_lang): """Create a simple HTML output with original and translated content.""" lang_name = LANGUAGES.get(to_lang, to_lang).title() html = f"""<!DOCTYPE html> <html lang="{to_lang}"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Translated: {original_url}</title> <style> body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }} .section {{ margin-bottom: 40px; }} h1 {{ color: #333; }} .original {{ background: #f9f9f9; padding: 20px; border-left: 4px solid #ccc; }} .translated {{ background: #e8f5e8; padding: 20px; border-left: 4px solid #4CAF50; }} </style> </head> <body> <h1>Web Content Translation ({lang_name})</h1> <p><strong>Original URL:</strong> <a href="{original_url}">{original_url}</a></p> <div class="section"> <h2>Original Content</h2> <div class="original">{original_text.replace('\n', '<br>')}</div> </div> <div class="section"> <h2>Translated Content ({lang_name})</h2> <div class="translated">{translated_text.replace('\n', '<br>')}</div> </div> </body> </html>""" return html def main(): parser = argparse.ArgumentParser(description="Scrape and translate website content.") parser.add_argument('url', help="URL to scrape") parser.add_argument('--from-lang', default='auto', help="Source language (default: auto)") parser.add_argument('--to-lang', default='en', help="Target language (default: en)") parser.add_argument('--output', '-o', help="Output HTML file (default: print to stdout)") args = parser.parse_args() print(f"Scraping {args.url}...", file=sys.stderr) original_text = scrape_website(args.url) if not original_text.strip(): print("No content found on the page.", file=sys.stderr) sys.exit(1) print("Translating...", file=sys.stderr) translated_text = translate_text(original_text, args.from_lang, args.to_lang) output_html = create_output_html(args.url, original_text, translated_text, args.to_lang) if args.output: try: with open(args.output, 'w', encoding='utf-8') as f: f.write(output_html) print(f"Translated content saved to {args.output}", file=sys.stderr) except IOError as e: print(f"Error writing to {args.output}: {e}", file=sys.stderr) sys.exit(1) else: print(output_html) if __name__ == '__main__': main()Quick Start
-
Install dependencies:
pip install requests beautifulsoup4 googletrans==4.0.0-rc1 -
Run the script:
python scraper_translator.py https://www.example.com --to-lang es --output output.html
Features
- Robust Scraping: Fetches HTML, removes junk (scripts, styles), extracts clean text from main content.
- Chunked Translation: Handles large pages without hitting Google Translate limits.
- Dual Output: Shows original + translated in a nice HTML file.
- Customizable: Languages via ISO codes (e.g., 'es' for Spanish, 'fr' for French). See Google Translate languages.
- Error Handling: Graceful failures with informative messages.
Ethical Note: This is for personal/educational use. Check the site's
robots.txtand ToS before scraping. Rate-limit in production.If you need enhancements (e.g., full DOM translation, proxy support, or DeepL integration), let me know! 🔍
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.