Skip to content
TheChatArchiveThe Chat Archive
Grok 4.1 Fast· xaiHugging Face Frontier Reasoning Traces

Implement a Python script to scrape and translate website content.

  1. User

    Implement a Python script to scrape and translate website content.

  2. Assistant
    #!/usr/bin/env python3
    """
    Sherlock's Web Scraper & Translator
    ===================================
    A Python script to scrape text content from a website and translate it to a target language.
    
    Requirements:
    - pip install requests beautifulsoup4 googletrans==4.0.0-rc1
    
    Usage:
        python scraper_translator.py <URL> [--from-lang <lang>] [--to-lang <lang>] [--output <file.html>]
    
    Examples:
        python scraper_translator.py https://example.com --to-lang es --output translated.html
        python scraper_translator.py https://example.com --from-lang fr --to-lang en
    
    Notes:
    - Uses googletrans (unofficial Google Translate API wrapper).
    - Extracts main text content, ignoring scripts, styles, nav, etc.
    - Translates in chunks to avoid rate limits.
    - Always respect robots.txt and website terms of service.
    - Output is a simple HTML file with translated content or printed to stdout.
    """
    
    import argparse
    import requests
    from bs4 import BeautifulSoup
    from googletrans import Translator, LANGUAGES
    import sys
    import re
    
    def clean_text(text):
        """Clean and normalize extracted text."""
        # Remove extra whitespace and newlines
        text = re.sub(r'\s+', ' ', text).strip()
        return text
    
    def scrape_website(url):
        """Scrape main text content from a website."""
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
        try:
            response = requests.get(url, headers=headers, timeout=10)
            response.raise_for_status()
        except requests.RequestException as e:
            print(f"Error fetching {url}: {e}", file=sys.stderr)
            sys.exit(1)
    
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Remove unwanted elements
        for element in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
            element.decompose()
        
        # Extract text from main content areas
        main_content = soup.find('main') or soup.find('article') or soup.find('body')
        if main_content:
            text = main_content.get_text()
        else:
            text = soup.get_text()
        
        return clean_text(text)
    
    def translate_text(text, from_lang='auto', to_lang='en', chunk_size=5000):
        """Translate text in chunks to avoid API limits."""
        translator = Translator()
        translated_parts = []
        
        # Split into chunks
        chunks = [text[i:i+chunk_size] for i in range(0, len(text), chunk_size)]
        
        for i, chunk in enumerate(chunks):
            try:
                result = translator.translate(chunk, src=from_lang, dest=to_lang)
                translated_parts.append(result.text)
                print(f"Translated chunk {i+1}/{len(chunks)}...", file=sys.stderr)
            except Exception as e:
                print(f"Translation error for chunk {i+1}: {e}", file=sys.stderr)
                translated_parts.append(chunk)  # Fallback to original
        
        return ' '.join(translated_parts)
    
    def create_output_html(original_url, original_text, translated_text, to_lang):
        """Create a simple HTML output with original and translated content."""
        lang_name = LANGUAGES.get(to_lang, to_lang).title()
        html = f"""<!DOCTYPE html>
    <html lang="{to_lang}">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>Translated: {original_url}</title>
        <style>
            body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }}
            .section {{ margin-bottom: 40px; }}
            h1 {{ color: #333; }}
            .original {{ background: #f9f9f9; padding: 20px; border-left: 4px solid #ccc; }}
            .translated {{ background: #e8f5e8; padding: 20px; border-left: 4px solid #4CAF50; }}
        </style>
    </head>
    <body>
        <h1>Web Content Translation ({lang_name})</h1>
        <p><strong>Original URL:</strong> <a href="{original_url}">{original_url}</a></p>
        
        <div class="section">
            <h2>Original Content</h2>
            <div class="original">{original_text.replace('\n', '<br>')}</div>
        </div>
        
        <div class="section">
            <h2>Translated Content ({lang_name})</h2>
            <div class="translated">{translated_text.replace('\n', '<br>')}</div>
        </div>
    </body>
    </html>"""
        return html
    
    def main():
        parser = argparse.ArgumentParser(description="Scrape and translate website content.")
        parser.add_argument('url', help="URL to scrape")
        parser.add_argument('--from-lang', default='auto', help="Source language (default: auto)")
        parser.add_argument('--to-lang', default='en', help="Target language (default: en)")
        parser.add_argument('--output', '-o', help="Output HTML file (default: print to stdout)")
        
        args = parser.parse_args()
        
        print(f"Scraping {args.url}...", file=sys.stderr)
        original_text = scrape_website(args.url)
        
        if not original_text.strip():
            print("No content found on the page.", file=sys.stderr)
            sys.exit(1)
        
        print("Translating...", file=sys.stderr)
        translated_text = translate_text(original_text, args.from_lang, args.to_lang)
        
        output_html = create_output_html(args.url, original_text, translated_text, args.to_lang)
        
        if args.output:
            try:
                with open(args.output, 'w', encoding='utf-8') as f:
                    f.write(output_html)
                print(f"Translated content saved to {args.output}", file=sys.stderr)
            except IOError as e:
                print(f"Error writing to {args.output}: {e}", file=sys.stderr)
                sys.exit(1)
        else:
            print(output_html)
    
    if __name__ == '__main__':
        main()
    

    Quick Start

    1. Install dependencies:

      pip install requests beautifulsoup4 googletrans==4.0.0-rc1
      
    2. Run the script:

      python scraper_translator.py https://www.example.com --to-lang es --output output.html
      

    Features

    • Robust Scraping: Fetches HTML, removes junk (scripts, styles), extracts clean text from main content.
    • Chunked Translation: Handles large pages without hitting Google Translate limits.
    • Dual Output: Shows original + translated in a nice HTML file.
    • Customizable: Languages via ISO codes (e.g., 'es' for Spanish, 'fr' for French). See Google Translate languages.
    • Error Handling: Graceful failures with informative messages.

    Ethical Note: This is for personal/educational use. Check the site's robots.txt and ToS before scraping. Rate-limit in production.

    If you need enhancements (e.g., full DOM translation, proxy support, or DeepL integration), let me know! 🔍

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.