Skip to content
TheChatArchiveThe Chat Archive
Mistralmistral-medium-2505· Mistral AIHugging Face LMArena Human Preference 140K

Can you please build me a web scraper in go

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Coherecommand-a-03-2025’s answer.
  1. User

    Can you please build me a web scraper in go

  2. Mistral Assistant

    Web Scraper in Go

    Here's a basic web scraper implementation in Go using the popular goquery package, which provides a jQuery-like syntax for HTML parsing.

    Prerequisites

    First, install the required package:

    go get github.com/PuerkitoBio/goquery
    

    Basic Web Scraper Implementation

    package main
    
    import (
    	"fmt"
    	"log"
    	"net/http"
    	"os"
    	"strings"
    
    	"github.com/PuerkitoBio/goquery"
    )
    
    type ScrapedData struct {
    	Title string
    	Links []string
    }
    
    func main() {
    	// Check if URL is provided as argument
    	if len(os.Args) < 2 {
    		fmt.Println("Please provide a URL to scrape")
    		fmt.Println("Usage: go run scraper.go <url>")
    		return
    	}
    	url := os.Args[1]
    
    	// Validate URL
    	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
    		url = "https://" + url
    	}
    
    	// Fetch the HTML document
    	doc, err := fetchHTML(url)
    	if err != nil {
    		log.Fatal(err)
    	}
    
    	// Scrape the data
    	data := scrapeData(doc)
    
    	// Print the results
    	fmt.Println("Title:", data.Title)
    	fmt.Println("Links found:")
    	for _, link := range data.Links {
    		fmt.Println(link)
    	}
    }
    
    func fetchHTML(url string) (*goquery.Document, error) {
    	// Make HTTP request
    	res, err := http.Get(url)
    	if err != nil {
    		return nil, fmt.Errorf("failed to fetch URL: %v", err)
    	}
    	defer res.Body.Close()
    
    	// Check status code
    	if res.StatusCode != 200 {
    		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
    	}
    
    	// Parse the HTML document
    	doc, err := goquery.NewDocumentFromReader(res.Body)
    	if err != nil {
    		return nil, fmt.Errorf("failed to parse HTML: %v", err)
    	}
    
    	return doc, nil
    }
    
    func scrapeData(doc *goquery.Document) ScrapedData {
    	var data ScrapedData
    
    	// Extract title
    	data.Title = doc.Find("title").Text()
    
    	// Extract all links
    	doc.Find("a").Each(func(i int, s *goquery.Selection) {
    		href, exists := s.Attr("href")
    		if exists {
    			data.Links = append(data.Links, href)
    		}
    	})
    
    	return data
    }
    

    How to Use

    1. Save the code to a file named scraper.go
    2. Run it with a URL as argument:
      go run scraper.go https://example.com
      

    Advanced Features to Consider Adding

    1. Concurrency: Use goroutines to scrape multiple pages simultaneously
    2. Persistence: Save results to a database or file
    3. Rate Limiting: Add delays between requests to be polite to servers
    4. User-Agent Rotation: Rotate user agents to avoid detection
    5. Proxy Support: Add proxy support for large-scale scraping
    6. Error Handling: More robust error handling and retries
    7. Configuration: Make the scraper configurable via config files

    Example with Concurrency

    Here's a more advanced version that scrapes multiple pages concurrently:

    package main
    
    import (
    	"fmt"
    	"log"
    	"net/http"
    	"sync"
    
    	"github.com/PuerkitoBio/goquery"
    )
    
    type PageData struct {
    	URL   string
    	Title string
    	Links []string
    }
    
    func scrapePage(url string, wg *sync.WaitGroup, results chan<- PageData) {
    	defer wg.Done()
    
    	res, err := http.Get(url)
    	if err != nil {
    		log.Printf("Error fetching %s: %v", url, err)
    		return
    	}
    	defer res.Body.Close()
    
    	doc, err := goquery.NewDocumentFromReader(res.Body)
    	if err != nil {
    		log.Printf("Error parsing %s: %v", url, err)
    		return
    	}
    
    	var links []string
    	doc.Find("a").Each(func(i int, s *goquery.Selection) {
    		if href, exists := s.Attr("href"); exists {
    			links = append(links, href)
    		}
    	})
    
    	results <- PageData{
    		URL:   url,
    		Title: doc.Find("title").Text(),
    		Links: links,
    	}
    }
    
    func main() {
    	urls := []string{
    		"https://example.com",
    		"https://golang.org",
    		"https://github.com",
    	}
    
    	var wg sync.WaitGroup
    	results := make(chan PageData, len(urls))
    
    	for _, url := range urls {
    		wg.Add(1)
    		go scrapePage(url, &wg, results)
    	}
    
    	// Wait for all goroutines to finish
    	go func() {
    		wg.Wait()
    		close(results)
    	}()
    
    	// Process results
    	for data := range results {
    		fmt.Printf("Page: %s\nTitle: %s\nLinks: %d\n\n", data.URL, data.Title, len(data.Links))
    	}
    }
    

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.