Skip to content
TheChatArchiveThe Chat Archive
Claudeclaude-opus-4-20250514· AnthropicHugging Face LMArena Human Preference 140K

写一个简单的爬虫

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Claudeclaude-3-5-sonnet-20241022’s answer.
  1. User

    写一个简单的爬虫

  2. Claude Assistant

    我可以帮你写一个简单的Python爬虫示例。这里是一个基础的网页爬虫,用于获取网页内容:

    import requests
    from bs4 import BeautifulSoup
    import time
    
    def simple_web_scraper(url):
        """
        简单的网页爬虫函数
        """
        try:
            # 设置请求头,模拟浏览器访问
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            }
            
            # 发送GET请求
            response = requests.get(url, headers=headers)
            response.raise_for_status()  # 检查请求是否成功
            
            # 使用BeautifulSoup解析HTML
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 获取页面标题
            title = soup.find('title').text if soup.find('title') else 'No title found'
            print(f"页面标题: {title}")
            
            # 获取所有段落文本
            paragraphs = soup.find_all('p')
            print(f"\n找到 {len(paragraphs)} 个段落")
            
            # 打印前3个段落
            for i, p in enumerate(paragraphs[:3]):
                print(f"\n段落 {i+1}: {p.text.strip()[:100]}...")
            
            # 获取所有链接
            links = soup.find_all('a', href=True)
            print(f"\n找到 {len(links)} 个链接")
            
            # 打印前5个链接
            for i, link in enumerate(links[:5]):
                print(f"链接 {i+1}: {link['href']}")
                
        except requests.RequestException as e:
            print(f"请求错误: {e}")
        except Exception as e:
            print(f"发生错误: {e}")
    
    # 使用示例
    if __name__ == "__main__":
        # 爬取一个示例网站
        url = "https://example.com"
        simple_web_scraper(url)
        
        # 添加延时,避免请求过快
        time.sleep(1)
    

    使用前需要安装必要的库:

    pip install requests beautifulsoup4
    

    这个爬虫包含了以下基本功能:

    1. 发送HTTP请求:使用requests库获取网页内容
    2. 设置请求头:模拟浏览器访问,避免被识别为爬虫
    3. 解析HTML:使用BeautifulSoup解析网页结构
    4. 提取数据:获取标题、段落文本和链接
    5. 错误处理:处理可能出现的请求错误

    重要提醒:

    • 使用爬虫时要遵守网站的robots.txt规则
    • 控制请求频率,避免对服务器造成负担
    • 尊重网站的使用条款和版权
    • 某些网站可能需要处理JavaScript渲染的内容,这时需要使用Selenium等工具

    需要更复杂的功能吗?比如处理分页、保存数据到文件、或者爬取特定类型的内容?

  3. User

    fastapi连接pgsql

  4. Claude Assistant

    我来帮你创建一个使用FastAPI连接PostgreSQL的完整示例:

    1. 安装依赖

    pip install fastapi uvicorn sqlalchemy psycopg2-binary python-dotenv
    

    2. 项目结构

    project/
    ├── app/
    │   ├── __init__.py
    │   ├── main.py
    │   ├── database.py
    │   ├── models.py
    │   ├── schemas.py
    │   └── crud.py
    ├── .env
    └── requirements.txt
    

    3. 数据库配置 (.env)

    DATABASE_URL=postgresql://username:password@localhost:5432/dbname
    

    4. 数据库连接 (app/database.py)

    from sqlalchemy import create_engine
    from sqlalchemy.ext.declarative import declarative_base
    from sqlalchemy.orm import sessionmaker
    import os
    from dotenv import load_dotenv
    
    load_dotenv()
    
    DATABASE_URL = os.getenv("DATABASE_URL")
    
    engine = create_engine(DATABASE_URL)
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    
    Base = declarative_base()
    
    # 依赖项
    def get_db():
        db = SessionLocal()
        try:
            yield db
        finally:
            db.close()
    

    5. 数据模型 (app/models.py)

    from sqlalchemy import Column, Integer, String, DateTime, Boolean
    from sqlalchemy.sql import func
    from .database import Base
    
    class User(Base):
        __tablename__ = "users"
    
        id = Column(Integer, primary_key=True, index=True)
        email = Column(String, unique=True, index=True, nullable=False)
        username = Column(String, unique=True, index=True, nullable=False)
        full_name = Column(String)
        is_active = Column(Boolean, default=True)
        created_at = Column(DateTime(timezone=True), server_default=func.now())
        updated_at = Column(DateTime(timezone=True), onupdate=func.now())
    

    6. Pydantic模型 (app/schemas.py)

    from pydantic import BaseModel, EmailStr
    from datetime import datetime
    from typing import Optional
    
    class UserBase(BaseModel):
        email: EmailStr
        username: str
        full_name: Optional[str] = None
    
    class UserCreate(UserBase):
        pass
    
    class UserUpdate(BaseModel):
        email: Optional[EmailStr] = None
        username: Optional[str] = None
        full_name: Optional[str] = None
        is_active: Optional[bool] = None
    
    class User(UserBase):
        id: int
        is_active: bool
        created_at: datetime
        updated_at: Optional[datetime] = None
    
        class Config:
            from_attributes = True
    

    7. CRUD操作 (app/crud.py)

    from sqlalchemy.orm import Session
    from . import models, schemas
    
    def get_user(db: Session, user_id: int):
        return db.query(models.User).filter(models.User.id == user_id).first()
    
    def get_user_by_email(db: Session, email: str):
        return db.query(models.User).filter(models.User.email == email).first()
    
    def get_users(db: Session, skip: int = 0, limit: int = 100):
        return db.query(models.User).offset(skip).limit(limit).all()
    
    def create_user(db: Session, user: schemas.UserCreate):
        db_user = models.User(**user.dict())
        db.add(db_user)
        db.commit()
        db.refresh(db_user)
        return db_user
    
    def update_user(db: Session, user_id: int, user: schemas.UserUpdate):
        db_user = db.query(models.User).filter(models.User.id == user_id).first()
        if db_user:
            update_data = user.dict(exclude_unset=True)
            for field, value in update_data.items():
                setattr(db_user, field, value)
            db.commit()
            db.refresh(db_user)
        return db_user
    
    def delete_user(db: Session, user_id: int):
        db_user = db.query(models.User).filter(models.User.id == user_id).first()
        if db_user:
            db.delete(db_user)
            db.commit()
        return db_user
    

    8. FastAPI主应用 (app/main.py)

    from fastapi import FastAPI, Depends, HTTPException
    from sqlalchemy.orm import Session
    from typing import List
    
    from . import crud, models, schemas
    from .database import SessionLocal, engine, get_db
    
    # 创建数据库表
    models.Base.metadata.create_all(bind=engine)
    
    app = FastAPI(title="FastAPI PostgreSQL Example")
    
    @app.get("/")
    def read_root():
        return {"message": "Welcome to FastAPI with PostgreSQL"}
    
    @app.post("/users/", response_model=schemas.User)
    def create_user(user: schemas.UserCreate, db: Session = Depends(get_db)):
        db_user = crud.get_user_by_email(db, email=user.email)
        if db_user:
            raise HTTPException(status_code=400, detail="Email already registered")
        return crud.create_user(db=db, user=user)
    
    @app.get("/users/", response_model=List[schemas.User])
    def read_users(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
        users = crud.get_users(db, skip=skip, limit=limit)
        return users
    
    @app.get("/users/{user_id}", response_model=schemas.User)
    def read_user(user_id: int, db: Session = Depends(get_db)):
        db_user = crud.get_user(db, user_id=user_id)
        if db_user is None:
            raise HTTPException(status_code=404, detail="User not found")
        return db_user
    
    @app.put("/users/{user_id}", response_model=schemas.User)
    def update_user(user_id: int, user: schemas.UserUpdate, db: Session = Depends(get_db)):
        db_user = crud.update_user(db, user_id=user_id, user=user)
        if db_user is None:
            raise HTTPException(status_code=404, detail="User not found")
        return db_user
    
    @app.delete("/users/{user_id}")
    def delete_user(user_id: int, db: Session = Depends(get_db)):
        db_user = crud.delete_user(db, user_id=user_id)
        if db_user is None:
            raise HTTPException(status_code=404, detail="User not found")
        return {"message": "User deleted successfully"}
    

    9. 运行应用

    uvicorn app.main:app --reload
    

    10. 高级功能示例

    异步数据库连接 (使用asyncpg)

    # 安装: pip install databases asyncpg
    
    from databases import Database
    from fastapi import FastAPI
    
    DATABASE_URL = "postgresql://user:password@localhost/dbname"
    database = Database(DATABASE_URL)
    
    app = FastAPI()
    
    @app.on_event("startup")
    async def startup():
        await database.connect()
    
    @app.on_event("shutdown")
    async def shutdown():
        await database.disconnect()
    
    @app.get("/users/")
    async def read_users():
        query = "SELECT * FROM users"
        return await database.fetch_all(query=query)
    

    数据库迁移 (使用Alembic)

    # 安装
    pip install alembic
    
    # 初始化
    alembic init alembic
    
    # 创建迁移
    alembic revision --autogenerate -m "Initial migration"
    
    # 应用迁移
    alembic upgrade head
    

    这个示例提供了一个完整的FastAPI + PostgreSQL应用,包括:

    • 数据库连接管理
    • 模型定义
    • CRUD操作
    • RESTful API端点
    • 错误处理
    • 数据验证

    你可以根据具体需求进行修改和扩展。需要帮助实现特定功能吗?

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.