写一个简单的爬虫
- User
写一个简单的爬虫
Assistant
我可以帮你写一个简单的Python爬虫示例。这里是一个基础的网页爬虫,用于获取网页内容:
import requests from bs4 import BeautifulSoup import time def simple_web_scraper(url): """ 简单的网页爬虫函数 """ try: # 设置请求头,模拟浏览器访问 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } # 发送GET请求 response = requests.get(url, headers=headers) response.raise_for_status() # 检查请求是否成功 # 使用BeautifulSoup解析HTML soup = BeautifulSoup(response.text, 'html.parser') # 获取页面标题 title = soup.find('title').text if soup.find('title') else 'No title found' print(f"页面标题: {title}") # 获取所有段落文本 paragraphs = soup.find_all('p') print(f"\n找到 {len(paragraphs)} 个段落") # 打印前3个段落 for i, p in enumerate(paragraphs[:3]): print(f"\n段落 {i+1}: {p.text.strip()[:100]}...") # 获取所有链接 links = soup.find_all('a', href=True) print(f"\n找到 {len(links)} 个链接") # 打印前5个链接 for i, link in enumerate(links[:5]): print(f"链接 {i+1}: {link['href']}") except requests.RequestException as e: print(f"请求错误: {e}") except Exception as e: print(f"发生错误: {e}") # 使用示例 if __name__ == "__main__": # 爬取一个示例网站 url = "https://example.com" simple_web_scraper(url) # 添加延时,避免请求过快 time.sleep(1)使用前需要安装必要的库:
pip install requests beautifulsoup4这个爬虫包含了以下基本功能:
- 发送HTTP请求:使用requests库获取网页内容
- 设置请求头:模拟浏览器访问,避免被识别为爬虫
- 解析HTML:使用BeautifulSoup解析网页结构
- 提取数据:获取标题、段落文本和链接
- 错误处理:处理可能出现的请求错误
重要提醒:
- 使用爬虫时要遵守网站的robots.txt规则
- 控制请求频率,避免对服务器造成负担
- 尊重网站的使用条款和版权
- 某些网站可能需要处理JavaScript渲染的内容,这时需要使用Selenium等工具
需要更复杂的功能吗?比如处理分页、保存数据到文件、或者爬取特定类型的内容?
- User
fastapi连接pgsql
Assistant
我来帮你创建一个使用FastAPI连接PostgreSQL的完整示例:
1. 安装依赖
pip install fastapi uvicorn sqlalchemy psycopg2-binary python-dotenv2. 项目结构
project/ ├── app/ │ ├── __init__.py │ ├── main.py │ ├── database.py │ ├── models.py │ ├── schemas.py │ └── crud.py ├── .env └── requirements.txt3. 数据库配置 (
.env)DATABASE_URL=postgresql://username:password@localhost:5432/dbname4. 数据库连接 (
app/database.py)from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker import os from dotenv import load_dotenv load_dotenv() DATABASE_URL = os.getenv("DATABASE_URL") engine = create_engine(DATABASE_URL) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) Base = declarative_base() # 依赖项 def get_db(): db = SessionLocal() try: yield db finally: db.close()5. 数据模型 (
app/models.py)from sqlalchemy import Column, Integer, String, DateTime, Boolean from sqlalchemy.sql import func from .database import Base class User(Base): __tablename__ = "users" id = Column(Integer, primary_key=True, index=True) email = Column(String, unique=True, index=True, nullable=False) username = Column(String, unique=True, index=True, nullable=False) full_name = Column(String) is_active = Column(Boolean, default=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), onupdate=func.now())6. Pydantic模型 (
app/schemas.py)from pydantic import BaseModel, EmailStr from datetime import datetime from typing import Optional class UserBase(BaseModel): email: EmailStr username: str full_name: Optional[str] = None class UserCreate(UserBase): pass class UserUpdate(BaseModel): email: Optional[EmailStr] = None username: Optional[str] = None full_name: Optional[str] = None is_active: Optional[bool] = None class User(UserBase): id: int is_active: bool created_at: datetime updated_at: Optional[datetime] = None class Config: from_attributes = True7. CRUD操作 (
app/crud.py)from sqlalchemy.orm import Session from . import models, schemas def get_user(db: Session, user_id: int): return db.query(models.User).filter(models.User.id == user_id).first() def get_user_by_email(db: Session, email: str): return db.query(models.User).filter(models.User.email == email).first() def get_users(db: Session, skip: int = 0, limit: int = 100): return db.query(models.User).offset(skip).limit(limit).all() def create_user(db: Session, user: schemas.UserCreate): db_user = models.User(**user.dict()) db.add(db_user) db.commit() db.refresh(db_user) return db_user def update_user(db: Session, user_id: int, user: schemas.UserUpdate): db_user = db.query(models.User).filter(models.User.id == user_id).first() if db_user: update_data = user.dict(exclude_unset=True) for field, value in update_data.items(): setattr(db_user, field, value) db.commit() db.refresh(db_user) return db_user def delete_user(db: Session, user_id: int): db_user = db.query(models.User).filter(models.User.id == user_id).first() if db_user: db.delete(db_user) db.commit() return db_user8. FastAPI主应用 (
app/main.py)from fastapi import FastAPI, Depends, HTTPException from sqlalchemy.orm import Session from typing import List from . import crud, models, schemas from .database import SessionLocal, engine, get_db # 创建数据库表 models.Base.metadata.create_all(bind=engine) app = FastAPI(title="FastAPI PostgreSQL Example") @app.get("/") def read_root(): return {"message": "Welcome to FastAPI with PostgreSQL"} @app.post("/users/", response_model=schemas.User) def create_user(user: schemas.UserCreate, db: Session = Depends(get_db)): db_user = crud.get_user_by_email(db, email=user.email) if db_user: raise HTTPException(status_code=400, detail="Email already registered") return crud.create_user(db=db, user=user) @app.get("/users/", response_model=List[schemas.User]) def read_users(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)): users = crud.get_users(db, skip=skip, limit=limit) return users @app.get("/users/{user_id}", response_model=schemas.User) def read_user(user_id: int, db: Session = Depends(get_db)): db_user = crud.get_user(db, user_id=user_id) if db_user is None: raise HTTPException(status_code=404, detail="User not found") return db_user @app.put("/users/{user_id}", response_model=schemas.User) def update_user(user_id: int, user: schemas.UserUpdate, db: Session = Depends(get_db)): db_user = crud.update_user(db, user_id=user_id, user=user) if db_user is None: raise HTTPException(status_code=404, detail="User not found") return db_user @app.delete("/users/{user_id}") def delete_user(user_id: int, db: Session = Depends(get_db)): db_user = crud.delete_user(db, user_id=user_id) if db_user is None: raise HTTPException(status_code=404, detail="User not found") return {"message": "User deleted successfully"}9. 运行应用
uvicorn app.main:app --reload10. 高级功能示例
异步数据库连接 (使用asyncpg)
# 安装: pip install databases asyncpg from databases import Database from fastapi import FastAPI DATABASE_URL = "postgresql://user:password@localhost/dbname" database = Database(DATABASE_URL) app = FastAPI() @app.on_event("startup") async def startup(): await database.connect() @app.on_event("shutdown") async def shutdown(): await database.disconnect() @app.get("/users/") async def read_users(): query = "SELECT * FROM users" return await database.fetch_all(query=query)数据库迁移 (使用Alembic)
# 安装 pip install alembic # 初始化 alembic init alembic # 创建迁移 alembic revision --autogenerate -m "Initial migration" # 应用迁移 alembic upgrade head这个示例提供了一个完整的FastAPI + PostgreSQL应用,包括:
- 数据库连接管理
- 模型定义
- CRUD操作
- RESTful API端点
- 错误处理
- 数据验证
你可以根据具体需求进行修改和扩展。需要帮助实现特定功能吗?
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.