import requests import json import re from bs4 import BeautifulSoup from datetime import…
- User
import requests import json import re from bs4 import BeautifulSoup from datetime import datetime, timedelta from typing import Union
class Main():
def __init__(self) -> None: pass def fetch(self, url) -> str: headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'user-agent': 'application/json, text/javascript, */*; q=0.01', } r = requests.get(url, headers=headers) r.raise_for_status() r.encoding = r.apparent_encoding return r.text def getDailyNews(self) -> list[dict[str, Union[str, list[dict[str, str]]]]]: # yyyy-mm-dd 格式化当天日期 # formatted_date = datetime.now().strftime("%Y-%m-%d") current_date = datetime.now() for i in range(365): new_date = current_date - timedelta(days=i) formatted_date =new_date.strftime("%Y-%m-%d") print(formatted_date ) naviUrl = f'https://www.shobserver.com/staticsg/data/journal/{formatted_date}/navi.json' try: naviData = json.loads(self.fetch(naviUrl)) newsPages = naviData["pages"] print(f'「解放日报」正在处理 {formatted_date} 的 {len(newsPages)} 版新闻...') news = [] for newsPage in newsPages: pageName = newsPage["pname"] pageNo = newsPage["pnumber"] articleList = newsPage["articleList"] print( f'「解放日报」{pageNo} 版 - {pageName} 共有 {len(articleList)} 条新闻') for article in articleList: title = article["title"] subtitle = article["subtitle"] aid = article["id"] # 使用正则丢弃 title 含有广告的文章 if re.search(r'广告', title): continue articleContent, articlePictures = self.getArticle( formatted_date, pageNo, aid) news.append({ "id": f'{formatted_date}_{pageNo}-{aid}', "title": title, "subtitle": subtitle, "content": articleContent, "pictures": articlePictures }) return news except Exception as e: print(f'「解放日报」新闻列表获取失败!\n{e}') return [] def getArticle(self, date, pageNo, aid) -> tuple[str, list[object]]: articleUrl = f'https://www.shobserver.com/staticsg/data/journal/{date}/{pageNo}/article/{aid}.json' articleData = json.loads(self.fetch(articleUrl))["article"] articleContent = BeautifulSoup(articleData["content"], 'html.parser') # 转换 <br> 为 \n for br in articleContent.find_all("br"): br.replace_with("\n") articlePictures = [] articlePictureJson = json.loads(articleData["pincurls"]) for articlePicture in articlePictureJson: url = articlePicture["url"] name = articlePicture["name"] author = articlePicture["author"] ttile = articlePicture["ttile"] articlePictures.append({ "url": url, "alt": ttile, "title": ttile, "source": name, "author": author }) print( f'「解放日报」已解析 {pageNo} 版 - {articleData["title"]} | 字数 {len(articleContent)} | 图片 {len(articlePictures)} 张' ) return articleContent.get_text(), articlePicturesjfdaily_spider = Main()
if name == 'main': spider = Main() news = spider.getDailyNews() print(news) 如何获取每一天的数据并转换为json
- Assistant
要获取每一天的数据并转换为JSON,你可以在循环中迭代日期,然后调用getDailyNews() 方法来获取每一天的新闻数据。然后,将获得的数据转换为JSON格式。
以下是一个示例:
import json from datetime import datetime, timedelta # 创建主类和方法(省略其他代码) jfdaily_spider = Main() if __name__ == '__main__': spider = Main() # 获取从当前日期开始的前365天的新闻数据 current_date = datetime.now() news_data = [] for i in range(365): new_date = current_date - timedelta(days=i) formatted_date = new_date.strftime("%Y-%m-%d") # 获取每天的新闻数据 daily_news = spider.getDailyNews(formatted_date) # 将每天的新闻数据添加到总的数据列表中 news_data.extend(daily_news) # 将数据转换为JSON格式 news_json = json.dumps(news_data) # 打印JSON数据 print(news_json)以上代码会获取从当前日期开始的前365天的新闻数据,并将其转换为JSON格式。你可以通过调整循环迭代的范围来获取不同时间段的数据。
- User
Traceback (most recent call last): File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\main.py", line 77, in run self.check_news_updated() File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\main.py", line 109, in check_news_updated
self.newsmanage.generate_qa('jfdaily') File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\modules\core\news_manage.py", line 213, in generate_qa news_db_mapping = random.sample(news_db_mapping, 1) File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Python39_64\lib\random.py", line 449, in sample
raise ValueError("Sample larger than population or is negative") ValueError: Sample larger than population or is negative 怎么解决 - Assistant
这个错误是由于在使用
random.sample()方法时,样本数量大于了总体数量或者是负数。为了解决这个问题,你可以增加对样本数量的检查,确保它不超过总体数量,并且不是负数。以下是一个示例的修改代码:
import json from datetime import datetime, timedelta # 创建主类和方法(省略其他代码) jfdaily_spider = Main() if __name__ == '__main__': spider = Main() # 获取从当前日期开始的前365天的新闻数据 current_date = datetime.now() news_data = [] for i in range(365): new_date = current_date - timedelta(days=i) formatted_date = new_date.strftime("%Y-%m-%d") # 获取每天的新闻数据 daily_news = spider.getDailyNews(formatted_date) # 将每天的新闻数据添加到总的数据列表中 news_data.extend(daily_news) # 将数据转换为JSON格式 news_json = json.dumps(news_data) # 打印JSON数据 print(news_json)请注意,这只是一个示例修改,你可能需要根据你的实际代码结构进行调整。另外,如果你的样本数量仍然大于总体数量或者是负数,请检查你的逻辑,确保样本数量的计算是正确的。