Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0613· OpenAIAi2 WildChat-4.8M (AI2)

import requests import json import re from bs4 import BeautifulSoup from datetime import…

  1. User

    import requests import json import re from bs4 import BeautifulSoup from datetime import datetime, timedelta from typing import Union

    class Main():

    def __init__(self) -> None:
        pass
    
    def fetch(self, url) -> str:
        headers = {
            'accept':
            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'user-agent': 'application/json, text/javascript, */*; q=0.01',
        }
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    
    def getDailyNews(self) -> list[dict[str, Union[str, list[dict[str, str]]]]]:
        # yyyy-mm-dd 格式化当天日期
       
        # formatted_date = datetime.now().strftime("%Y-%m-%d")
        current_date = datetime.now()
    
        for i in range(365):
            new_date = current_date - timedelta(days=i)
            formatted_date =new_date.strftime("%Y-%m-%d")
            print(formatted_date )
    
        naviUrl = f'https://www.shobserver.com/staticsg/data/journal/{formatted_date}/navi.json'
    
        try:
            naviData = json.loads(self.fetch(naviUrl))
            newsPages = naviData["pages"]
            print(f'「解放日报」正在处理 {formatted_date} 的 {len(newsPages)} 版新闻...')
    
            news = []
            for newsPage in newsPages:
                pageName = newsPage["pname"]
                pageNo = newsPage["pnumber"]
                articleList = newsPage["articleList"]
                print(
                    f'「解放日报」{pageNo} 版 - {pageName} 共有 {len(articleList)} 条新闻')
                for article in articleList:
                    title = article["title"]
                    subtitle = article["subtitle"]
                    aid = article["id"]
    
                    # 使用正则丢弃 title 含有广告的文章
                    if re.search(r'广告', title):
                        continue
    
                    articleContent, articlePictures = self.getArticle(
                        formatted_date, pageNo, aid)
                    news.append({
                        "id": f'{formatted_date}_{pageNo}-{aid}',
                        "title": title,
                        "subtitle": subtitle,
                        "content": articleContent,
                        "pictures": articlePictures
                    })
    
            return news
    
        except Exception as e:
            print(f'「解放日报」新闻列表获取失败!\n{e}')
            return []
    
    def getArticle(self, date, pageNo, aid) -> tuple[str, list[object]]:
        articleUrl = f'https://www.shobserver.com/staticsg/data/journal/{date}/{pageNo}/article/{aid}.json'
    
        articleData = json.loads(self.fetch(articleUrl))["article"]
        articleContent = BeautifulSoup(articleData["content"], 'html.parser')
        # 转换 <br> 为 \n
        for br in articleContent.find_all("br"):
            br.replace_with("\n")
    
        articlePictures = []
        articlePictureJson = json.loads(articleData["pincurls"])
        for articlePicture in articlePictureJson:
            url = articlePicture["url"]
            name = articlePicture["name"]
            author = articlePicture["author"]
            ttile = articlePicture["ttile"]
            articlePictures.append({
                "url": url,
                "alt": ttile,
                "title": ttile,
                "source": name,
                "author": author
            })
    
        print(
            f'「解放日报」已解析 {pageNo} 版 - {articleData["title"]} | 字数 {len(articleContent)} | 图片 {len(articlePictures)} 张'
        )
        return articleContent.get_text(), articlePictures
    

    jfdaily_spider = Main()

    if name == 'main': spider = Main() news = spider.getDailyNews() print(news) 如何获取每一天的数据并转换为json

  2. Assistant

    要获取每一天的数据并转换为JSON,你可以在循环中迭代日期,然后调用getDailyNews() 方法来获取每一天的新闻数据。然后,将获得的数据转换为JSON格式。

    以下是一个示例:

    import json
    from datetime import datetime, timedelta
    
    # 创建主类和方法(省略其他代码)
    
    jfdaily_spider = Main()
    
    if __name__ == '__main__':
        spider = Main()
        
        # 获取从当前日期开始的前365天的新闻数据
        current_date = datetime.now()
        news_data = []
        
        for i in range(365):
            new_date = current_date - timedelta(days=i)
            formatted_date = new_date.strftime("%Y-%m-%d")
            
            # 获取每天的新闻数据
            daily_news = spider.getDailyNews(formatted_date)
            
            # 将每天的新闻数据添加到总的数据列表中
            news_data.extend(daily_news)
        
        # 将数据转换为JSON格式
        news_json = json.dumps(news_data)
        
        # 打印JSON数据
        print(news_json)
    

    以上代码会获取从当前日期开始的前365天的新闻数据,并将其转换为JSON格式。你可以通过调整循环迭代的范围来获取不同时间段的数据。

  3. User

    Traceback (most recent call last): File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\main.py", line 77, in run self.check_news_updated() File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\main.py", line 109, in check_news_updated
    self.newsmanage.generate_qa('jfdaily') File "f:\tpyrced_inf-delivery-sjtu-main\inf-delivery-sjtu-main\modules\core\news_manage.py", line 213, in generate_qa news_db_mapping = random.sample(news_db_mapping, 1) File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Python39_64\lib\random.py", line 449, in sample
    raise ValueError("Sample larger than population or is negative") ValueError: Sample larger than population or is negative 怎么解决

  4. Assistant

    这个错误是由于在使用random.sample()方法时,样本数量大于了总体数量或者是负数。为了解决这个问题,你可以增加对样本数量的检查,确保它不超过总体数量,并且不是负数。

    以下是一个示例的修改代码:

    import json
    from datetime import datetime, timedelta
    
    # 创建主类和方法(省略其他代码)
    
    jfdaily_spider = Main()
    
    if __name__ == '__main__':
        spider = Main()
        
        # 获取从当前日期开始的前365天的新闻数据
        current_date = datetime.now()
        news_data = []
        
        for i in range(365):
            new_date = current_date - timedelta(days=i)
            formatted_date = new_date.strftime("%Y-%m-%d")
            
            # 获取每天的新闻数据
            daily_news = spider.getDailyNews(formatted_date)
            
            # 将每天的新闻数据添加到总的数据列表中
            news_data.extend(daily_news)
        
        # 将数据转换为JSON格式
        news_json = json.dumps(news_data)
        
        # 打印JSON数据
        print(news_json)
    

    请注意,这只是一个示例修改,你可能需要根据你的实际代码结构进行调整。另外,如果你的样本数量仍然大于总体数量或者是负数,请检查你的逻辑,确保样本数量的计算是正确的。