Skip to content
TheChatArchiveThe Chat Archive
gpt-4.1-mini-2025-04-14· OpenAIAi2 WildChat-4.8M (AI2)

我有以下程式碼用作獲取微信文章的內容,並轉存為指定格式的CSV輸出: import requests from bs4 import BeautifulSoup import…

  1. User

    我有以下程式碼用作獲取微信文章的內容,並轉存為指定格式的CSV輸出: import requests from bs4 import BeautifulSoup import base64 from urllib.parse import urljoin import csv import re import os

    def get_base64_image(url, headers): try: resp = requests.get(url, headers=headers, timeout=10) resp.raise_for_status() b64 = base64.b64encode(resp.content).decode() return b64 except Exception as e: print(f"圖片下載失敗 {url}: {e}") return None

    def parse_publish_time(timestr): if not timestr: return ""

    match = re.match(r'(\d{4})年(\d{1,2})月(\d{1,2})日\s+(\d{1,2}):(\d{1,2})', timestr)
    if not match:
        return ""
    
    year, month, day, hour, minute = match.groups()
    month = month.zfill(2)
    day = day.zfill(2)
    hour = hour.zfill(2)
    minute = minute.zfill(2)
    
    return f"{year}-{month}-{day} {hour}:{minute}:00"
    

    def get_article_info(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/90.0.4430.212 Safari/537.36', 'Referer': 'https://mp.weixin.qq.com/' }

    try:
        resp = requests.get(url, headers=headers, timeout=10)
        resp.encoding = 'utf-8'
        if resp.status_code != 200:
            print(f"無法取得頁面:狀態碼 {resp.status_code} URL:{url}")
            return None
    
        soup = BeautifulSoup(resp.text, 'html.parser')
    
        title_tag = soup.find('h1', id='activity-name')
        title = title_tag.get_text(strip=True) if title_tag else ""
        if not title:
            print(f"找不到標題,URL: {url}")
            return None
    
        title_base64 = base64.b64encode(title.encode('utf-8')).decode()
    
        publish_time_el = soup.find('em', id='publish_time')
        publish_time_str = publish_time_el.get_text(strip=True) if publish_time_el else ""
        held_date = parse_publish_time(publish_time_str)
    
        cover_base64 = ""
        cover_div = soup.find('div', id='js_row_immersive_cover_img')
        if cover_div:
            cover_img = cover_div.find('img')
            if cover_img and cover_img.get('src'):
                cover_url = cover_img['src']
                if cover_url.startswith('//'):
                    cover_url = 'https:' + cover_url
                elif cover_url.startswith('/'):
                    cover_url = urljoin(url, cover_url)
                b64 = get_base64_image(cover_url, headers)
                if b64:
                    ext = cover_url.split('.')[-1].lower()
                    if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']:
                        ext = 'jpeg'
                    cover_base64 = f"data:image/{ext};base64,{b64}"
    
        content_div = soup.find('div', id='js_content')
        if not content_div:
            print(f"找不到文章內容節點,URL: {url}")
            return None
    
        content_soup = BeautifulSoup(str(content_div), 'html.parser')
    
        imgs = content_soup.find_all('img')
        for img in imgs:
            src = img.get('data-src') or img.get('src')
            if not src:
                continue
            if src.startswith('//'):
                src = 'https:' + src
            elif src.startswith('/'):
                src = urljoin(url, src)
            b64 = get_base64_image(src, headers)
            if b64:
                ext = src.split('.')[-1].lower()
                if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']:
                    ext = 'jpeg'
                data_uri = f"data:image/{ext};base64,{b64}"
                img['src'] = data_uri
                if 'data-src' in img.attrs:
                    del img.attrs['data-src']
    
        full_html = f"""<html>
    

    {str(content_soup)} """

        description_b64 = base64.b64encode(full_html.encode('utf-8')).decode()
    
        category_id = 12
        author_id = 1
        slug = title
        status = 1
    
        return {
            'category_id': category_id,
            'author_id': author_id,
            'title': title,
            'title_base64': title_base64,
            'description': description_b64,
            'image': cover_base64,
            'slug': slug,
            'status': status,
            'held_date': held_date
        }
    except Exception as e:
        print(f"處理文章失敗 {url} 錯誤: {e}")
        return None
    

    def save_to_csv_append(data, filename='wechat_articles.csv', fieldnames=None): """ 將單筆資料追加寫入CSV (第一筆會寫入標題列) """ file_exists = os.path.isfile(filename) write_header = not file_exists if fieldnames is None: fieldnames = [ 'category_id', 'author_id', 'title', 'title_base64', 'description', 'image', 'slug', 'status', 'held_date' ] with open(filename, 'a', newline='', encoding='utf-8') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) if write_header: writer.writeheader() writer.writerow(data) csvfile.flush() # 強制寫入硬碟

    def process_folder_csv(folder_path, output_csv='wechat_articles.csv'): fieldnames = [ 'category_id', 'author_id', 'title', 'title_base64', 'description', 'image', 'slug', 'status', 'held_date' ] # 先刪除或清空 output_csv 避免舊資料殘留 if os.path.exists(output_csv): os.remove(output_csv)

    for filename in os.listdir(folder_path):
        if filename.lower().endswith('.csv'):
            csv_path = os.path.join(folder_path, filename)
            print(f"處理檔案: {csv_path}")
            try:
                with open(csv_path, 'r', encoding='utf-8') as f:
                    reader = csv.reader(f)
                    # 假設第一列是標題,跳過
                    headers = next(reader, None)
    
                    for row_num, row in enumerate(reader, start=2):
                        if len(row) < 2:
                            print(f"跳過行 {row_num},欄位小於2")
                            continue
                        url = row[1].strip()
                        if not url:
                            print(f"跳過空白URL,行{row_num}")
                            continue
                        print(f"讀取文章URL:{url}")
    
                        article_data = get_article_info(url)
                        if article_data:
                            save_to_csv_append(article_data, filename=output_csv, fieldnames=fieldnames)
                            print(f"成功寫入文章:{article_data['title']}")
                        else:
                            print(f"文章擷取失敗: {url}")
    
            except Exception as e:
                print(f"讀取CSV檔錯誤: {csv_path} 錯誤:{e}")
    

    if name == 'main': folder = './input_csvs' # 你放CSV檔的資料夾路徑,請依自己修改 output_filename = 'wechat_articles.csv' process_folder_csv(folder, output_csv=output_filename) print(f"全部處理完成,結果儲存於 {output_filename}")

    有關發佈時間的獲取,請改為在Input CSV中獲取,在第二欄可獲取URL,而第三欄為發佈時間,其格式為日/月/年 時:分:秒,而轉為 年-月-日 時:分:秒 請更改程式碼以達至效果

  2. Assistant

    你目前程式中是從文章頁面擷取發佈時間(publish_time),但你希望改成從輸入CSV檔中讀取,且CSV的第三欄為發佈時間,格式是「日/月/年 時:分:秒」,然後轉換成「年-月-日 時:分:秒」。

    以下是你需要主要修改的地方:

    1. 在 process_folder_csv 裡讀第三欄作為發佈時間
    2. 新增一個函式用來將日/月/年 時:分:秒格式轉成 年-月-日 時:分:秒格式
    3. get_article_info 函式改為接收 held_date 作為參數,不從頁面擷取時間

    修改後的重要程式碼片段

    1. 新增轉換日期格式的函式(例如 parse_input_date)

    def parse_input_date(timestr):
        """
        將日/月/年 時:分:秒格式轉換成 年-月-日 時:分:秒
        timestr範例: '24/06/2024 14:30:59'
        """
        if not timestr:
            return ""
    
        # 使用正規表達式來解析日期時間
        match = re.match(r'(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?', timestr)
        if not match:
            return ""
    
        day, month, year, hour, minute, second = match.groups()
        day = day.zfill(2)
        month = month.zfill(2)
        hour = hour.zfill(2)
        minute = minute.zfill(2)
        second = second if second else "00"
        second = second.zfill(2)
    
        return f"{year}-{month}-{day} {hour}:{minute}:{second}"
    

    2. 修改 get_article_info 增加 held_date 參數,改用傳入的日期

    def get_article_info(url, held_date=""):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                          'AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/90.0.4430.212 Safari/537.36',
            'Referer': 'https://mp.weixin.qq.com/'
        }
    
        try:
            resp = requests.get(url, headers=headers, timeout=10)
            resp.encoding = 'utf-8'
            if resp.status_code != 200:
                print(f"無法取得頁面:狀態碼 {resp.status_code} URL:{url}")
                return None
    
            soup = BeautifulSoup(resp.text, 'html.parser')
    
            title_tag = soup.find('h1', id='activity-name')
            title = title_tag.get_text(strip=True) if title_tag else ""
            if not title:
                print(f"找不到標題,URL: {url}")
                return None
    
            title_base64 = base64.b64encode(title.encode('utf-8')).decode()
    
            # 這裡改成用傳入的 held_date,不再從頁面找
            held_date_val = held_date
    
            cover_base64 = ""
            cover_div = soup.find('div', id='js_row_immersive_cover_img')
            if cover_div:
                cover_img = cover_div.find('img')
                if cover_img and cover_img.get('src'):
                    cover_url = cover_img['src']
                    if cover_url.startswith('//'):
                        cover_url = 'https:' + cover_url
                    elif cover_url.startswith('/'):
                        cover_url = urljoin(url, cover_url)
                    b64 = get_base64_image(cover_url, headers)
                    if b64:
                        ext = cover_url.split('.')[-1].lower()
                        if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']:
                            ext = 'jpeg'
                        cover_base64 = f"data:image/{ext};base64,{b64}"
    
            content_div = soup.find('div', id='js_content')
            if not content_div:
                print(f"找不到文章內容節點,URL: {url}")
                return None
    
            content_soup = BeautifulSoup(str(content_div), 'html.parser')
    
            imgs = content_soup.find_all('img')
            for img in imgs:
                src = img.get('data-src') or img.get('src')
                if not src:
                    continue
                if src.startswith('//'):
                    src = 'https:' + src
                elif src.startswith('/'):
                    src = urljoin(url, src)
                b64 = get_base64_image(src, headers)
                if b64:
                    ext = src.split('.')[-1].lower()
                    if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']:
                        ext = 'jpeg'
                    data_uri = f"data:image/{ext};base64,{b64}"
                    img['src'] = data_uri
                    if 'data-src' in img.attrs:
                        del img.attrs['data-src']
    
            full_html = f"""<html>
    {str(content_soup)}
    """
    
            description_b64 = base64.b64encode(full_html.encode('utf-8')).decode()
    
            category_id = 12
            author_id = 1
            slug = title
            status = 1
    
            return {
                'category_id': category_id,
                'author_id': author_id,
                'title': title,
                'title_base64': title_base64,
                'description': description_b64,
                'image': cover_base64,
                'slug': slug,
                'status': status,
                'held_date': held_date_val  # 使用傳入參數
            }
        except Exception as e:
            print(f"處理文章失敗 {url} 錯誤: {e}")
            return None
    

    3. 修改 process_folder_csv,在讀取每一行時,取得第三欄並轉換成目標格式,再傳給 get_article_info

    def process_folder_csv(folder_path, output_csv='wechat_articles.csv'):
        fieldnames = [
            'category_id',
            'author_id',
            'title',
            'title_base64',
            'description',
            'image',
            'slug',
            'status',
            'held_date'
        ]
        # 先刪除或清空 output_csv 避免舊資料殘留
        if os.path.exists(output_csv):
            os.remove(output_csv)
    
        for filename in os.listdir(folder_path):
            if filename.lower().endswith('.csv'):
                csv_path = os.path.join(folder_path, filename)
                print(f"處理檔案: {csv_path}")
                try:
                    with open(csv_path, 'r', encoding='utf-8') as f:
                        reader = csv.reader(f)
                        # 假設第一列是標題,跳過
                        headers = next(reader, None)
    
                        for row_num, row in enumerate(reader, start=2):
                            if len(row) < 3:
                                print(f"跳過行 {row_num},欄位小於3")
                                continue
                            url = row[1].strip()
                            raw_date = row[2].strip()
                            if not url:
                                print(f"跳過空白URL,行{row_num}")
                                continue
    
                            held_date = parse_input_date(raw_date)
                            print(f"讀取文章URL:{url},發佈時間:{held_date}")
    
                            article_data = get_article_info(url, held_date=held_date)
                            if article_data:
                                save_to_csv_append(article_data, filename=output_csv, fieldnames=fieldnames)
                                print(f"成功寫入文章:{article_data['title']}")
                            else:
                                print(f"文章擷取失敗: {url}")
    
                except Exception as e:
                    print(f"讀取CSV檔錯誤: {csv_path} 錯誤:{e}")
    

    總結

    • 用 parse_input_date 把從 CSV 第三欄讀到的日期字串轉成 「年-月-日 時:分:秒」格式。
    • 改 process_folder_csv 從第三欄抓取並轉換日期,再傳入 get_article_info。
    • get_article_info 改成傳入參數 held_date,不再從頁面元素讀取時間。

    這樣就能讓你的程式利用 CSV 的第三欄日期,不從頁面抓時間。
    也更確保時間格式轉換正確。


    有需要我可以幫你整合成完整程式碼版本。