我有以下程式碼用作獲取微信文章的內容,並轉存為指定格式的CSV輸出: import requests from bs4 import BeautifulSoup import…
- User
我有以下程式碼用作獲取微信文章的內容,並轉存為指定格式的CSV輸出: import requests from bs4 import BeautifulSoup import base64 from urllib.parse import urljoin import csv import re import os
def get_base64_image(url, headers): try: resp = requests.get(url, headers=headers, timeout=10) resp.raise_for_status() b64 = base64.b64encode(resp.content).decode() return b64 except Exception as e: print(f"圖片下載失敗 {url}: {e}") return None
def parse_publish_time(timestr): if not timestr: return ""
match = re.match(r'(\d{4})年(\d{1,2})月(\d{1,2})日\s+(\d{1,2}):(\d{1,2})', timestr) if not match: return "" year, month, day, hour, minute = match.groups() month = month.zfill(2) day = day.zfill(2) hour = hour.zfill(2) minute = minute.zfill(2) return f"{year}-{month}-{day} {hour}:{minute}:00"def get_article_info(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/90.0.4430.212 Safari/537.36', 'Referer': 'https://mp.weixin.qq.com/' }
try: resp = requests.get(url, headers=headers, timeout=10) resp.encoding = 'utf-8' if resp.status_code != 200: print(f"無法取得頁面:狀態碼 {resp.status_code} URL:{url}") return None soup = BeautifulSoup(resp.text, 'html.parser') title_tag = soup.find('h1', id='activity-name') title = title_tag.get_text(strip=True) if title_tag else "" if not title: print(f"找不到標題,URL: {url}") return None title_base64 = base64.b64encode(title.encode('utf-8')).decode() publish_time_el = soup.find('em', id='publish_time') publish_time_str = publish_time_el.get_text(strip=True) if publish_time_el else "" held_date = parse_publish_time(publish_time_str) cover_base64 = "" cover_div = soup.find('div', id='js_row_immersive_cover_img') if cover_div: cover_img = cover_div.find('img') if cover_img and cover_img.get('src'): cover_url = cover_img['src'] if cover_url.startswith('//'): cover_url = 'https:' + cover_url elif cover_url.startswith('/'): cover_url = urljoin(url, cover_url) b64 = get_base64_image(cover_url, headers) if b64: ext = cover_url.split('.')[-1].lower() if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']: ext = 'jpeg' cover_base64 = f"data:image/{ext};base64,{b64}" content_div = soup.find('div', id='js_content') if not content_div: print(f"找不到文章內容節點,URL: {url}") return None content_soup = BeautifulSoup(str(content_div), 'html.parser') imgs = content_soup.find_all('img') for img in imgs: src = img.get('data-src') or img.get('src') if not src: continue if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = urljoin(url, src) b64 = get_base64_image(src, headers) if b64: ext = src.split('.')[-1].lower() if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']: ext = 'jpeg' data_uri = f"data:image/{ext};base64,{b64}" img['src'] = data_uri if 'data-src' in img.attrs: del img.attrs['data-src'] full_html = f"""<html>{str(content_soup)} """
description_b64 = base64.b64encode(full_html.encode('utf-8')).decode() category_id = 12 author_id = 1 slug = title status = 1 return { 'category_id': category_id, 'author_id': author_id, 'title': title, 'title_base64': title_base64, 'description': description_b64, 'image': cover_base64, 'slug': slug, 'status': status, 'held_date': held_date } except Exception as e: print(f"處理文章失敗 {url} 錯誤: {e}") return Nonedef save_to_csv_append(data, filename='wechat_articles.csv', fieldnames=None): """ 將單筆資料追加寫入CSV (第一筆會寫入標題列) """ file_exists = os.path.isfile(filename) write_header = not file_exists if fieldnames is None: fieldnames = [ 'category_id', 'author_id', 'title', 'title_base64', 'description', 'image', 'slug', 'status', 'held_date' ] with open(filename, 'a', newline='', encoding='utf-8') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) if write_header: writer.writeheader() writer.writerow(data) csvfile.flush() # 強制寫入硬碟
def process_folder_csv(folder_path, output_csv='wechat_articles.csv'): fieldnames = [ 'category_id', 'author_id', 'title', 'title_base64', 'description', 'image', 'slug', 'status', 'held_date' ] # 先刪除或清空 output_csv 避免舊資料殘留 if os.path.exists(output_csv): os.remove(output_csv)
for filename in os.listdir(folder_path): if filename.lower().endswith('.csv'): csv_path = os.path.join(folder_path, filename) print(f"處理檔案: {csv_path}") try: with open(csv_path, 'r', encoding='utf-8') as f: reader = csv.reader(f) # 假設第一列是標題,跳過 headers = next(reader, None) for row_num, row in enumerate(reader, start=2): if len(row) < 2: print(f"跳過行 {row_num},欄位小於2") continue url = row[1].strip() if not url: print(f"跳過空白URL,行{row_num}") continue print(f"讀取文章URL:{url}") article_data = get_article_info(url) if article_data: save_to_csv_append(article_data, filename=output_csv, fieldnames=fieldnames) print(f"成功寫入文章:{article_data['title']}") else: print(f"文章擷取失敗: {url}") except Exception as e: print(f"讀取CSV檔錯誤: {csv_path} 錯誤:{e}")if name == 'main': folder = './input_csvs' # 你放CSV檔的資料夾路徑,請依自己修改 output_filename = 'wechat_articles.csv' process_folder_csv(folder, output_csv=output_filename) print(f"全部處理完成,結果儲存於 {output_filename}")
有關發佈時間的獲取,請改為在Input CSV中獲取,在第二欄可獲取URL,而第三欄為發佈時間,其格式為日/月/年 時:分:秒,而轉為 年-月-日 時:分:秒 請更改程式碼以達至效果
- Assistant
你目前程式中是從文章頁面擷取發佈時間(
publish_time),但你希望改成從輸入CSV檔中讀取,且CSV的第三欄為發佈時間,格式是「日/月/年 時:分:秒」,然後轉換成「年-月-日 時:分:秒」。以下是你需要主要修改的地方:
- 在
process_folder_csv裡讀第三欄作為發佈時間 - 新增一個函式用來將日/月/年 時:分:秒格式轉成 年-月-日 時:分:秒格式
get_article_info函式改為接收held_date作為參數,不從頁面擷取時間
修改後的重要程式碼片段
1. 新增轉換日期格式的函式(例如
parse_input_date)def parse_input_date(timestr): """ 將日/月/年 時:分:秒格式轉換成 年-月-日 時:分:秒 timestr範例: '24/06/2024 14:30:59' """ if not timestr: return "" # 使用正規表達式來解析日期時間 match = re.match(r'(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?', timestr) if not match: return "" day, month, year, hour, minute, second = match.groups() day = day.zfill(2) month = month.zfill(2) hour = hour.zfill(2) minute = minute.zfill(2) second = second if second else "00" second = second.zfill(2) return f"{year}-{month}-{day} {hour}:{minute}:{second}"
2. 修改
get_article_info增加held_date參數,改用傳入的日期def get_article_info(url, held_date=""): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/90.0.4430.212 Safari/537.36', 'Referer': 'https://mp.weixin.qq.com/' } try: resp = requests.get(url, headers=headers, timeout=10) resp.encoding = 'utf-8' if resp.status_code != 200: print(f"無法取得頁面:狀態碼 {resp.status_code} URL:{url}") return None soup = BeautifulSoup(resp.text, 'html.parser') title_tag = soup.find('h1', id='activity-name') title = title_tag.get_text(strip=True) if title_tag else "" if not title: print(f"找不到標題,URL: {url}") return None title_base64 = base64.b64encode(title.encode('utf-8')).decode() # 這裡改成用傳入的 held_date,不再從頁面找 held_date_val = held_date cover_base64 = "" cover_div = soup.find('div', id='js_row_immersive_cover_img') if cover_div: cover_img = cover_div.find('img') if cover_img and cover_img.get('src'): cover_url = cover_img['src'] if cover_url.startswith('//'): cover_url = 'https:' + cover_url elif cover_url.startswith('/'): cover_url = urljoin(url, cover_url) b64 = get_base64_image(cover_url, headers) if b64: ext = cover_url.split('.')[-1].lower() if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']: ext = 'jpeg' cover_base64 = f"data:image/{ext};base64,{b64}" content_div = soup.find('div', id='js_content') if not content_div: print(f"找不到文章內容節點,URL: {url}") return None content_soup = BeautifulSoup(str(content_div), 'html.parser') imgs = content_soup.find_all('img') for img in imgs: src = img.get('data-src') or img.get('src') if not src: continue if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = urljoin(url, src) b64 = get_base64_image(src, headers) if b64: ext = src.split('.')[-1].lower() if ext not in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']: ext = 'jpeg' data_uri = f"data:image/{ext};base64,{b64}" img['src'] = data_uri if 'data-src' in img.attrs: del img.attrs['data-src'] full_html = f"""<html> {str(content_soup)} """ description_b64 = base64.b64encode(full_html.encode('utf-8')).decode() category_id = 12 author_id = 1 slug = title status = 1 return { 'category_id': category_id, 'author_id': author_id, 'title': title, 'title_base64': title_base64, 'description': description_b64, 'image': cover_base64, 'slug': slug, 'status': status, 'held_date': held_date_val # 使用傳入參數 } except Exception as e: print(f"處理文章失敗 {url} 錯誤: {e}") return None
3. 修改
process_folder_csv,在讀取每一行時,取得第三欄並轉換成目標格式,再傳給get_article_infodef process_folder_csv(folder_path, output_csv='wechat_articles.csv'): fieldnames = [ 'category_id', 'author_id', 'title', 'title_base64', 'description', 'image', 'slug', 'status', 'held_date' ] # 先刪除或清空 output_csv 避免舊資料殘留 if os.path.exists(output_csv): os.remove(output_csv) for filename in os.listdir(folder_path): if filename.lower().endswith('.csv'): csv_path = os.path.join(folder_path, filename) print(f"處理檔案: {csv_path}") try: with open(csv_path, 'r', encoding='utf-8') as f: reader = csv.reader(f) # 假設第一列是標題,跳過 headers = next(reader, None) for row_num, row in enumerate(reader, start=2): if len(row) < 3: print(f"跳過行 {row_num},欄位小於3") continue url = row[1].strip() raw_date = row[2].strip() if not url: print(f"跳過空白URL,行{row_num}") continue held_date = parse_input_date(raw_date) print(f"讀取文章URL:{url},發佈時間:{held_date}") article_data = get_article_info(url, held_date=held_date) if article_data: save_to_csv_append(article_data, filename=output_csv, fieldnames=fieldnames) print(f"成功寫入文章:{article_data['title']}") else: print(f"文章擷取失敗: {url}") except Exception as e: print(f"讀取CSV檔錯誤: {csv_path} 錯誤:{e}")
總結
- 用
parse_input_date把從 CSV 第三欄讀到的日期字串轉成 「年-月-日 時:分:秒」格式。 - 改
process_folder_csv從第三欄抓取並轉換日期,再傳入get_article_info。 get_article_info改成傳入參數held_date,不再從頁面元素讀取時間。
這樣就能讓你的程式利用 CSV 的第三欄日期,不從頁面抓時間。
也更確保時間格式轉換正確。
有需要我可以幫你整合成完整程式碼版本。
- 在