Untitled conversation
- User
def playwright_follow_redirects(redirector_url): with sync_playwright() as p: # Consider trying with headless=False if True fails, for debugging browser = p.chromium.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3" ) page = context.new_page() redirected_url = None status_code = None try: logger.debug(f"Navigating to: {redirector_url}") response = page.goto(redirector_url, wait_until="networkidle", timeout=30000) # 30s timeout redirected_url = page.url status_code = response.status if response else None logger.info("Navigation response", extra={"final_url": redirected_url, "status_code": status_code}) if status_code and not (200 <= status_code < 400): # Check for client/server errors on final page raise Exception(f"Final page loaded with status {status_code}") except Exception as e: logger.critical("Unexpected error", exc_info=True) page.screenshot(path="playwright_error.png") redirected_url = None finally: browser.close() return redirected_url def fetch_feed(feed_url: str, before: datetime = None, after: datetime = None): """Fetches the feed from the given URL and returns the parsed feed.""" try: response = httpx.get(feed_url, follow_redirects=True, timeout=20) if response.status_code >= 400: logger.critical(f"Failed to fetch {feed_url}: {response.status_code}", extra={"response": response.text}) return None parsed = feedparser.parse(response.text) if before: parsed.entries = [ entry for entry in parsed.entries if parse_date(entry, "published") < before ] if after: parsed.entries = [ entry for entry in parsed.entries if parse_date(entry, "published") > after ] return parsed except httpx.RequestError as e: logger.error(f"Error fetching feed: {e}") return None @app.command() def download(feed_id: str, force: bool = False, refetch: bool = False): """ Checks in up to three places: 1. Emails that are actually in the inbox 2. The RSS feeds from which some set of forwarded emails come 3. Feeds saved in `feeds.json` in the directory for that user. refetch: If this is True then that means we need to look at each entry that has come into the main feed (i.e. the kill the newsletter feed) and fetch items from the feed attached that entry. Step by step: - For each entry in the parent feed, get the child entry's feed URL - From the unique list of child feed URLs, fetch their entries - Flatten the entries from all the feeds - Sort them by published date, filter them by `before` and `after` - Assign them to .entries on the parent """ href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403 then try again with Playwright to get around Cloudflare parsed_feed = feedparser.parse(resp.text) if parsed_feed.entries and not refetch: before = max([parse_date(p, "published") for p in parsed_feed.entries]) else: before = datetime.now().astimezone(tz=timezone.utc) after = before - timedelta(days=7) logger.info("Parsing feed", extra={"feed_id": feed_id, "before": before, "after": after}) run_dir = Path(f"output/{feed_id}/{before.strftime('%Y%m%d')}") os.makedirs(run_dir, exist_ok=True) substack_emails = [] for entry in parsed_feed.entries: if "substack" not in entry.content[0].value: logger.warning(f"Found a non-Substack email, removing it: '{entry.title}'", extra={"entry_id": getattr(entry, "id", None), "url": entry.link}) else: substack_emails.append(entry) if refetch: new_entries = [] for entry in substack_emails: child_feed_url = get_feed_url_from_email(entry.content[0].value, read_cache=True) if child_feed_url: child_feed = fetch_feed(child_feed_url, before=before, after=after) if child_feed: transformed_entries = [transform_fetched(e) for e in child_feed.entries] new_entries.extend(transformed_entries) logger.info("Fetched child feed", extra={"child_feed_url": child_feed_url}) else: logger.error(f"Failed to fetch child feed: {child_feed_url}") logger.info("Fetched child feeds", extra={"new_entries_count": len(new_entries)}) entries_to_save = new_entries else: entries_to_save = substack_emails feeds_json_path = run_dir.parent / "feeds.json" if feeds_json_path.exists(): logger.debug("Found feeds.json", extra={"feed_id": feed_id}) try: with open(feeds_json_path) as f: feeds_json = json.load(f) feed_urls = feeds_json['feeds'] for feed_url in feed_urls: feed = fetch_feed(feed_url, before=before, after=after) if feed: transformed_entries = [transform_fetched(e) for e in feed.entries] entries_to_save.extend(transformed_entries) logger.info("Fetched feed from feeds.json", extra={"feed_url": feed_url}) else: logger.error(f"Failed to fetch feed: {feed_url}") except json.JSONDecodeError: logger.error("Invalid JSON in feeds.json.", exc_info=True) for entry in entries_to_save: # TODO: ensure fwd: is actually removed # and truncate slug = str(entry.title).replace("fwd:", "").strip().replace(" ", "-").replace("/", "-").lower() published_date = parse_date(entry, "published") filename = f"{published_date.strftime('%Y%m%d_%H%M%S')}_{slug}.html" output_path = Path(f"{run_dir}/{filename}") if output_path.exists() and not force: logger.error( f"Output file already exists: {filename}" ) continue with open(output_path, "w", encoding="utf-8") as f: # NOTE: Not wrapping in <DOCTYPE html> and <html> because it may already # have it. # Instead will leave to the cleaning stage. f.write(f"{entry.content[0].value}") logger.info(f"Saved entry to {output_path}")Write the 403 / playwright bit
Assistant
To handle the 403 error and use Playwright to bypass Cloudflare, you can add a conditional check after the initial
httpx.getrequest. If the response status code is 403, you can call theplaywright_follow_redirectsfunction to fetch the content using Playwright. Here's how you can implement this:from datetime import datetime, timedelta, timezone from pathlib import Path import os import json import logging import httpx import feedparser from playwright.sync_api import sync_playwright from typing import Optional # Assuming logger is already configured logger = logging.getLogger(__name__) def playwright_follow_redirects(redirector_url: str) -> Optional[str]: """Follows redirects using Playwright and returns the final URL.""" with sync_playwright() as p: browser = p.chromium.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3" ) page = context.new_page() redirected_url = None try: logger.debug(f"Navigating to: {redirector_url}") response = page.goto(redirector_url, wait_until="networkidle", timeout=30000) redirected_url = page.url status_code = response.status if response else None logger.info("Navigation response", extra={"final_url": redirected_url, "status_code": status_code}) if status_code and not (200 <= status_code < 400): raise Exception(f"Final page loaded with status {status_code}") except Exception as e: logger.critical("Unexpected error", exc_info=True) page.screenshot(path="playwright_error.png") redirected_url = None finally: browser.close() return redirected_url def fetch_feed_with_fallback(feed_url: str, before: datetime = None, after: datetime = None) -> Optional[feedparser.FeedParserDict]: """Fetches the feed with fallback to Playwright if 403 is encountered.""" try: response = httpx.get(feed_url, follow_redirects=True, timeout=20) if response.status_code == 403: logger.warning(f"Received 403 for {feed_url}. Falling back to Playwright.") final_url = playwright_follow_redirects(feed_url) if final_url: response = httpx.get(final_url, follow_redirects=True, timeout=20) else: logger.error(f"Playwright failed to fetch {feed_url}.") return None elif response.status_code >= 400: logger.critical(f"Failed to fetch {feed_url}: {response.status_code}", extra={"response": response.text}) return None parsed = feedparser.parse(response.text) if before: parsed.entries = [entry for entry in parsed.entries if parse_date(entry, "published") < before] if after: parsed.entries = [entry for entry in parsed.entries if parse_date(entry, "published") > after] return parsed except httpx.RequestError as e: logger.error(f"Error fetching feed: {e}") return None # Assuming other functions (parse_date, get_feed_url_from_email, transform_fetched) are defined elsewhere @app.command() def download(feed_id: str, force: bool = False, refetch: bool = False): """Downloads and processes the feed.""" href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" parsed_feed = fetch_feed_with_fallback(href) if not parsed_feed: logger.error(f"Failed to fetch or parse feed: {href}") return # Rest of the download function remains unchanged... # (existing code for processing the feed, saving entries, etc.)Key Changes:
- Fallback Mechanism: Added
fetch_feed_with_fallbackto handle 403 errors by falling back to Playwright. - Playwright Integration: If
httpx.getreturns a 403,playwright_follow_redirectsis called to bypass Cloudflare. - Logging: Added logs for debugging and error tracking during the fallback process.
This approach ensures that if the initial HTTP request fails due to a 403 error, Playwright is used as a fallback to fetch the content.
- Fallback Mechanism: Added
- User
No only modify the bit around the comment
Assistant
Here’s the modified section of the code that specifically addresses the 403 error and uses Playwright as a fallback:
href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403 then try again with Playwright to get around Cloudflare if resp.status_code == 403: logger.warning(f"Received 403 for {href}. Falling back to Playwright.") final_url = playwright_follow_redirects(href) if final_url: resp = httpx.get(final_url, follow_redirects=True, timeout=20) else: logger.critical(f"Playwright failed to fetch {href}. Aborting.") return # Exit the function if Playwright fails elif resp.status_code >= 400: logger.critical(f"Failed to fetch {href}: {resp.status_code}", extra={"response": resp.text}) return # Exit the function if HTTP request fails with non-403 error parsed_feed = feedparser.parse(resp.text)Explanation of Changes:
- 403 Handling: Added a conditional check for a 403 status code. If encountered, the code logs a warning and calls
playwright_follow_redirectsto bypass Cloudflare. - Playwright Fallback: If Playwright successfully retrieves a
final_url, the code reattempts the HTTP request using that URL. - Error Handling: If Playwright fails to retrieve a valid URL, the function logs a critical error and exits early.
- Non-403 Errors: If the initial HTTP request fails with any status code >= 400 (except 403), the function logs the error and exits.
This ensures the 403 error is specifically handled with a Playwright fallback while maintaining the existing flow for other errors.
- 403 Handling: Added a conditional check for a 403 status code. If encountered, the code logs a warning and calls
- User
The whole point is that Cloudflare is blocking the
http.get. So surely following up with that after the initial Playwright will also fail? Surely we need instead to fetch the feed using Playwright too? Assistant
You are absolutely correct! If Cloudflare is blocking the initial
httpx.getrequest, simply following up with anotherhttpx.getafter Playwright retrieves the final URL won't solve the problem. Instead, Playwright should be used to fetch the feed content directly when a 403 is encountered.Here’s the corrected implementation:
href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403, use Playwright to fetch the feed content directly if resp.status_code == 403: logger.warning(f"Received 403 for {href}. Falling back to Playwright.") try: with sync_playwright() as p: browser = p.chromium.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3" ) page = context.new_page() page.goto(href, wait_until="networkidle", timeout=30000) feed_content = page.content() # Fetch the page content directly resp = httpx.Response(status_code=200, content=feed_content) # Wrap in httpx.Response browser.close() except Exception as e: logger.critical(f"Playwright failed to fetch {href}: {e}", exc_info=True) return # Exit if Playwright fails elif resp.status_code >= 400: logger.critical(f"Failed to fetch {href}: {resp.status_code}", extra={"response": resp.text}) return # Exit if HTTP request fails with non-403 error parsed_feed = feedparser.parse(resp.text)Key Changes:
- Direct Content Fetching: When a 403 is encountered, Playwright is used to fetch the feed content directly using
page.content(). - Wrap Content in
httpx.Response: The content fetched by Playwright is wrapped in anhttpx.Responseobject to maintain consistency with the original flow. - Error Handling: If Playwright fails, the function logs a critical error and exits.
This ensures that if Cloudflare blocks the initial
httpx.getrequest, Playwright is used to fetch the feed content directly, bypassing the block.- Direct Content Fetching: When a 403 is encountered, Playwright is used to fetch the feed content directly using
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.