Untitled conversation
- User
def playwright_follow_redirects(redirector_url): with sync_playwright() as p: # Consider trying with headless=False if True fails, for debugging browser = p.chromium.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3" ) page = context.new_page() redirected_url = None status_code = None try: logger.debug(f"Navigating to: {redirector_url}") response = page.goto(redirector_url, wait_until="networkidle", timeout=30000) # 30s timeout redirected_url = page.url status_code = response.status if response else None logger.info("Navigation response", extra={"final_url": redirected_url, "status_code": status_code}) if status_code and not (200 <= status_code < 400): # Check for client/server errors on final page raise Exception(f"Final page loaded with status {status_code}") except Exception as e: logger.critical("Unexpected error", exc_info=True) page.screenshot(path="playwright_error.png") redirected_url = None finally: browser.close() return redirected_url def fetch_feed(feed_url: str, before: datetime = None, after: datetime = None): """Fetches the feed from the given URL and returns the parsed feed.""" try: response = httpx.get(feed_url, follow_redirects=True, timeout=20) if response.status_code >= 400: logger.critical(f"Failed to fetch {feed_url}: {response.status_code}", extra={"response": response.text}) return None parsed = feedparser.parse(response.text) if before: parsed.entries = [ entry for entry in parsed.entries if parse_date(entry, "published") < before ] if after: parsed.entries = [ entry for entry in parsed.entries if parse_date(entry, "published") > after ] return parsed except httpx.RequestError as e: logger.error(f"Error fetching feed: {e}") return None @app.command() def download(feed_id: str, force: bool = False, refetch: bool = False): """ Checks in up to three places: 1. Emails that are actually in the inbox 2. The RSS feeds from which some set of forwarded emails come 3. Feeds saved in `feeds.json` in the directory for that user. refetch: If this is True then that means we need to look at each entry that has come into the main feed (i.e. the kill the newsletter feed) and fetch items from the feed attached that entry. Step by step: - For each entry in the parent feed, get the child entry's feed URL - From the unique list of child feed URLs, fetch their entries - Flatten the entries from all the feeds - Sort them by published date, filter them by `before` and `after` - Assign them to .entries on the parent """ href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403 then try again with Playwright to get around Cloudflare parsed_feed = feedparser.parse(resp.text) if parsed_feed.entries and not refetch: before = max([parse_date(p, "published") for p in parsed_feed.entries]) else: before = datetime.now().astimezone(tz=timezone.utc) after = before - timedelta(days=7) logger.info("Parsing feed", extra={"feed_id": feed_id, "before": before, "after": after}) run_dir = Path(f"output/{feed_id}/{before.strftime('%Y%m%d')}") os.makedirs(run_dir, exist_ok=True) substack_emails = [] for entry in parsed_feed.entries: if "substack" not in entry.content[0].value: logger.warning(f"Found a non-Substack email, removing it: '{entry.title}'", extra={"entry_id": getattr(entry, "id", None), "url": entry.link}) else: substack_emails.append(entry) if refetch: new_entries = [] for entry in substack_emails: child_feed_url = get_feed_url_from_email(entry.content[0].value, read_cache=True) if child_feed_url: child_feed = fetch_feed(child_feed_url, before=before, after=after) if child_feed: transformed_entries = [transform_fetched(e) for e in child_feed.entries] new_entries.extend(transformed_entries) logger.info("Fetched child feed", extra={"child_feed_url": child_feed_url}) else: logger.error(f"Failed to fetch child feed: {child_feed_url}") logger.info("Fetched child feeds", extra={"new_entries_count": len(new_entries)}) entries_to_save = new_entries else: entries_to_save = substack_emails feeds_json_path = run_dir.parent / "feeds.json" if feeds_json_path.exists(): logger.debug("Found feeds.json", extra={"feed_id": feed_id}) try: with open(feeds_json_path) as f: feeds_json = json.load(f) feed_urls = feeds_json['feeds'] for feed_url in feed_urls: feed = fetch_feed(feed_url, before=before, after=after) if feed: transformed_entries = [transform_fetched(e) for e in feed.entries] entries_to_save.extend(transformed_entries) logger.info("Fetched feed from feeds.json", extra={"feed_url": feed_url}) else: logger.error(f"Failed to fetch feed: {feed_url}") except json.JSONDecodeError: logger.error("Invalid JSON in feeds.json.", exc_info=True) for entry in entries_to_save: # TODO: ensure fwd: is actually removed # and truncate slug = str(entry.title).replace("fwd:", "").strip().replace(" ", "-").replace("/", "-").lower() published_date = parse_date(entry, "published") filename = f"{published_date.strftime('%Y%m%d_%H%M%S')}_{slug}.html" output_path = Path(f"{run_dir}/{filename}") if output_path.exists() and not force: logger.error( f"Output file already exists: {filename}" ) continue with open(output_path, "w", encoding="utf-8") as f: # NOTE: Not wrapping in <DOCTYPE html> and <html> because it may already # have it. # Instead will leave to the cleaning stage. f.write(f"{entry.content[0].value}") logger.info(f"Saved entry to {output_path}")Write the 403 / playwright bit
- Assistant
Below is the updated code snippet for the
downloadfunction, incorporating the handling for a 403 status code. This involves checking if the initial HTTP response fromhttpx.getis 403, and if so, using theplaywright_follow_redirectsfunction to attempt to bypass potential issues like Cloudflare. I've also modified theplaywright_follow_redirectsfunction to return both the final URL and the page content (e.g., the XML feed content), as you'll need the content to parse the feed withfeedparser.Key Changes:
-
In
playwright_follow_redirects:- Updated the function to return a tuple:
(final_url, content). If the navigation is successful (status 200), it retrieves the page content usingpage.content(). This assumes the final page is the XML feed. - Added error handling to log issues and return
Nonefor both values if something goes wrong.
- Updated the function to return a tuple:
-
In
download:- After the
httpx.getcall, check if the status code is 403. - If it is, call
playwright_follow_redirectsand use the returned content to parse the feed. - If not, proceed with the original parsing.
- Added logging for clarity and debugging.
- After the
Here's the updated code:
import httpx import feedparser from playwright.sync_api import sync_playwright import logging # Assuming you're using the logger from the original code logger = logging.getLogger(__name__) # Or whatever your logger setup is def playwright_follow_redirects(redirector_url): with sync_playwright() as p: browser = p.chromium.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" ) page = context.new_page() try: logger.debug(f"Navigating to: {redirector_url} with Playwright") response = page.goto(redirector_url, wait_until="networkidle", timeout=30000) # 30s timeout final_url = page.url status_code = response.status if response else None if status_code == 200: content = page.content() # Get the full content (e.g., XML feed) logger.info("Successfully fetched content with Playwright", extra={"final_url": final_url}) return final_url, content else: logger.error(f"Playwright navigation failed with status {status_code}") return final_url, None # Still return the final URL for reference, but no content except Exception as e: logger.error(f"Unexpected error in Playwright: {e}", exc_info=True) page.screenshot(path="playwright_error.png") # Save a screenshot for debugging return None, None # Return None for both on failure finally: browser.close() def download(feed_id: str, force: bool = False, refetch: bool = False): href = f"https://kill-the-newsletter.com/feeds/{feed_id}.xml" resp = httpx.get(href, follow_redirects=True, timeout=20) if resp.status_code == 403: logger.warning(f"Received 403 for {href}. Attempting to bypass with Playwright.") final_url, content = playwright_follow_redirects(href) if content: parsed_feed = feedparser.parse(content) # Parse the content from Playwright else: logger.error("Failed to retrieve content via Playwright. Aborting.") return # Or handle this error as needed (e.g., raise an exception) else: parsed_feed = feedparser.parse(resp.text) # Original parsing if not 403 # The rest of the function continues as before... if parsed_feed.entries and not refetch: before = max([parse_date(p, "published") for p in parsed_feed.entries]) else: before = datetime.now().astimezone(tz=timezone.utc) after = before - timedelta(days=7) logger.info("Parsing feed", extra={"feed_id": feed_id, "before": before, "after": after}) run_dir = Path(f"output/{feed_id}/{before.strftime('%Y%m%d')}") os.makedirs(run_dir, exist_ok=True) substack_emails = [] for entry in parsed_feed.entries: if "substack" not in entry.content[0].value: logger.warning(f"Found a non-Substack email, removing it: '{entry.title}'", extra={"entry_id": getattr(entry, "id", None), "url": entry.link}) else: substack_emails.append(entry) # ... (the rest of the function remains unchanged)Explanation:
-
Why modify
playwright_follow_redirects?- The original function only returned the final URL. To make it useful for fetching the feed, we need the actual content (e.g., the XML string). Using
page.content()retrieves the full HTML/XML of the page after navigation, which can then be passed tofeedparser.parse().
- The original function only returned the final URL. To make it useful for fetching the feed, we need the actual content (e.g., the XML string). Using
-
Error Handling:
- If Playwright fails (e.g., due to timeouts or other exceptions), it logs the error and returns
None, None, so thedownloadfunction can gracefully handle it. - In
download, if Playwright doesn't return content, we log an error and exit early to prevent further processing.
- If Playwright fails (e.g., due to timeouts or other exceptions), it logs the error and returns
-
Potential Improvements:
- Timeout and Retries: You might want to add retries for the
httpx.getcall or Playwright navigation if this is a common issue. - Content Validation: After parsing with
feedparser, you could add checks to ensure the parsed feed is valid (e.g., check ifparsed_feed.entriesis not empty). - Headless Mode: The code uses
headless=True, but if you're debugging, you can temporarily set it toFalseinplaywright_follow_redirects. - Performance: Playwright is resource-intensive, so use it only as a fallback.
- Timeout and Retries: You might want to add retries for the
This should integrate seamlessly into your existing code. If you encounter any issues or need further adjustments, let me know!
-
- User
No only modify the bit around the comment
- Assistant
It seems like you meant "No, only modify the bit around the comment." I'll focus solely on the section of code around the comment you referenced in the original code. Specifically, I'll provide the updated version of the code starting from the
httpx.getcall up to whereparsed_feedis defined.Here's the modified snippet. I've kept the changes minimal, incorporating the 403 handling logic directly around the comment, as I suggested in my previous response. This ensures that if a 403 status is encountered, it falls back to using
playwright_follow_redirectsto fetch the content.Updated Code Snippet:
resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403 then try again with Playwright to get around Cloudflare if resp.status_code == 403: logger.warning(f"Received 403 for {href}. Attempting to bypass with Playwright.") final_url, content = playwright_follow_redirects(href) # Assuming playwright_follow_redirects is defined as before if content: parsed_feed = feedparser.parse(content) # Parse the content from Playwright else: logger.error("Failed to retrieve content via Playwright. Aborting.") return # Exit early or handle as needed else: parsed_feed = feedparser.parse(resp.text) # Original parsing if not 403Explanation of Changes:
- Condition for 403: Immediately after the
httpx.getcall, we check if the status code is 403. If it is, we callplaywright_follow_redirectsto attempt retrieval. - Handling the Result: If
playwright_follow_redirectssucceeds and returns content, we parse it withfeedparser. Otherwise, we log an error and return early from the function. - No Other Changes: The rest of your
downloadfunction remains unchanged. This snippet slots directly into your original code.
If you need any further adjustments or clarifications, let me know!
- Condition for 403: Immediately after the
- User
The whole point is that Cloudflare is blocking the
http.get. So surely following up with that after the initial Playwright will also fail? Surely we need instead to fetch the feed using Playwright too? - Assistant
You're absolutely right to question this, and it's a valid concern. If Cloudflare is blocking the initial
httpx.getrequest (likely due to bot detection), it's possible that subsequent direct HTTP requests could also fail. However, the key advantage here is that Playwright simulates a real browser environment, which often bypasses Cloudflare's protections. Cloudflare typically allows requests from actual browsers because they can execute JavaScript, handle cookies, and mimic human behavior, whereashttpxis a simple HTTP client that might be flagged as a bot.In the code I suggested, we're not making another
httpx.getafter the initial one—we're switching to Playwright specifically to handle the 403 case. This means Playwright will navigate to the URL in a full browser context, potentially resolving any challenges like CAPTCHAs, JavaScript challenges, or redirects thathttpxcan't handle on its own.Why This Approach Makes Sense:
- Playwright's Strength: It launches a Chromium browser (or another engine), which can interact with the page dynamically. If Cloudflare presents a challenge (e.g., "I'm not a robot" page), Playwright could theoretically handle it if we extend the function to do so. In your current setup, we're assuming that just navigating with Playwright will get us to the final content.
- No Follow-Up HTTP Requests: Once we detect a 403, we're relying entirely on Playwright to fetch the content. The code doesn't attempt another plain HTTP request; it uses the browser to get the XML feed directly.
- Potential for Failure: If Playwright also fails (e.g., due to aggressive Cloudflare blocks or timeouts), it will log the error and abort, as per the code. You could enhance it further with retries or more advanced handling if needed.
Updated Code Suggestion:
Your current code snippet is already on the right track, but I'll refine it slightly for clarity and to emphasize that we're fully fetching the feed via Playwright in the 403 case. Here's the modified section around the comment:
resp = httpx.get(href, follow_redirects=True, timeout=20) # If the response is 403 then try again with Playwright to get around Cloudflare if resp.status_code == 403: logger.warning(f"Received 403 for {href}. Attempting to bypass with Playwright.") final_url, content = playwright_follow_redirects(href) # Playwright handles navigation and content retrieval if content and final_url: # Ensure both are successfully retrieved parsed_feed = feedparser.parse(content) # Parse the content fetched by Playwright else: logger.error("Playwright failed to retrieve the feed content. Aborting.") return # Exit early to prevent further processing else: parsed_feed = feedparser.parse(resp.text) # Original parsing if not 403Additional Recommendations:
- Test Thoroughly: Try running this against the problematic URL to confirm it works. If Playwright is still blocked, you might need to add more browser-like behavior, such as waiting for specific elements or handling interactions (e.g., if a CAPTCHA appears).
- Error Handling Enhancements: In
playwright_follow_redirects, you could add more robust checks, like verifying if the content is valid XML before returning it. - Performance Considerations: Playwright is heavier than
httpx, so use it only as a fallback to avoid unnecessary overhead in cases wherehttpxsucceeds. - If It Still Fails: If Cloudflare is particularly aggressive, you might explore other options like rotating proxies or user agents, but that's more advanced and could complicate your code.
This setup should address your concern by ensuring that the feed is fetched via Playwright when needed, rather than relying on another HTTP request. If you run into issues or need further tweaks, feel free to provide more details!
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.