๐ Query: generative interpretation Solve CAPTCHA in the windowโฆ Traceback (most recentโฆ
- User
๐ Query: generative interpretation Solve CAPTCHA in the windowโฆ Traceback (most recent call last): File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 183, in asyncio.run(main()) File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/asyncio/runners.py", line 44, in run return loop.run_until_complete(main) File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete return future.result() File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 171, in main res = await s.search(args.query, args.max) File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 153, in search results = await self._parse_results(page, k) File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 110, in _parse_results await page.wait_for_selector("div.gs_r", timeout=15_000) File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/async_api/_generated.py", line 8180, in wait_for_selector await self._impl_obj.wait_for_selector( File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_page.py", line 425, in wait_for_selector return await self._main_frame.wait_for_selector(**locals_to_params(locals())) File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_frame.py", line 323, in wait_for_selector await self._channel.send("waitForSelector", locals_to_params(locals())) File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_connection.py", line 61, in send return await self._connection.wrap_api_call( File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_connection.py", line 528, in wrap_api_call raise rewrite_error(error, f"{parsed_st['apiName']}: {error}") from None playwright._impl._errors.TargetClosedError: Page.wait_for_selector: Target page, context or browser has been closed (gen) yarbel@UAs-MacBook-Pro bbgpt %
โโ OVERWRITE gschol2.py โโ
cat > gschol2.py << 'EOF' """ gschol2.py ยท Headless-by-default Google Scholar scraper with one-time CAPTCHA fallback (WebKit by default, Chromium/Firefox optional).
Key points โโโโโโโโโโ โข Runs headless. If Scholar blocks with the โprove youโre not a robotโ page (including Hebrew โืืืื ืฉืืื ื ืจืืืืโ), it automatically pops a visible WebKit window so you can tick the box. โข After you solve once, cookies are saved to --user-data-dir (default: ~/.scholar_state.json). Subsequent headless runs reuse them, so you shouldnโt see the CAPTCHA again. โข Clicks the dynamic โCiteโ link and scrapes MLA/Chicago rows to grab full author names (flagged with โ ).
Dependencies โ uv pip install playwright playwright-stealth rich Playwright โ playwright install chromium webkit firefox """
import asyncio, random, re, json, pathlib, os from typing import List, Dict, Any, Optional from rich.console import Console from playwright.async_api import async_playwright, Page from playwright_stealth import stealth_async
console = Console()
โญโโโโโโโโโโโโ helpers โโโโโโโโโโโโโฎ
async def snooze(a=0.5, b=2.0): # human-ish pauses await asyncio.sleep(random.uniform(a, b))
def is_captcha(html: str) -> bool: return ("gs_captcha_f" in html or "prove you are not a robot" in html.lower() or "ืืืื ืฉืืื ื ืจืืืื" in html)
def file_exists(path: Optional[str]) -> bool: return bool(path) and pathlib.Path(path).expanduser().exists()
โญโโโโโโโโโโโโ citation extractor โโโโโโโโโโโโโฎ
async def full_authors_from_modal(page: Page) -> Optional[str]: rows = await page.query_selector_all("tr.gs_citi_tr") for row in rows: lbl = await row.query_selector("th.gs_citi_lbl") if lbl and ("MLA" in await lbl.inner_text() or "Chicago" in await lbl.inner_text()): txt = await (await row.query_selector("div.gs_citr")).inner_text() m = re.match(r'^([^".(]+?)(?:.|")', txt.strip()) if m and re.search(r'[A-Z][a-z]{2,}\s+[A-Z][a-z]{2,}', m[1]): return m[1].strip() return None
โญโโโโโโโโโโโโ scraper class โโโโโโโโโโโโโฎ
class ScholarScraper: def init(self, engine: str = "webkit", state_file: str = "~/.scholar_state.json"): self.engine_name = engine self.state_path = str(pathlib.Path(state_file).expanduser())
async def _launch(self, p, headless=True): launcher = {"chromium": p.chromium, "firefox": p.firefox, "webkit": p.webkit}[self.engine_name] browser = await launcher.launch(headless=headless, args=["--disable-blink-features=AutomationControlled", "--disable-infobars", "--window-size=1920,1080"]) ctx_kwargs = { "viewport": {"width": 1920, "height": 1080}, # generic UA (Scholar doesnโt care which engine) "user_agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/120.0.0.0 Safari/537.36") } # load cookies if we have them if file_exists(self.state_path): ctx_kwargs["storage_state"] = self.state_path context = await browser.new_context(**ctx_kwargs) await context.add_init_script( """ Object.defineProperty(navigator,'webdriver',{get:()=>undefined}); Object.defineProperty(navigator,'plugins',{get:()=>[1,2,3]}); Object.defineProperty(navigator,'languages',{get:()=>['en-US','en']}); const gp=WebGLRenderingContext.prototype.getParameter; WebGLRenderingContext.prototype.getParameter=function(p){ if(p===37445) return 'Intel Inc.'; if(p===37446) return 'Intel Iris OpenGL Engine'; return gp.call(this,p); }; """) page = await context.new_page() await stealth_async(page) return browser, context, page # manual CAPTCHA path (always WebKit headful to minimise risk) async def _solve_captcha(self, query: str): async with async_playwright() as p: browser, ctx, page = await self._launch(p, headless=False) await page.goto("https://scholar.google.com", wait_until="domcontentloaded") console.print("[yellow]Solve CAPTCHA in the windowโฆ[/yellow]") await page.wait_for_selector("input#gs_hdr_tsi", timeout=120_000) box = await page.query_selector("input#gs_hdr_tsi") await box.fill(query); await box.press("Enter") await page.wait_for_selector("div.gs_r", timeout=20_000) # save cookies pathlib.Path(self.state_path).parent.mkdir(parents=True, exist_ok=True) await ctx.storage_state(path=self.state_path) # do not close contextโparse results directly return ctx, page async def _parse_results(self, page: Page, k: int) -> List[Dict[str, Any]]: await page.wait_for_selector("div.gs_r", timeout=15_000) entries = (await page.query_selector_all("div.gs_r"))[:k] out=[] for e in entries: t_el = await e.query_selector("h3.gs_rt") title = (await t_el.inner_text()).replace("[PDF]","").replace("[HTML]","").strip() url = await (await t_el.query_selector("a")).get_attribute("href") if t_el else "" meta = await (await e.query_selector("div.gs_a")).inner_text() authors_orig = meta.split(" - ")[0] year_match = re.search(r"\b(19|20)\d{2}\b", meta) year = year_match.group() if year_match else "N/A" cites = 0 cb = await e.query_selector("a:has-text('Cited by')") if cb: m = re.search(r"Cited by (\d+)", await cb.inner_text()) cites = int(m[1]) if m else 0 cite_btn = await e.query_selector("a.gs_or_cit") full=None if cite_btn: await snooze(0.8,2.0); await cite_btn.click() await page.wait_for_selector("div.gs_citi", timeout=8_000) full = await full_authors_from_modal(page) await page.keyboard.press("Escape") out.append({ "title": title, "url": url, "authors": full or authors_orig, "full_names": bool(full), "year": year, "citations": cites, }) return out async def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]: async with async_playwright() as p: # first try headless browser, ctx, page = await self._launch(p, headless=True) await page.goto(f"https://scholar.google.com/scholar?q={query}&hl=en&num={k}", wait_until="domcontentloaded") if is_captcha(await page.content()): await ctx.close(); await browser.close() # fallback headful solve ctx, page = await self._solve_captcha(query) results = await self._parse_results(page, k) await ctx.close() return results results = await self._parse_results(page, k) await ctx.close(); await browser.close() return resultsโญโโโโโโโโโโโโ CLI โโโโโโโโโโโโโฎ
async def main(): import argparse ap = argparse.ArgumentParser(description="Scholar scraper headless โ headful fallback") ap.add_argument("query"); ap.add_argument("--max", type=int, default=5) ap.add_argument("--browser", choices=["webkit","chromium","firefox"], default="webkit") ap.add_argument("--user-data-dir", default="~/.scholar_state.json", help="where to cache cookies (JSON)") args = ap.parse_args() console.print(f"๐ [green]Query:[/green] {args.query}") s = ScholarScraper(engine=args.browser, state_file=args.user_data_dir) res = await s.search(args.query, args.max) if not res: console.print("[red]No results.[/red]"); return for i,r in enumerate(res,1): tick="โ " if r["full_names"] else "" console.print(f"{i}. [magenta]{r['title']}[/magenta] {tick}") console.print(f" โข {r['authors']} ({r['year']}) โ cites: {r['citations']}\n") # save with open("scholar_results.json","w",encoding="utf-8") as f: json.dump(res,f,indent=2,ensure_ascii=False) console.print("[cyan]Saved to scholar_results.json[/cyan]")
if name == "main": asyncio.run(main()) EOF
โโ immediate test run โโ
python gschol2.py "generative interpretation" --max 3 --browser webkit --user-data-dir ~/.scholar_state.json
Find a better implementation, one that will slay the captcha nonsense, and will allow us to run this in headless while being able to click the cite button to get bibtext information. Write it in cat > file format.
Assistant
Looking at the error, it seems the CAPTCHA handling is causing issues when the page closes unexpectedly. Let me create a more robust implementation that better handles CAPTCHAs and citation extraction:
cat > gschol3.py << 'EOF' """ gschol3.py - Robust Google Scholar scraper with smart CAPTCHA handling and BibTeX extraction via citation modal. Key improvements: - Better CAPTCHA detection and handling - Persistent session management - BibTeX extraction from citation modal - Retry logic with exponential backoff - Better error handling for closed pages """ import asyncio import random import re import json import time from pathlib import Path from typing import List, Dict, Any, Optional, Tuple from dataclasses import dataclass, asdict from playwright.async_api import async_playwright, Page, BrowserContext, Browser from playwright_stealth import stealth_async from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn console = Console() @dataclass class ScholarResult: title: str url: str authors: str year: str citations: int bibtex: Optional[str] = None full_citation: Optional[str] = None class ScholarScraper: def __init__(self, browser_type: str = "chromium", state_file: str = "~/.scholar_state.json", headless: bool = True): self.browser_type = browser_type self.state_path = Path(state_file).expanduser() self.headless = headless self.browser: Optional[Browser] = None self.context: Optional[BrowserContext] = None async def __aenter__(self): await self.setup() return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.cleanup() async def setup(self): """Initialize browser with anti-detection measures""" self.playwright = await async_playwright().start() # Browser launch options launch_args = [ '--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--disable-web-security', '--disable-features=IsolateOrigins,site-per-process', '--disable-site-isolation-trials', '--no-sandbox', '--disable-setuid-sandbox', '--disable-accelerated-2d-canvas', '--disable-gpu', '--window-size=1920,1080', '--start-maximized', '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' ] browser_launcher = getattr(self.playwright, self.browser_type) self.browser = await browser_launcher.launch( headless=self.headless, args=launch_args ) # Context options context_options = { 'viewport': {'width': 1920, 'height': 1080}, 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'locale': 'en-US', 'timezone_id': 'America/New_York', } # Load saved state if exists if self.state_path.exists(): context_options['storage_state'] = str(self.state_path) self.context = await self.browser.new_context(**context_options) # Add stealth scripts await self.context.add_init_script(""" // Override webdriver detection Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); // Mock plugins Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); // Mock languages Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); // Mock permissions const originalQuery = window.navigator.permissions.query; window.navigator.permissions.query = (parameters) => ( parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters) ); // Mock WebGL const getParameter = WebGLRenderingContext.prototype.getParameter; WebGLRenderingContext.prototype.getParameter = function(parameter) { if (parameter === 37445) return 'Intel Inc.'; if (parameter === 37446) return 'Intel Iris OpenGL Engine'; return getParameter.call(this, parameter); }; """) async def cleanup(self): """Clean up browser resources""" if self.context: await self.context.close() if self.browser: await self.browser.close() if hasattr(self, 'playwright'): await self.playwright.stop() async def save_state(self): """Save browser state for future sessions""" if self.context: self.state_path.parent.mkdir(parents=True, exist_ok=True) await self.context.storage_state(path=str(self.state_path)) async def is_captcha_page(self, page: Page) -> bool: """Check if current page is a CAPTCHA challenge""" try: content = await page.content() captcha_indicators = [ 'gs_captcha_f', 'prove you are not a robot', 'ืืืื ืฉืืื ื ืจืืืื', 'recaptcha', 'captcha-form', '/sorry/index' ] return any(indicator in content.lower() for indicator in captcha_indicators) except: return False async def handle_captcha(self, page: Page) -> bool: """Handle CAPTCHA by switching to headed mode""" console.print("[yellow]CAPTCHA detected! Opening browser window...[/yellow]") # If already headless, we need to restart in headed mode if self.headless: await self.cleanup() self.headless = False await self.setup() # Navigate to the same URL new_page = await self.context.new_page() await stealth_async(new_page) await new_page.goto(page.url, wait_until='domcontentloaded') console.print("[yellow]Please solve the CAPTCHA in the browser window[/yellow]") # Wait for user to solve CAPTCHA try: await new_page.wait_for_selector('input#gs_hdr_tsi', timeout=120000) await self.save_state() console.print("[green]CAPTCHA solved! Continuing...[/green]") return True except: console.print("[red]CAPTCHA timeout or error[/red]") return False else: # Already in headed mode, just wait console.print("[yellow]Please solve the CAPTCHA[/yellow]") try: await page.wait_for_selector('input#gs_hdr_tsi', timeout=120000) await self.save_state() return True except: return False async def extract_bibtex(self, page: Page) -> Optional[str]: """Extract BibTeX from citation modal""" try: # Click BibTeX link bibtex_link = await page.wait_for_selector('a.gs_citi[href*="bibtex"]', timeout=5000) await bibtex_link.click() # Wait for new tab/window async with page.context.expect_page() as new_page_info: new_page = await new_page_info.value await new_page.wait_for_load_state('domcontentloaded') # Get BibTeX content bibtex_content = await new_page.content() # Extract just the BibTeX part if '<pre>' in bibtex_content: match = re.search(r'<pre>(.*?)</pre>', bibtex_content, re.DOTALL) if match: bibtex = match.group(1).strip() await new_page.close() return bibtex # Fallback: get all text bibtex = await new_page.inner_text('body') await new_page.close() return bibtex.strip() except Exception as e: console.print(f"[dim]BibTeX extraction failed: {e}[/dim]") return None async def extract_full_citation(self, page: Page) -> Optional[str]: """Extract full citation from modal""" try: # Look for MLA or APA citation rows = await page.query_selector_all('tr.gs_citi_tr') for row in rows: label = await row.query_selector('th.gs_citi_lbl') if label: label_text = await label.inner_text() if any(fmt in label_text for fmt in ['MLA', 'APA', 'Chicago']): citation_div = await row.query_selector('div.gs_citr') if citation_div: return await citation_div.inner_text() return None except: return None async def parse_result(self, result_div, page: Page) -> Optional[ScholarResult]: """Parse a single search result""" try: # Title and URL title_elem = await result_div.query_selector('h3.gs_rt a') if not title_elem: return None title = await title_elem.inner_text() title = re.sub(r'\[PDF\]|\[HTML\]', '', title).strip() url = await title_elem.get_attribute('href') or '' # Metadata meta_elem = await result_div.query_selector('div.gs_a') meta_text = await meta_elem.inner_text() if meta_elem else '' # Parse authors and year parts = meta_text.split(' - ') authors = parts[0] if parts else 'Unknown' year_match = re.search(r'\b(19|20)\d{2}\b', meta_text) year = year_match.group() if year_match else 'N/A' # Citations citations = 0 cite_link = await result_div.query_selector('a:has-text("Cited by")') if cite_link: cite_text = await cite_link.inner_text() cite_match = re.search(r'Cited by (\d+)', cite_text) citations = int(cite_match.group(1)) if cite_match else 0 # Try to get BibTeX and full citation bibtex = None full_citation = None cite_button = await result_div.query_selector('a.gs_or_cit') if cite_button: await asyncio.sleep(random.uniform(0.5, 1.5)) await cite_button.click() # Wait for modal await page.wait_for_selector('div#gs_cit', timeout=5000) # Extract BibTeX bibtex = await self.extract_bibtex(page) # Extract full citation full_citation = await self.extract_full_citation(page) # Close modal close_button = await page.query_selector('a#gs_cit-x') if close_button: await close_button.click() else: await page.keyboard.press('Escape') await asyncio.sleep(random.uniform(0.3, 0.8)) return ScholarResult( title=title, url=url, authors=authors, year=year, citations=citations, bibtex=bibtex, full_citation=full_citation ) except Exception as e: console.print(f"[dim]Error parsing result: {e}[/dim]") return None async def search(self, query: str, max_results: int = 10) -> List[ScholarResult]: """Perform a search with automatic CAPTCHA handling""" results = [] page = None try: page = await self.context.new_page() await stealth_async(page) # Build search URL search_url = f"https://scholar.google.com/scholar?q={query}&hl=en&num={max_results}" # Navigate with retry logic max_retries = 3 for attempt in range(max_retries): try: await page.goto(search_url, wait_until='domcontentloaded') # Check for CAPTCHA if await self.is_captcha_page(page): if not await self.handle_captcha(page): console.print("[red]Failed to handle CAPTCHA[/red]") return results # Re-navigate after CAPTCHA await page.goto(search_url, wait_until='domcontentloaded') # Wait for results await page.wait_for_selector('div.gs_r', timeout=10000) break except Exception as e: if attempt == max_retries - 1: raise console.print(f"[dim]Retry {attempt + 1}/{max_retries}: {e}[/dim]") await asyncio.sleep(2 ** attempt) # Parse results result_divs = await page.query_selector_all('div.gs_r') with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console ) as progress: task = progress.add_task(f"Parsing {len(result_divs)} results...", total=len(result_divs)) for i, div in enumerate(result_divs[:max_results]): result = await self.parse_result(div, page) if result: results.append(result) progress.update(task, advance=1) await self.save_state() except Exception as e: console.print(f"[red]Search error: {e}[/red]") finally: if page: await page.close() return results async def main(): import argparse parser = argparse.ArgumentParser(description='Google Scholar scraper with BibTeX support') parser.add_argument('query', help='Search query') parser.add_argument('--max', type=int, default=10, help='Maximum results') parser.add_argument('--browser', choices=['chromium', 'firefox', 'webkit'], default='chromium') parser.add_argument('--headless', action='store_true', help='Run in headless mode') parser.add_argument('--output', default='scholar_results.json', help='Output file') args = parser.parse_args() console.print(f"[green]๐ Searching for:[/green] {args.query}") async with ScholarScraper( browser_type=args.browser, headless=args.headless ) as scraper: results = await- User
continue from wheree you left off, I need the code in full
Assistant
cat > gschol3.py << 'EOF' """ gschol3.py - Robust Google Scholar scraper with smart CAPTCHA handling and BibTeX extraction via citation modal. Key improvements: - Better CAPTCHA detection and handling - Persistent session management - BibTeX extraction from citation modal - Retry logic with exponential backoff - Better error handling for closed pages """ import asyncio import random import re import json import time from pathlib import Path from typing import List, Dict, Any, Optional, Tuple from dataclasses import dataclass, asdict from playwright.async_api import async_playwright, Page, BrowserContext, Browser from playwright_stealth import stealth_async from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn console = Console() @dataclass class ScholarResult: title: str url: str authors: str year: str citations: int bibtex: Optional[str] = None full_citation: Optional[str] = None class ScholarScraper: def __init__(self, browser_type: str = "chromium", state_file: str = "~/.scholar_state.json", headless: bool = True): self.browser_type = browser_type self.state_path = Path(state_file).expanduser() self.headless = headless self.browser: Optional[Browser] = None self.context: Optional[BrowserContext] = None async def __aenter__(self): await self.setup() return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.cleanup() async def setup(self): """Initialize browser with anti-detection measures""" self.playwright = await async_playwright().start() # Browser launch options launch_args = [ '--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--disable-web-security', '--disable-features=IsolateOrigins,site-per-process', '--disable-site-isolation-trials', '--no-sandbox', '--disable-setuid-sandbox', '--disable-accelerated-2d-canvas', '--disable-gpu', '--window-size=1920,1080', '--start-maximized', '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' ] browser_launcher = getattr(self.playwright, self.browser_type) self.browser = await browser_launcher.launch( headless=self.headless, args=launch_args ) # Context options context_options = { 'viewport': {'width': 1920, 'height': 1080}, 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'locale': 'en-US', 'timezone_id': 'America/New_York', } # Load saved state if exists if self.state_path.exists(): context_options['storage_state'] = str(self.state_path) self.context = await self.browser.new_context(**context_options) # Add stealth scripts await self.context.add_init_script(""" // Override webdriver detection Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); // Mock plugins Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); // Mock languages Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); // Mock permissions const originalQuery = window.navigator.permissions.query; window.navigator.permissions.query = (parameters) => ( parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters) ); // Mock WebGL const getParameter = WebGLRenderingContext.prototype.getParameter; WebGLRenderingContext.prototype.getParameter = function(parameter) { if (parameter === 37445) return 'Intel Inc.'; if (parameter === 37446) return 'Intel Iris OpenGL Engine'; return getParameter.call(this, parameter); }; """) async def cleanup(self): """Clean up browser resources""" if self.context: await self.context.close() if self.browser: await self.browser.close() if hasattr(self, 'playwright'): await self.playwright.stop() async def save_state(self): """Save browser state for future sessions""" if self.context: self.state_path.parent.mkdir(parents=True, exist_ok=True) await self.context.storage_state(path=str(self.state_path)) async def is_captcha_page(self, page: Page) -> bool: """Check if current page is a CAPTCHA challenge""" try: content = await page.content() captcha_indicators = [ 'gs_captcha_f', 'prove you are not a robot', 'ืืืื ืฉืืื ื ืจืืืื', 'recaptcha', 'captcha-form', '/sorry/index' ] return any(indicator in content.lower() for indicator in captcha_indicators) except: return False async def handle_captcha(self, page: Page) -> bool: """Handle CAPTCHA by switching to headed mode""" console.print("[yellow]CAPTCHA detected! Opening browser window...[/yellow]") # If already headless, we need to restart in headed mode if self.headless: await self.cleanup() self.headless = False await self.setup() # Navigate to the same URL new_page = await self.context.new_page() await stealth_async(new_page) await new_page.goto(page.url, wait_until='domcontentloaded') console.print("[yellow]Please solve the CAPTCHA in the browser window[/yellow]") # Wait for user to solve CAPTCHA try: await new_page.wait_for_selector('input#gs_hdr_tsi', timeout=120000) await self.save_state() console.print("[green]CAPTCHA solved! Continuing...[/green]") return True except: console.print("[red]CAPTCHA timeout or error[/red]") return False else: # Already in headed mode, just wait console.print("[yellow]Please solve the CAPTCHA[/yellow]") try: await page.wait_for_selector('input#gs_hdr_tsi', timeout=120000) await self.save_state() return True except: return False async def extract_bibtex(self, page: Page) -> Optional[str]: """Extract BibTeX from citation modal""" try: # Click BibTeX link bibtex_link = await page.wait_for_selector('a.gs_citi[href*="bibtex"]', timeout=5000) await bibtex_link.click() # Wait for new tab/window async with page.context.expect_page() as new_page_info: new_page = await new_page_info.value await new_page.wait_for_load_state('domcontentloaded') # Get BibTeX content bibtex_content = await new_page.content() # Extract just the BibTeX part if '<pre>' in bibtex_content: match = re.search(r'<pre>(.*?)</pre>', bibtex_content, re.DOTALL) if match: bibtex = match.group(1).strip() await new_page.close() return bibtex # Fallback: get all text bibtex = await new_page.inner_text('body') await new_page.close() return bibtex.strip() except Exception as e: console.print(f"[dim]BibTeX extraction failed: {e}[/dim]") return None async def extract_full_citation(self, page: Page) -> Optional[str]: """Extract full citation from modal""" try: # Look for MLA or APA citation rows = await page.query_selector_all('tr.gs_citi_tr') for row in rows: label = await row.query_selector('th.gs_citi_lbl') if label: label_text = await label.inner_text() if any(fmt in label_text for fmt in ['MLA', 'APA', 'Chicago']): citation_div = await row.query_selector('div.gs_citr') if citation_div: return await citation_div.inner_text() return None except: return None async def parse_result(self, result_div, page: Page) -> Optional[ScholarResult]: """Parse a single search result""" try: # Title and URL title_elem = await result_div.query_selector('h3.gs_rt a') if not title_elem: return None title = await title_elem.inner_text() title = re.sub(r'\[PDF\]|\[HTML\]', '', title).strip() url = await title_elem.get_attribute('href') or '' # Metadata meta_elem = await result_div.query_selector('div.gs_a') meta_text = await meta_elem.inner_text() if meta_elem else '' # Parse authors and year parts = meta_text.split(' - ') authors = parts[0] if parts else 'Unknown' year_match = re.search(r'\b(19|20)\d{2}\b', meta_text) year = year_match.group() if year_match else 'N/A' # Citations citations = 0 cite_link = await result_div.query_selector('a:has-text("Cited by")') if cite_link: cite_text = await cite_link.inner_text() cite_match = re.search(r'Cited by (\d+)', cite_text) citations = int(cite_match.group(1)) if cite_match else 0 # Try to get BibTeX and full citation bibtex = None full_citation = None cite_button = await result_div.query_selector('a.gs_or_cit') if cite_button: await asyncio.sleep(random.uniform(0.5, 1.5)) await cite_button.click() # Wait for modal await page.wait_for_selector('div#gs_cit', timeout=5000) # Extract BibTeX bibtex = await self.extract_bibtex(page) # Extract full citation full_citation = await self.extract_full_citation(page) # Close modal close_button = await page.query_selector('a#gs_cit-x') if close_button: await close_button.click() else: await page.keyboard.press('Escape') await asyncio.sleep(random.uniform(0.3, 0.8)) return ScholarResult( title=title, url=url, authors=authors, year=year, citations=citations, bibtex=bibtex, full_citation=full_citation ) except Exception as e: console.print(f"[dim]Error parsing result: {e}[/dim]") return None async def search(self, query: str, max_results: int = 10) -> List[ScholarResult]: """Perform a search with automatic CAPTCHA handling""" results = [] page = None try: page = await self.context.new_page() await stealth_async(page) # Build search URL search_url = f"https://scholar.google.com/scholar?q={query}&hl=en&num={max_results}" # Navigate with retry logic max_retries = 3 for attempt in range(max_retries): try: await page.goto(search_url, wait_until='domcontentloaded') # Check for CAPTCHA if await self.is_captcha_page(page): if not await self.handle_captcha(page): console.print("[red]Failed to handle CAPTCHA[/red]") return results # Re-navigate after CAPTCHA await page.goto(search_url, wait_until='domcontentloaded') # Wait for results await page.wait_for_selector('div.gs_r', timeout=10000) break except Exception as e: if attempt == max_retries - 1: raise console.print(f"[dim]Retry {attempt + 1}/{max_retries}: {e}[/dim]") await asyncio.sleep(2 ** attempt) # Parse results result_divs = await page.query_selector_all('div.gs_r') with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console ) as progress: task = progress.add_task(f"Parsing {len(result_divs)} results...", total=len(result_divs)) for i, div in enumerate(result_divs[:max_results]): result = await self.parse_result(div, page) if result: results.append(result) progress.update(task, advance=1) await self.save_state() except Exception as e: console.print(f"[red]Search error: {e}[/red]") finally: if page: await page.close() return results async def main(): import argparse parser = argparse.ArgumentParser(description='Google Scholar scraper with BibTeX support') parser.add_argument('query', help='Search query') parser.add_argument('--max', type=int, default=10, help='Maximum results') parser.add_argument('--browser', choices=['chromium', 'firefox', 'webkit'], default='chromium') parser.add_argument('--headless', action='store_true', help='Run in headless mode') parser.add_argument('--output', default='scholar_results.json', help='Output file') args = parser.parse_args() console.print(f"[green]๐ Searching for:[/green] {args.query}") async with ScholarScraper( browser_type=args.browser, headless=args.headless ) as scraper: results = await scraper.search(args.query, args.max) if not results: console.print("[red]No results found[/red]") return
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.