import time import random import pygame import webbrowser import requests import re…
- User
import time import random import pygame import webbrowser import requests import re import json import os from datetime import datetime, timedelta from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from webdriver_manager.chrome import ChromeDriverManager import traceback
Configuration
Replace this with your filtered URL from Boligportal
#SEARCH_URL = "https://www.boligportal.dk/r%C3%A6kkehuse,lejligheder/k%C3%B8benhavn/2+-v%C3%A6relser/?max_monthly_rent=11000&min_size_m2=30" SEARCH_URL = "https://www.boligportal.dk/lejeboliger/k%C3%B8benhavn/alle-v%C3%A6relser/"
File to store persistent data
DATA_FILE = "boligportal_data.json" LOG_FILE = "boligportal_log.txt"
Initialize data storage
data = { "seen_ids": [], # Format: list of seen IDs "last_check": None, # Last time we checked }
Load existing data if available
if os.path.exists(DATA_FILE): try: with open(DATA_FILE, 'r') as f: data = json.load(f) except: print(f"Error loading data file, starting fresh")
Extract existing data
seen_ids = set(data["seen_ids"]) first_run = len(seen_ids) == 0 check_interval = 15 # seconds between checks - more frequent for quicker detection
Debug mode - set to True for more verbose logging
DEBUG = True
def log_debug(message): """Log debug messages if debug mode is enabled""" if DEBUG: print(f"[DEBUG] {message}") with open(LOG_FILE, 'a', encoding='utf-8') as f: f.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}\n")
def setup_driver(): """Set up the Chrome WebDriver with appropriate options""" try: options = Options() # Add options for better performance and reliability options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-gpu") options.add_argument("--window-size=1920,1080")
# Use a real user agent to avoid detection options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") # Uncomment below to run headless (no visible browser window) # options.add_argument("--headless") service = Service(ChromeDriverManager().install()) driver = webdriver.Chrome(service=service, options=options) return driver except Exception as e: print(f"Error setting up driver: {e}") send_telegram_message(f"❌ Failed to set up Chrome driver: {str(e)}") raisedef save_data(): """Save the current data to file""" try: # Update last check time data["last_check"] = datetime.now().isoformat() data["seen_ids"] = list(seen_ids) with open(DATA_FILE, 'w') as f: json.dump(data, f) except Exception as e: print(f"Error saving data: {e}")
def play_sound(filename="ding.mp3"): try: pygame.mixer.init() pygame.mixer.music.load(filename) pygame.mixer.music.play() while pygame.mixer.music.get_busy(): time.sleep(0.1) except Exception as e: print(f"Sound error: {e}")
def send_telegram_message(message): try: bot_token = '7675660843:AAGDJOnKz4CYQRcMMX1rfWOKdhjLrDc5fow' chat_id = 7933422517 url = f"https://api.telegram.org/bot{bot_token}/sendMessage" payload = { 'chat_id': chat_id, 'text': message, 'parse_mode': 'HTML' } response = requests.post(url, data=payload) if response.status_code != 200: print(f"Telegram error: {response.text}") except Exception as e: print(f"Telegram notification error: {e}")
def extract_listing_id(url): """Extract the listing ID from a Boligportal URL""" match = re.search(r'-id-(\d+)', url) if match: return match.group(1) return None
def is_likely_old_listing(text): """Check if the listing text indicates it's likely an old listing""" text = text.lower()
# Month patterns in Danish and English month_patterns = [ 'januar', 'februar', 'marts', 'april', 'maj', 'juni', 'juli', 'august', 'september', 'oktober', 'november', 'december', 'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december', 'jan', 'feb', 'mar', 'apr', 'maj', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' ] # Check for date patterns like "23. apr" or "23/4" date_patterns = [ r'\d{1,2}\.\s*(?:jan|feb|mar|apr|maj|jun|jul|aug|sep|okt|nov|dec)', # Danish format r'\d{1,2}/\d{1,2}', # Date format like 23/4 r'\d{1,2}-\d{1,2}', # Date format like 23-4 ] # If any month name is present, it's definitely an old listing if any(month in text for month in month_patterns): log_debug(f"Detected month name in: {text[:100]}...") return True # Check for date patterns for pattern in date_patterns: if re.search(pattern, text): log_debug(f"Detected date pattern '{pattern}' in: {text[:100]}...") return True # Patterns that indicate older listings old_patterns = [ 'i går', 'igår', 'yesterday', 'for 1 dag', 'for 2 dag', 'for 3 dag', 'for 1 d', 'for 2 d', 'for 3 d', 'week', 'uge', 'uger', 'måned', 'month', 'oprettet', 'created on' # These often indicate older listings ] # Check for hour values greater than 1 hour_match = re.search(r'(\d+)\s*(?:hour|time|timer|hr|t)\.?\s*(?:siden|ago)', text) if hour_match and int(hour_match.group(1)) > 1: log_debug(f"Detected {hour_match.group(0)} in: {text[:100]}...") return True for pattern in old_patterns: if pattern in text: log_debug(f"Detected old pattern '{pattern}' in: {text[:100]}...") return True return Falsedef get_listing_freshness(text): """Return a freshness score, flag for the listing, and age in minutes""" text = text.lower()
# Check for "netop nu" / "lige nu" first if any(phrase in text for phrase in ["netop nu", "lige nu", "just now"]): log_debug(f"Ultra fresh! Found 'netop nu'/'lige nu' in: {text[:100]}...") return 1, True, 0 # These are typically very fresh # Check for seconds (freshest) sec_match = re.search(r'(\d+)\s*(?:sec|sek|sekund)\.?\s*(?:siden|ago)', text) if sec_match: seconds = int(sec_match.group(1)) minutes_age = seconds / 60 if seconds < 300: # Less than 5 minutes log_debug(f"Found seconds indication: {sec_match.group(0)} in: {text[:100]}...") return 2, True, minutes_age return 3, False, minutes_age # Check for minutes min_match = re.search(r'(\d+)\s*(?:min|minute|minut)\.?\s*(?:siden|ago)', text) if min_match: minutes = int(min_match.group(1)) minutes_age = minutes if minutes <= 10: # 10 minutes or less is considered fresh log_debug(f"Found minutes indication: {min_match.group(0)} in: {text[:100]}...") return 4, minutes <= 3, minutes_age return 5, False, minutes_age # Check for hours hour_match = re.search(r'(\d+)\s*(?:hour|time|timer|hr|t)\.?\s*(?:siden|ago)', text) if hour_match: hours = int(hour_match.group(1)) minutes_age = hours * 60 log_debug(f"Found hours indication: {hour_match.group(0)} in: {text[:100]}...") return 6, False, minutes_age # Handle "for X min siden" format (Danish) min_ago_match = re.search(r'for\s+(\d+)\s+min(?:ut)?(?:ter)?\s+siden', text) if min_ago_match: minutes = int(min_ago_match.group(1)) minutes_age = minutes if minutes <= 10: log_debug(f"Found Danish minutes format: {min_ago_match.group(0)} in: {text[:100]}...") return 4, minutes <= 3, minutes_age return 5, False, minutes_age # If we can't determine the age but it passed the old-listing check, # assume it might be new-ish but give it a lower priority log_debug(f"Could not determine age from text: {text[:100]}...") return 8, False, 15 # Assume it's about 15 minutes old if we can't telldef extract_timestamp_info(text): """Extract timestamp information from the listing text""" text = text.lower()
# Check all time patterns in priority order time_patterns = [ # Just now patterns (r'netop\s+nu', 'Netop nu', 0), (r'lige\s+nu', 'Lige nu', 0), (r'just\s+now', 'Just now', 0), # Seconds patterns (r'(\d+)\s*(?:sec|sek|sekund)\.?\s*siden', '{} sekunder siden', 1), (r'(\d+)\s*(?:sec|sek|sekund)\.?\s*ago', '{} seconds ago', 1), # Minutes patterns (r'(\d+)\s*min\.?\s*siden', '{} min siden', 2), (r'(\d+)\s*min\.?\s*ago', '{} min ago', 2), (r'for\s+(\d+)\s+min', 'For {} min siden', 2), (r'(\d+)\s+min\w*\s+siden', '{} min siden', 2), # Hours patterns (r'(\d+)\s*(?:hour|time|timer|hr|t)\.?\s*siden', '{} timer siden', 3), (r'(\d+)\s*(?:hour|time|timer|hr|t)\.?\s*ago', '{} hours ago', 3), (r'for\s+(\d+)\s+time', 'For {} timer siden', 3), # Today patterns (r'i\s+dag', 'I dag', 4), (r'today', 'Today', 4), ] for pattern, format_str, priority in time_patterns: match = re.search(pattern, text) if match: if len(match.groups()) > 0: return format_str.format(match.group(1)), priority else: return format_str, priority return "Unknown time", 10def check_new_listings(driver): global first_run, seen_ids current_time = datetime.now() ultra_fresh_found = False
print(f"\nChecking for new listings at {current_time.strftime('%H:%M:%S')}...") try: # Load the page driver.get(SEARCH_URL) print("Page loaded, waiting for content...") # Wait for listings to load WebDriverWait(driver, 15).until( EC.presence_of_element_located((By.CSS_SELECTOR, "a[href*='-id-']")) ) print("Content loaded!") # Sometimes need to scroll to load all listings driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);") time.sleep(1) # Wait for possible lazy-loading driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(1) # Wait for possible lazy-loading # Get all listing links - try to get them in order from top to bottom listing_links = driver.find_elements(By.CSS_SELECTOR, "a[href*='-id-']") print(f"Found {len(listing_links)} listing links") # Process each listing current_ids = set() new_listings = [] # Sort listing links by vertical position (to process from top to bottom) try: # This helps with listing order, but might fail - we'll catch the exception listing_links = sorted(listing_links, key=lambda link: link.location['y']) except: pass for link in listing_links: try: url = link.get_attribute("href") if not url or "-id-" not in url: continue listing_id = extract_listing_id(url) if not listing_id: continue current_ids.add(listing_id) # Check if listing is new if listing_id not in seen_ids: # Try to get the parent container with all listing information # Going up 3-4 levels usually gets the full listing card parent = link for _ in range(5): # Try up to 5 levels up try: temp_parent = parent.find_element(By.XPATH, "./..") # Only update if we got a valid parent if temp_parent: parent = temp_parent except: break # Get the text to check if it's an old listing listing_text = parent.text log_debug(f"Checking listing ID {listing_id} with text: {listing_text}") # Skip obvious old listings if is_likely_old_listing(listing_text): print(f"Skipping likely old listing: {listing_id}") seen_ids.add(listing_id) continue # Check listing freshness freshness_score, is_ultra_fresh, minutes_age = get_listing_freshness(listing_text) # Extract timestamp info time_info, time_priority = extract_timestamp_info(listing_text) # For first run or if we can't determine age, be more permissive max_age_minutes = 20 if first_run else 10 # Skip listings older than our threshold if minutes_age > max_age_minutes and not first_run: print(f"Skipping listing older than {max_age_minutes} minutes: {listing_id} ({minutes_age:.1f} min)") seen_ids.add(listing_id) continue log_debug(f"ACCEPTED listing ID {listing_id} with freshness {freshness_score}, age {minutes_age} min") # Add to new listings new_listings.append({ 'id': listing_id, 'url': url, 'element': parent, 'text': listing_text, 'position': link.location['y'] if hasattr(link, 'location') else 0, 'freshness': freshness_score, 'ultra_fresh': is_ultra_fresh, 'minutes_age': minutes_age, 'time_info': time_info, 'time_priority': time_priority }) except Exception as e: print(f"Error processing listing link: {e}") log_debug(f"Exception while processing link: {traceback.format_exc()}") # Process new listings if first_run: # First run - just catalog without notifying print(f"First run: Cataloging {len(current_ids)} existing listings") seen_ids.update(current_ids) save_data() first_run = False send_telegram_message(f"✅ Initial scan complete. Tracking {len(seen_ids)} listings. Will notify about new ones <= 10 minutes old.") else: # Regular run - notify about new listings if new_listings: # Sort by freshness first, then position on page new_listings.sort(key=lambda x: (x['freshness'], x.get('position', 999999))) # Alert if ultra-fresh listings found ultra_fresh_count = sum(1 for listing in new_listings if listing['ultra_fresh']) if ultra_fresh_count > 0: ultra_fresh_alert = f"🔥 ALERT! {ultra_fresh_count} ultra-fresh listings just appeared!" print(ultra_fresh_alert) send_telegram_message(ultra_fresh_alert) play_sound() # Play sound immediately for ultra-fresh ultra_fresh_found = True print(f"Found {len(new_listings)} new listings <= 10 minutes old") # Limit the number of notifications to avoid spam (if many listings appear at once) max_notifications = 5 for i, listing in enumerate(new_listings): if i >= max_notifications: remaining = len(new_listings) - max_notifications print(f"Skipping {remaining} additional new listings to avoid spam") send_telegram_message(f"⚠️ Found {remaining} more new listings but skipping notifications to avoid spam.") break url = listing['url'] listing_id = listing['id'] parent = listing['element'] is_ultra_fresh = listing.get('ultra_fresh', False) minutes_age = listing.get('minutes_age', 0) time_info = listing.get('time_info', 'Unknown time') try: # Extract listing information title = "New apartment listing" # Default title price = "N/A" size = "N/A" location = "N/A" # Try to get title title_elements = parent.find_elements(By.CSS_SELECTOR, "h2, h3, .title, [class*='title']") if title_elements: title = title_elements[0].text.strip() # Get the full text all_text = listing['text'] # Look for key information with better patterns # Price patterns (Danish and English formats) price_patterns = [ r'(\d{1,3}(?:,\d{3})*(?:\.\d+)?)\s*kr(?:/md)?', # Danish format r'kr\.?\s*(\d{1,3}(?:,\d{3})*(?:\.\d+)?)', # Danish format reversed r'(\d{1,3}(?:,\d{3})*(?:\.\d+)?)\s*DKK', # DKK format ] for pattern in price_patterns: price_match = re.search(pattern, all_text, re.IGNORECASE) if price_match: price = price_match.group(0).strip() break # Size patterns size_patterns = [ r'(\d+(?:\.\d+)?)\s*m²', # Size in m² r'(\d+(?:\.\d+)?)\s*kvm', # Size in kvm (Danish) ] for pattern in size_patterns: size_match = re.search(pattern, all_text) if size_match: size = size_match.group(0).strip() break # Location - try to extract from text location_keywords = [ "copenhagen", "københavn", "frederiksberg", "nørrebro", "vesterbro", "østerbro", "nordhavn", "amager", "valby", "vanløse", "brønshøj", "nordvest" ] # Text elements for detailed info extraction text_elements = parent.find_elements(By.XPATH, ".//*") for element in text_elements: text = element.text.strip().lower() # Look for location if location == "N/A": for keyword in location_keywords: if keyword in text: location = element.text.strip() break # Backup price search if price == "N/A" and ("kr" in text or "dkk" in text): price = element.text.strip() # Backup size search if size == "N/A" and ("m²" in text or "kvm" in text): size = element.text.strip() # Notify about the new listing print(f"New listing found: {url} (ID: {listing_id})") print(f"Title: {title}") print(f"Posted: {time_info}") print(f"Price: {price}") print(f"Size: {size}") if location != "N/A": print(f"Location: {location}") # Notification prefix for ultra-fresh listings notification_prefix = "🔥 ULTRA-FRESH! 🔥\n" if is_ultra_fresh else "" # Add a debug note about freshness/age for troubleshooting debug_time = f"\n🔍 Freshness: {freshness_score}, Age: ~{minutes_age:.1f} min" if DEBUG else "" # Notify user notification = f"{notification_prefix}🏠 New listing!\n{title}" notification += f"\n⏰ {time_info}" notification += f"\n💰 {price}\n📏 {size}" if location != "N/A": notification += f"\n📍 {location}" notification += f"\n🔗 {url}" notification += debug_time if not ultra_fresh_found: # Only play sound here if we didn't play it earlier for ultra-fresh alert play_sound() send_telegram_message(notification) webbrowser.open(url) except Exception as e: # If detailed extraction fails, just notify about the URL print(f"Error extracting details: {e}") log_debug(f"Exception extracting details: {traceback.format_exc()}") notification_prefix = "🔥 ULTRA-FRESH! 🔥\n" if is_ultra_fresh else "" notification = f"{notification_prefix}🏠 New listing found!\nID: {listing_id}\n⏰ {time_info}\n{url}" if not ultra_fresh_found: play_sound() send_telegram_message(notification) webbrowser.open(url) # Add to seen listings seen_ids.add(listing_id) # Save all new IDs to database for listing in new_listings[max_notifications:]: seen_ids.add(listing['id']) save_data() else: print("No new listings found") # Occasionally clear out really old IDs to keep the database manageable if random.random() < 0.01: # 1% chance each run if len(seen_ids) > 1000: print("Pruning old listing IDs from database...") # Keep only the 500 most recent IDs # This assumes IDs are sequential and higher numbers are newer sorted_ids = sorted(seen_ids, key=lambda x: int(x) if x.isdigit() else 0, reverse=True) seen_ids = set(sorted_ids[:500]) data["seen_ids"] = list(seen_ids) save_data() print(f"Database pruned. Now tracking {len(seen_ids)} listings.") except Exception as e: print(f"Error checking listings: {e}") log_debug(f"Main exception: {traceback.format_exc()}") play_sound("ding-error.mp3") send_telegram_message(f"❌ Error checking listings: {str(e)}") # Try to recover from errors by cleaning up driver try: driver.delete_all_cookies() driver.refresh() except: passMain program
print("Starting Boligportal monitor with Selenium...")
Create or truncate the log file
with open(LOG_FILE, 'w', encoding='utf-8') as f: f.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting Boligportal monitor\n")
play_sound("ding.mp3") print("Sound test complete.") send_telegram_message("✅ Bot is working and monitoring Boligportal! Set to flag new listings within seconds!")
try: # Set up Chrome driver driver = setup_driver()
# Add a retry counter to handle possible site blocking consecutive_errors = 0 max_consecutive_errors = 5 # Check listings in a loop while True: try: check_new_listings(driver) consecutive_errors = 0 # Reset error counter on success # Add some randomness to avoid detection, but keep checks frequent sleep_time = random.randint(10, 20) # 10-20 seconds for more frequent checking print(f"Waiting {sleep_time} seconds until next check...") time.sleep(sleep_time) except Exception as e: consecutive_errors += 1 print(f"Error during run (#{consecutive_errors}): {e}") log_debug(f"Loop exception #{consecutive_errors}: {traceback.format_exc()}") if consecutive_errors >= max_consecutive_errors: print("Too many consecutive errors. Restarting driver...") try: driver.quit() except: pass driver = setup_driver() consecutive_errors = 0 send_telegram_message("⚠️ Bot encountered multiple errors and restarted the browser.") # Wait a bit longer after errors time.sleep(30)except KeyboardInterrupt: print("\nBot stopped by user.") send_telegram_message("❌ Bot has been stopped manually.") except Exception as e: print(f"Fatal error: {e}") log_debug(f"Fatal exception: {traceback.format_exc()}") send_telegram_message(f"⚠️ The bot crashed with error: {str(e)}") finally: # Clean up try: driver.quit() except: pass
what are the issues with this code?
Assistant
This Python script is designed to monitor Boligportal for new rental listings using Selenium, a browser automation tool. It attempts to identify new listings based on their ID and some text patterns indicating freshness, notifying the user via Telegram and opening the listing in a browser.
While the concept is sound for automating a search, there are several potential issues with this code, particularly concerning its reliability, maintainability, and potential for detection and blocking by the website.
Here's a breakdown of the issues:
-
Fragility due to Website Changes:
- Reliance on CSS Selectors and XPATH: The script heavily depends on specific CSS selectors (
a[href*='-id-']) and XPATH expressions (./..) to find listing elements and their parent containers. Websites frequently change their HTML structure (class names, element hierarchy, etc.). Any change to Boligportal's layout or element attributes could easily break these selectors, causing the script to fail to find listings or extract information. (See search results [9]) - Reliance on Text Patterns for Freshness and Information Extraction: The
is_likely_old_listing,get_listing_freshness, and the detail extraction logic (price_patterns,size_patterns,location_keywords) rely on specific text strings and regular expressions found within the listing text. This is highly fragile. Boligportal could change how they phrase timestamps ("just now", "X min ago"), the format of price/size, or the general wording in listings. This would lead to inaccurate freshness detection and failed information extraction. (See search result [13] mentions extracting data via API, which is more structured than scraping text).
- Reliance on CSS Selectors and XPATH: The script heavily depends on specific CSS selectors (
-
Detection and Blocking Risks:
- Standard Selenium Fingerprint: Basic Selenium usage is often detectable by anti-bot systems. Websites can identify automated browsers by checking JavaScript properties like
navigator.webdriver[1]. While a user-agent is set, the script doesn't employ more advanced techniques to mask its automation fingerprint. (See search results [1, 2, 3, 4, 5]). Running non-headless might appear more human but is impractical for long-term monitoring on a server. - Request Rate and Patterns: Checking every 10-20 seconds (
random.randint(10, 20)) is a very high frequency for a single IP address. This consistent and rapid pattern of requests is a common indicator of bot activity and is likely to trigger rate-limiting or IP blocking by Boligportal's anti-bot defenses over time. (See search results [4, 6, 12]). Websites track IP behavior[2]. - Simple Scrolling: The scrolling logic (
execute_script("window.scrollTo...")) is basic. More sophisticated anti-bot systems might look for more human-like scrolling behavior or other interactions. - Lack of Proxies: Using a single IP address for frequent scraping makes it easy to block. Implementing proxy rotation would significantly reduce the risk of IP bans. (See search results [4, 6, 12]).
- Standard Selenium Fingerprint: Basic Selenium usage is often detectable by anti-bot systems. Websites can identify automated browsers by checking JavaScript properties like
-
Efficiency and Performance:
- Scraping Full Page and All Links: The script loads the entire search results page and iterates through all listing links found. For a busy portal like Boligportal in Copenhagen, this could be a large number of listings. While it sorts by position, it still processes every link to check its ID and text. It might be more efficient to parse only the top portion of the page or use a different strategy if possible.
- Redundant Text Extraction: The script gets
parent.textonce, and then potentially iterates throughparent.find_elements(By.XPATH, ".//*")to find text again for detail extraction. This iteration through all nested elements is inefficient and prone to picking up incorrect text.
-
Data Management and Logic:
seen_idsPruning Assumption: The pruning logic assumes that higher listing IDs are always newer (sorted(seen_ids, key=lambda x: int(x) if x.isdigit() else 0, reverse=True)). While often true on many platforms, this is not guaranteed and could potentially prune legitimate recent listings if ID allocation isn't strictly sequential or chronological. (Search result [13] mentions Unix timestamps, suggesting time-based tracking might be possible if that data is accessible, though the scraping script doesn't use it).- Freshness Calculation Accuracy: The calculation of
minutes_agefrom text patterns like "X min siden" is approximate and depends entirely on the literal text provided by Boligportal. It's not a reliable measure of exact age. - "Likely Old Listing" Logic: The
is_likely_old_listingfunction's reliance on keywords and date patterns might incorrectly flag newly listed properties if those patterns appear in the description text (e.g., "available from May 1st", "last renovated in January"). This could cause genuinely new listings to be missed.
-
Error Handling:
- Broad
exceptblocks: Manytry...exceptblocks use a generalException as e. While catching errors is good, using specific exception types (likeNoSuchElementException,TimeoutException,StaleElementReferenceException) allows for more targeted error handling and debugging. - Stale Element References: As the script interacts with the page (like scrolling), elements previously found might become "stale." The current error handling might not specifically address
StaleElementReferenceException, which is common in Selenium when the DOM changes.
- Broad
-
Security and Configuration:
- Hardcoded Telegram Token/Chat ID: Storing the Telegram bot token and chat ID directly in the script is a security risk. These should be stored in environment variables or a separate configuration file that is not committed to version control.
-
Pygame Dependency:
- The script relies on
pygamefor sound notifications. This adds an external dependency that might not be necessary or desirable, especially if running the script on a server where a GUI environment with audio playback is not available. Alternative notification methods or using a simpler audio library might be more suitable.
- The script relies on
In summary, the code is a functional starting point but is vulnerable to website changes, highly susceptible to bot detection and blocking due to rapid request patterns and standard Selenium usage, and relies on fragile text-based parsing for crucial listing information. Improving robustness would involve more sophisticated anti-detection techniques, more resilient element selection, potentially using an API if available (though less likely for public listing data), and more robust data extraction methods.Here are the potential issues with the provided Python code for scraping Boligportal:
-
Fragility Due to Website Structure and Text Changes:
- Reliance on CSS Selectors and XPATH: The code uses hardcoded CSS selectors (
a[href*='-id-']) and XPATH (./..) to locate listings and their parent elements. Boligportal could change its website's HTML structure, class names, or element hierarchy at any time. This would likely break these selectors, preventing the script from finding or processing listings correctly. (See search results [9]). - Dependency on Text Patterns for Freshness and Data: The functions
is_likely_old_listing,get_listing_freshness, and the logic for extracting price, size, and location rely heavily on specific Danish/English text patterns and regular expressions found within the listing text (e.g., "netop nu", "for X min siden", "kr", "m²"). Boligportal or individual landlords could use different phrasing, date formats, or structures, leading to incorrect freshness detection, missed new listings, or failure to extract key details. (Search result [13] suggests Boligportal may have internal timestamps, but the script doesn't access this).
- Reliance on CSS Selectors and XPATH: The code uses hardcoded CSS selectors (
-
High Risk of Detection and Blocking:
- Standard Selenium Fingerprint: While setting a user agent is a good step, standard Selenium usage has detectable characteristics (like the
navigator.webdriverproperty) that anti-bot systems look for[1]. The script does not employ more advanced stealth techniques (e.g.,selenium-stealth,undetected_chromedriver) which makes it vulnerable to detection. (See search results [1, 2, 3, 4, 5]). - Aggressive Check Interval: A check interval of 10-20 seconds (
random.randint(10, 20)) is very frequent for a single IP address accessing the same page repeatedly. This rapid, patterned access is a strong indicator of bot activity and will likely lead to the IP address being rate-limited or blocked by Boligportal's anti-bot measures over time. (See search results [4, 6, 12]). - Basic Interaction: The scrolling logic is simple. Real user behavior is more varied.
- Lack of Proxies: Using a single IP address makes it easy for the website to identify and block the scraper. Implementing proxy rotation would make the scraper much more resilient to IP-based blocking. (See search results [4, 6, 12]).
- Standard Selenium Fingerprint: While setting a user agent is a good step, standard Selenium usage has detectable characteristics (like the
-
Efficiency and Performance:
- Processing All Listings: The script retrieves and attempts to process all listing links on the page in each check cycle. For a busy market, this could be a significant number, even if only a few are new. This can be inefficient compared to potentially stopping after processing the first few results at the top of the sorted list if they are identified as recent.
- Redundant Text Access: Extracting the full
parent.textand then later iterating through potentially many child elements (parent.find_elements(By.XPATH, ".//*")) to find specific pieces of text like price or size is inefficient and may retrieve text from unintended parts of the listing card.
-
Data Management Assumptions:
seen_idsPruning Logic: The pruning ofseen_idsassumes that higher numeric IDs correspond to newer listings. While this is often true, it's not guaranteed for all websites and could potentially remove recent listings if the ID assignment is not strictly sequential or chronological. A more robust approach would be to store a timestamp with each seen ID and prune based on age.
-
Robustness and Error Handling:
- Broad Exception Handling: While
try...except Exception as ecatches errors, it makes debugging harder as you don't know the specific type of error that occurred (e.g., network issue, element not found, stale element). Using more specific exception types is recommended. - Stale Element References: Selenium can encounter
StaleElementReferenceExceptionif the DOM changes after elements are found but before they are interacted with. The current error handling doesn't specifically address this common issue. - Sound File Dependency: The script relies on specific sound files (
ding.mp3,ding-error.mp3). If these files are missing or there are issues with thepygame.mixer, the sound notifications will fail.
- Broad Exception Handling: While
-
Security:
- Hardcoded Telegram Token and Chat ID: Storing sensitive information like the Telegram bot token and chat ID directly in the script's source code is a security vulnerability. These should be stored in environment variables or a secure configuration file outside of the codebase.
-
External Dependencies:
- The script has several external dependencies (
pygame,requests,selenium,webdriver_manager). While necessary for the functionality, managing these and ensuring they are installed correctly is required.pygamefor sound might be an unnecessary dependency if running in an environment without a GUI or audio output.
- The script has several external dependencies (
In summary, the code is a functional proof-of-concept but is highly susceptible to breaking due to website changes and is very likely to be detected and blocked by Boligportal's anti-bot defenses due to its aggressive and predictable behavior using standard Selenium.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.