Why is this only adding in the first entry import sqlite3 import logging from…
- User
Why is this only adding in the first entry
import sqlite3 import logging from scrape_weather import WeatherScraper
class DBOperations: """Class for performing operations on a SQLite database"""
def __init__(self, dbname): """ Constructor for DBOperations class. Parameters: - dbname: str, the name of the SQLite database file to use """ self.dbname = dbname self.logger = logging.getLogger(__name__) def initialize_db(self): """ Initialize the SQLite database by creating the weather_data table. This method should be called every time the program runs. """ with self.get_cursor() as cursor: try: cursor.execute(''' CREATE TABLE IF NOT EXISTS weather_data ( id INTEGER PRIMARY KEY AUTOINCREMENT, sample_date TEXT UNIQUE, location TEXT, min_temp REAL, max_temp REAL, avg_temp REAL ) ''') self.logger.info("Initialized database successfully.") except sqlite3.Error as e: self.logger.error(f"An error occurred while creating the table: {e}") def save_data(self, data): """ Save weather data to the SQLite database. If the data already exists in the database, it will not be duplicated. Parameters: - data: dict, the weather data to save to the database. Must have keys for sample_date, location, min_temp, max_temp, and avg_temp. """ with self.get_cursor() as cursor: try: cursor.execute(''' INSERT OR IGNORE INTO weather_data (sample_date, location, min_temp, max_temp, avg_temp) VALUES (?, ?, ?, ?, ?) ''', (data['sample_date'], data['location'], data['min_temp'], data['max_temp'], data['avg_temp'])) self.logger.info("Data saved successfully.") except sqlite3.Error as e: self.logger.error(f"An error occurred while saving data to the database: {e}") def fetch_data(self, location): """ Fetch weather data from the SQLite database for a specified location. Parameters: - location: str, the location to fetch weather data for Returns: - A list of tuples containing the weather data for the specified location, where each tuple has the format (sample_date, min_temp, max_temp, avg_temp). Returns an empty list if no data is found for the specified location. """ with self.get_cursor() as cursor: try: cursor.execute(''' SELECT sample_date, min_temp, max_temp, avg_temp FROM weather_data WHERE location = ? ''', (location,)) data = cursor.fetchall() self.logger.info("Data fetched successfully.") return data except sqlite3.Error as e: self.logger.error(f"An error occurred while fetching data from the database: {e}") return [] def purge_data(self): """ Purge all weather data from the SQLite database. """ with self.get_cursor() as cursor: try: cursor.execute('DELETE FROM weather_data') self.logger.info("Data purged successfully.") except sqlite3.Error as e: self.logger.error(f"An error occurred while purging data from the database: {e}") def get_cursor(self): """ Get a cursor to use for database operations. Returns: - A cursor object for the SQLite database. """ return DBCM(self.dbname)class DBCM: def init(self, dbname): self.dbname = dbname self.logger = logging.getLogger(name)
def __enter__(self): try: self.conn = sqlite3.connect(self.dbname) self.cursor = self.conn.cursor() self.logger.info("Connection to database established successfully.") return self.cursor except sqlite3.Error as e: self.logger.error(f"An error occurred while connecting to the database: {e}") return None def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: self.conn.rollback() else: try: self.conn.commit() self.logger.info("Changes committed successfully.") except sqlite3.Error as e: self.logger.error(f"An error occurred while committing changes to the database: {e}") try: self.cursor.close() self.conn.close() self.logger.info("Connection to database closed successfully.") except sqlite3.Error as e: self.logger.error(f"An error occurred while closing the database connection: {e}")def main(): # Initialize the database db = DBOperations("mydatabase.db") db.initialize_db()
# Get the weather data scraper = WeatherScraper() data = scraper.get_data() # Process the data and prepare the rows rows = [] for date, temps in data.items(): row = ( date, "Winnipeg", temps["Max"], temps["Min"], temps["Mean"] ) rows.append(row) # Save the data to the database with db.get_cursor() as cursor: try: cursor.executemany(''' INSERT OR IGNORE INTO weather_data (sample_date, location, min_temp, max_temp, avg_temp) VALUES (?, ?, ?, ?, ?) ''', rows) db.logger.info(f"Inserted {len(rows)} rows into the database.") except sqlite3.Error as e: db.logger.error(f"An error occurred while inserting data: {e}")if name == 'main': main()
from this class from html.parser import HTMLParser import urllib.request from datetime import datetime, timedelta import logging from dateutil.parser import parse
class WeatherScraper(HTMLParser): """A parser for extracting temperature values from a website."""
logger = logging.getLogger("main." + __name__) def __init__(self): try: super().__init__() self.is_tbody = False self.is_td = False self.is_tr = False self.last_page = False self.counter = 0 self.daily_temps = {} self.weather = {} self.row_date = "" except Exception as e: self.logger.error("scrape:init:%s", e) def is_valid_date(self, date_str): """Check if a given string is a valid date.""" try: parse(date_str, default=datetime(1900, 1, 1)) return True except ValueError: return False def is_numeric(self, temp_str): """Check if given temperature string can be converted to a float.""" try: float(temp_str) return True except ValueError: return False def handle_starttag(self, tag, attrs): """Handle the opening tags.""" try: if tag == "tbody": self.is_tbody = True if tag == "tr" and self.is_tbody: self.is_tr = True if tag == "td" and self.is_tr: self.counter += 1 self.is_td = True if tag == "abbr" and self.is_tr and self.is_valid_date(attrs[0][1]): # Only parses the valid dates, all other values are excluded. self.row_date = str(datetime.strptime(attrs[0][1], "%B %d, %Y").date()) # if len(attrs) == 2: # if attrs[1][1] == "previous disabled": # self.last_page = True except Exception as e: self.logger.error("scrape:starttag:%s", e) def handle_endtag(self, tag): """Handle the closing tags.""" try: if tag == "td": self.is_td = False if tag == "tr": self.counter = 0 self.is_tr = False except Exception as e: self.logger.error("scrape:end:%s", e) def handle_data(self, data): """Handle the data inside the tags.""" if data.startswith("Daily Data Report for January"): self.last_page = True try: if self.is_tbody and self.is_td and self.counter <= 3 and data.strip(): if self.counter == 1 and self.is_numeric(data.strip()): self.daily_temps["Max"] = float(data.strip()) if self.counter == 2 and self.is_numeric(data.strip()): self.daily_temps["Min"] = float(data.strip()) if self.counter == 3 and self.is_numeric(data.strip()): self.daily_temps["Mean"] = float(data.strip()) self.weather[self.row_date] = self.daily_temps self.daily_temps = {} except Exception as e: self.logger.error("scrape:data:%s", e) def get_data(self): """Fetch the weather data and return it as a dictionary of dictionaries.""" current_date = datetime.now() while not self.last_page: try: url = f"https://climate.weather.gc.ca/climate_data/daily_data_e.html?StationID=27174&timeframe=2&StartYear=1840&EndYear=2018&Day={current_date.day}&Year={current_date.year}&Month={current_date.month}" with urllib.request.urlopen(url) as response: html = response.read().decode() self.feed(html) current_date -= timedelta(days=1) # Subtracts one day from the current date and assigns the resulting date back to the current_date variable. except Exception as e: self.logger.error("scrape:get_data:%s", e) return self.weatherTest program.
if name == "main": print_data = WeatherScraper().get_data() for k, v in print_data.items(): print(k, v)
- Assistant
It seems that the problem may lie in the
WeatherScraperclass, as theDBOperationsclass, doesn't seem to have any issues with how it's adding data to the database.Upon further inspection, I believe I found a possible issue in your
WeatherScraperclass. In thehandle_datamethod, theself.daily_tempsdictionary is being reused, which might be causing issues when adding data to theself.weatherdictionary.I recommend modifying the line where you reassign
self.daily_tempsto an empty dictionary, as follows:Replace this code:
self.daily_temps = {}With this code:
self.daily_temps = {"Max": None, "Min": None, "Mean": None}This change ensures that the
self.daily_tempsdictionary always has the keys "Max", "Min", and "Mean" with None as their initial value.If this doesn't solve the issue, please provide some more details about the problem you're facing, such as any errors or unexpected behavior you observe while running the code.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.