Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· openaiHugging Face Suzhen/CodeChat

Why is this only adding in the first entry import sqlite3 import logging from…

  1. User

    Why is this only adding in the first entry

    import sqlite3 import logging from scrape_weather import WeatherScraper

    class DBOperations: """Class for performing operations on a SQLite database"""

    def __init__(self, dbname):
        """
        Constructor for DBOperations class.
    
        Parameters:
        - dbname: str, the name of the SQLite database file to use
        """
        self.dbname = dbname
        self.logger = logging.getLogger(__name__)
    
    def initialize_db(self):
        """
        Initialize the SQLite database by creating the weather_data table.
    
        This method should be called every time the program runs.
        """
        with self.get_cursor() as cursor:
            try:
                cursor.execute('''
                    CREATE TABLE IF NOT EXISTS weather_data (
                        id INTEGER PRIMARY KEY AUTOINCREMENT,
                        sample_date TEXT UNIQUE,
                        location TEXT,
                        min_temp REAL,
                        max_temp REAL,
                        avg_temp REAL
                    )
                ''')
                self.logger.info("Initialized database successfully.")
            except sqlite3.Error as e:
                self.logger.error(f"An error occurred while creating the table: {e}")
    
    def save_data(self, data):
        """
        Save weather data to the SQLite database.
    
        If the data already exists in the database, it will not be duplicated.
    
        Parameters:
        - data: dict, the weather data to save to the database. Must have keys for
          sample_date, location, min_temp, max_temp, and avg_temp.
        """
        with self.get_cursor() as cursor:
            try:
                cursor.execute('''
                    INSERT OR IGNORE INTO weather_data
                    (sample_date, location, min_temp, max_temp, avg_temp)
                    VALUES (?, ?, ?, ?, ?)
                ''', (data['sample_date'], data['location'], data['min_temp'], data['max_temp'], data['avg_temp']))
                self.logger.info("Data saved successfully.")
            except sqlite3.Error as e:
                self.logger.error(f"An error occurred while saving data to the database: {e}")
    
    def fetch_data(self, location):
        """
        Fetch weather data from the SQLite database for a specified location.
    
        Parameters:
        - location: str, the location to fetch weather data for
    
        Returns:
        - A list of tuples containing the weather data for the specified location,
          where each tuple has the format (sample_date, min_temp, max_temp, avg_temp).
          Returns an empty list if no data is found for the specified location.
        """
        with self.get_cursor() as cursor:
            try:
                cursor.execute('''
                    SELECT sample_date, min_temp, max_temp, avg_temp
                    FROM weather_data
                    WHERE location = ?
                ''', (location,))
                data = cursor.fetchall()
                self.logger.info("Data fetched successfully.")
                return data
            except sqlite3.Error as e:
                self.logger.error(f"An error occurred while fetching data from the database: {e}")
                return []
    
    def purge_data(self):
        """
        Purge all weather data from the SQLite database.
        """
        with self.get_cursor() as cursor:
            try:
                cursor.execute('DELETE FROM weather_data')
                self.logger.info("Data purged successfully.")
            except sqlite3.Error as e:
                self.logger.error(f"An error occurred while purging data from the database: {e}")
    
    def get_cursor(self):
        """
        Get a cursor to use for database operations.
    
        Returns:
        - A cursor object for the SQLite database.
        """
        return DBCM(self.dbname)
    

    class DBCM: def init(self, dbname): self.dbname = dbname self.logger = logging.getLogger(name)

    def __enter__(self):
        try:
            self.conn = sqlite3.connect(self.dbname)
            self.cursor = self.conn.cursor()
            self.logger.info("Connection to database established successfully.")
            return self.cursor
        except sqlite3.Error as e:
            self.logger.error(f"An error occurred while connecting to the database: {e}")
            return None
    
    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type is not None:
            self.conn.rollback()
        else:
            try:
                self.conn.commit()
                self.logger.info("Changes committed successfully.")
            except sqlite3.Error as e:
                self.logger.error(f"An error occurred while committing changes to the database: {e}")
        try:
            self.cursor.close()
            self.conn.close()
            self.logger.info("Connection to database closed successfully.")
        except sqlite3.Error as e:
            self.logger.error(f"An error occurred while closing the database connection: {e}")
    

    def main(): # Initialize the database db = DBOperations("mydatabase.db") db.initialize_db()

    # Get the weather data
    scraper = WeatherScraper()
    data = scraper.get_data()
    
    # Process the data and prepare the rows
    rows = []
    for date, temps in data.items():
        row = (
            date,
            "Winnipeg",
            temps["Max"],
            temps["Min"],
            temps["Mean"]
        )
        rows.append(row)
    
    # Save the data to the database
    with db.get_cursor() as cursor:
        try:
            cursor.executemany('''
                INSERT OR IGNORE INTO weather_data
                (sample_date, location, min_temp, max_temp, avg_temp)
                VALUES (?, ?, ?, ?, ?)
            ''', rows)
            db.logger.info(f"Inserted {len(rows)} rows into the database.")
        except sqlite3.Error as e:
            db.logger.error(f"An error occurred while inserting data: {e}")
    

    if name == 'main': main()

    from this class from html.parser import HTMLParser import urllib.request from datetime import datetime, timedelta import logging from dateutil.parser import parse

    class WeatherScraper(HTMLParser): """A parser for extracting temperature values from a website."""

    logger = logging.getLogger("main." + __name__)
    
    def __init__(self):
        try:
            super().__init__()
            self.is_tbody = False
            self.is_td = False
            self.is_tr = False
            self.last_page = False
            self.counter = 0
            self.daily_temps = {}
            self.weather = {}
            self.row_date = ""
        except Exception as e:
            self.logger.error("scrape:init:%s", e)
    
    def is_valid_date(self, date_str):
        """Check if a given string is a valid date."""
    
        try:
            parse(date_str, default=datetime(1900, 1, 1))
            return True
        except ValueError:
            return False
    
    def is_numeric(self, temp_str):
        """Check if given temperature string can be converted to a float."""
    
        try:
            float(temp_str)
            return True
        except ValueError:
            return False
    
    def handle_starttag(self, tag, attrs):
        """Handle the opening tags."""
    
        try:
            if tag == "tbody":
                self.is_tbody = True
            if tag == "tr" and self.is_tbody:
                self.is_tr = True
            if tag == "td" and self.is_tr:
                self.counter += 1
                self.is_td = True
            if tag == "abbr" and self.is_tr and self.is_valid_date(attrs[0][1]): # Only parses the valid dates, all other values are excluded.
                self.row_date = str(datetime.strptime(attrs[0][1], "%B %d, %Y").date())
                
            # if len(attrs) == 2:
            #     if attrs[1][1] == "previous disabled":
            #          self.last_page = True
        except Exception as e:
            self.logger.error("scrape:starttag:%s", e)
    
    def handle_endtag(self, tag):
        """Handle the closing tags."""
    
        try:
            if tag == "td":
                self.is_td = False
            if tag == "tr":
                self.counter = 0
                self.is_tr = False
        except Exception as e:
            self.logger.error("scrape:end:%s", e)
    
    def handle_data(self, data):
        """Handle the data inside the tags."""
        
        if data.startswith("Daily Data Report for January"):
                self.last_page = True
    
        try:
            if self.is_tbody and self.is_td and self.counter <= 3 and data.strip():
                if self.counter == 1 and self.is_numeric(data.strip()):
                    self.daily_temps["Max"] = float(data.strip())
                if self.counter == 2 and self.is_numeric(data.strip()):
                    self.daily_temps["Min"] = float(data.strip())
                if self.counter == 3 and self.is_numeric(data.strip()):
                    self.daily_temps["Mean"] = float(data.strip())
                    self.weather[self.row_date] = self.daily_temps
                    self.daily_temps = {}
        except Exception as e:
            self.logger.error("scrape:data:%s", e)
    
    def get_data(self):
        """Fetch the weather data and return it as a dictionary of dictionaries."""
    
        current_date = datetime.now()
    
        while not self.last_page:
            try:
                url = f"https://climate.weather.gc.ca/climate_data/daily_data_e.html?StationID=27174&timeframe=2&StartYear=1840&EndYear=2018&Day={current_date.day}&Year={current_date.year}&Month={current_date.month}"
    
                with urllib.request.urlopen(url) as response:
                    html = response.read().decode()
                self.feed(html)
    
                current_date -= timedelta(days=1) # Subtracts one day from the current date and assigns the resulting date back to the current_date variable.
            except Exception as e:
                self.logger.error("scrape:get_data:%s", e)
    
        return self.weather
    

    Test program.

    if name == "main": print_data = WeatherScraper().get_data() for k, v in print_data.items(): print(k, v)

  2. Assistant

    It seems that the problem may lie in the WeatherScraper class, as the DBOperations class, doesn't seem to have any issues with how it's adding data to the database.

    Upon further inspection, I believe I found a possible issue in your WeatherScraper class. In the handle_data method, the self.daily_temps dictionary is being reused, which might be causing issues when adding data to the self.weather dictionary.

    I recommend modifying the line where you reassign self.daily_temps to an empty dictionary, as follows:

    Replace this code:

    self.daily_temps = {}
    

    With this code:

    self.daily_temps = {"Max": None, "Min": None, "Mean": None}
    

    This change ensures that the self.daily_temps dictionary always has the keys "Max", "Min", and "Mean" with None as their initial value.

    If this doesn't solve the issue, please provide some more details about the problem you're facing, such as any errors or unexpected behavior you observe while running the code.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.