Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

#Cell 2 import nltk import torch import requests import feedparser import sqlite3 import…

  1. User

    #Cell 2

    import nltk import torch import requests import feedparser import sqlite3 import pandas as pd import plotly.graph_objects as go import numpy as np from plotly.subplots import make_subplots from datetime import datetime, timedelta from nltk.sentiment import SentimentIntensityAnalyzer from transformers import ( AutoTokenizer, AutoModelForSequenceClassification, pipeline ) from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import logging from typing import List, Tuple, Set from wordcloud import WordCloud import seaborn as sns from collections import Counter import matplotlib.pyplot as plt from nltk.tokenize import word_tokenize from nltk.corpus import stopwords import warnings from difflib import SequenceMatcher

    warnings.filterwarnings('ignore')

    Configure logging

    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

    Constants

    DATABASE_NAME = 'news_sentiment.db' NEWS_FEED_URL = "https://www.thehindu.com/feeder/default.rss" DATE_FORMAT = "%Y-%m-%d" DATETIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"

    class DatabaseManager: def init(self, db_name: str): self.db_name = db_name self._setup_database() self.connection = None self._setup_connection()

    def _setup_database(self):
        with sqlite3.connect(self.db_name) as conn:
            conn.execute('''
                CREATE TABLE IF NOT EXISTS sentiment_scores (
                    date TEXT,
                    time TEXT,
                    title TEXT,
                    summary TEXT,
                    score REAL
                )
            ''')
            # Add indexes for better query performance
            conn.execute('CREATE INDEX IF NOT EXISTS idx_date ON sentiment_scores(date)')
            conn.execute('CREATE INDEX IF NOT EXISTS idx_title ON sentiment_scores(title)')
            conn.execute('CREATE INDEX IF NOT EXISTS idx_score ON sentiment_scores(score)')
    
    def _setup_connection(self):
        """Setup persistent connection with optimized parameters"""
        if not self.connection:
            self.connection = sqlite3.connect(self.db_name, check_same_thread=False)
            self.connection.execute('PRAGMA journal_mode=WAL')  # Write-Ahead Logging
            self.connection.execute('PRAGMA synchronous=NORMAL')
            self.connection.execute('PRAGMA cache_size=-2000')  # 2MB cache
            self.connection.execute('PRAGMA temp_store=MEMORY')
    
    def store_score(self, date: str, time: str, title: str, summary: str, score: float):
        try:
            self.connection.execute(
                "INSERT INTO sentiment_scores VALUES (?, ?, ?, ?, ?)",
                (date, time, title, summary, score)
            )
            self.connection.commit()
        except Exception as e:
            logging.error(f"Error storing score: {e}")
    
    @lru_cache(maxsize=128)
    def get_daily_average(self, date: str) -> float:
        try:
            result = self.connection.execute(
                "SELECT AVG(score) FROM sentiment_scores WHERE date = ?",
                (date,)
            ).fetchone()
            return result[0] if result[0] is not None else 0.0
        except Exception as e:
            logging.error(f"Error getting daily average: {e}")
            return 0.0
    
    def get_headlines_with_scores(self, limit: int = 10, min_summary_length: int = 20) -> pd.DataFrame:
        """Retrieve recent headlines with their sentiment scores, excluding entries with very short summaries."""
        try:
            query = """
                SELECT date, time, title, score
                FROM sentiment_scores
                WHERE LENGTH(summary) >= ?
                ORDER BY date DESC, time DESC
                LIMIT ?
            """
            return pd.read_sql_query(query, self.connection, params=(min_summary_length, limit))
        except Exception as e:
            logging.error(f"Error retrieving headlines: {e}")
            return pd.DataFrame()
    
    def get_extreme_sentiment_headlines(self) -> Tuple[pd.DataFrame, pd.DataFrame]:
        """Get headlines with highest and lowest sentiment scores"""
        try:
            most_positive = pd.read_sql_query("""
                SELECT date, time, title, score
                FROM sentiment_scores
                ORDER BY score DESC
                LIMIT 5
            """, self.connection)
    
            most_negative = pd.read_sql_query("""
                SELECT date, time, title, score
                FROM sentiment_scores
                ORDER BY score ASC
                LIMIT 5
            """, self.connection)
    
            return most_positive, most_negative
        except Exception as e:
            logging.error(f"Error retrieving extreme headlines: {e}")
            return pd.DataFrame(), pd.DataFrame()
    
    def get_headline_stats(self) -> dict:
        """Get statistical information about headlines"""
        try:
            stats = {}
    
            # Get average sentiment score
            avg_query = "SELECT AVG(score) FROM sentiment_scores"
            stats['average_sentiment'] = self.connection.execute(avg_query).fetchone()[0]
    
            # Get total number of headlines
            count_query = "SELECT COUNT(*) FROM sentiment_scores"
            stats['total_headlines'] = self.connection.execute(count_query).fetchone()[0]
    
            # Get date range
            range_query = """
                SELECT
                    MIN(date) as earliest_date,
                    MAX(date) as latest_date
                FROM sentiment_scores
            """
            earliest, latest = self.connection.execute(range_query).fetchone()
            stats['date_range'] = f"{earliest} to {latest}"
    
            return stats
        except Exception as e:
            logging.error(f"Error getting headline stats: {e}")
            return {}
    
    def close(self):
        if self.connection:
            self.connection.close()
    

    class SentimentAnalyzer: def init(self): self._initialize_models() self._setup_device()

    def _initialize_models(self):
        # Download NLTK data only if not already present
        for resource in ['vader_lexicon', 'punkt', 'stopwords']:
            try:
                nltk.data.find(f'tokenizers/{resource}')
            except LookupError:
                nltk.download(resource, quiet=True)
    
        self.sia = SentimentIntensityAnalyzer()
    
        # Initialize models with better memory management
        self.finbert_tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert")
        self.finbert_model = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert")
        self.roberta_tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment")
        self.roberta_model = AutoModelForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment")
        self.summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
    
    def _setup_device(self):
        """Setup device and move models to it"""
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.finbert_model.to(self.device)
        self.roberta_model.to(self.device)
    
    @lru_cache(maxsize=1024)
    def analyze_sentiment(self, text: str) -> float:
        if not text or len(text.strip()) < 10:
            return 5.0
    
        try:
            # Run sentiment analysis in parallel
            with ThreadPoolExecutor(max_workers=3) as executor:
                vader_future = executor.submit(self._get_vader_score, text)
                finbert_future = executor.submit(self._get_finbert_score, text)
                roberta_future = executor.submit(self._get_roberta_score, text)
    
                vader_score, vader_confidence = vader_future.result()
                finbert_score, finbert_confidence = finbert_future.result()
                roberta_score, roberta_confidence = roberta_future.result()
    
            # Calculate weighted average
            weights = np.array([vader_confidence, finbert_confidence, roberta_confidence])
            weights = weights / weights.sum()  # Normalize weights
            scores = np.array([vader_score, finbert_score, roberta_score])
    
            combined_score = np.dot(weights, scores)
            return max(0, min(combined_score, 10))
    
        except Exception as e:
            logging.error(f"Error in sentiment analysis: {e}")
            return 5.0
    
    def _get_vader_score(self, text: str) -> Tuple[float, float]:
        scores = self.sia.polarity_scores(text)
        return (scores['compound'] + 1) * 5, abs(scores['compound'])
    
    def _get_finbert_score(self, text: str) -> Tuple[float, float]:
        with torch.no_grad():
            inputs = self.finbert_tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(self.device)
            outputs = self.finbert_model(**inputs)
            probs = torch.nn.functional.softmax(outputs.logits, dim=1)
            sentiment = torch.argmax(probs).item()
            return sentiment * 5, torch.max(probs).item()
    
    def _get_roberta_score(self, text: str) -> Tuple[float, float]:
        with torch.no_grad():
            inputs = self.roberta_tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(self.device)
            outputs = self.roberta_model(**inputs)
            probs = torch.nn.functional.softmax(outputs.logits, dim=1)
            sentiment = torch.argmax(probs).item()
            return sentiment * 5, torch.max(probs).item()
    

    import sqlite3 import pandas as pd import plotly.graph_objects as go import logging

    class DataVisualizer: @staticmethod def create_visualizations(db_name: str): """Create enhanced visualizations with proper layout""" with sqlite3.connect(db_name) as conn: df = pd.read_sql_query("SELECT * FROM sentiment_scores", conn)

        if df.empty:
            logging.warning("No data available for visualization")
            return
    
        # Convert dates and create hour column
        df['date'] = pd.to_datetime(df['date'])
        df['hour'] = pd.to_datetime(df['time']).dt.hour
    
        # Create figure with subplots
        fig = make_subplots(
            rows=3, cols=3,
            subplot_titles=(
                'Daily Entry Counts',
                'Hourly Distribution',
                'Sentiment Timeline',
                'Summary Length Distribution',
                'Sentiment Distribution',
                'Weekly Patterns',
                'Sentiment Moving Average',
                'Headline Length vs Sentiment',
                'Time of Day Sentiment'
            ),
            specs=[
                [{'type': 'scatter'}, {'type': 'bar'}, {'type': 'scatter'}],
                [{'type': 'histogram'}, {'type': 'histogram'}, {'type': 'heatmap'}],
                [{'type': 'scatter'}, {'type': 'scatter'}, {'type': 'scatter'}]
            ],
            horizontal_spacing=0.12,  # Increased spacing between columns
            vertical_spacing=0.15     # Increased spacing between rows
        )
    
        # Daily Entry Counts
        daily_counts = df.groupby('date').size()
        fig.add_trace(
            go.Scatter(
                x=daily_counts.index,
                y=daily_counts.values,
                mode='lines+markers',
                name='Daily Entries',
                line=dict(width=2, color='royalblue'),
                marker=dict(size=6)
            ),
            row=1, col=1
        )
    
        # Hourly Distribution
        hourly_counts = df['hour'].value_counts().sort_index()
        fig.add_trace(
            go.Bar(
                x=hourly_counts.index,
                y=hourly_counts.values,
                name='Hourly Distribution',
                marker_color='lightblue'
            ),
            row=1, col=2
        )
    
        # Sentiment Timeline
        daily_sentiment = df.groupby('date')['score'].mean()
        fig.add_trace(
            go.Scatter(
                x=daily_sentiment.index,
                y=daily_sentiment.values,
                mode='lines',
                name='Daily Sentiment',
                line=dict(color='green', width=2)
            ),
            row=1, col=3
        )
    
        # Summary Length Distribution
        summary_lengths = DataVisualizer.get_summary_lengths(db_name)
        fig.add_trace(
            go.Histogram(
                x=summary_lengths,
                name='Summary Lengths',
                nbinsx=30,
                marker_color='lightgreen'
            ),
            row=2, col=1
        )
    
        # Sentiment Distribution
        fig.add_trace(
            go.Histogram(
                x=df['score'],
                name='Sentiment Distribution',
                nbinsx=20,
                histnorm='probability',
                marker_color='coral'
            ),
            row=2, col=2
        )
    
        # Weekly Patterns
        df['weekday'] = pd.to_datetime(df['date']).dt.day_name()
        weekly_sentiment = df.pivot_table(
            values='score',
            index='weekday',
            columns='hour',
            aggfunc='mean'
        )
    
        # Ensure consistent weekday order
        weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        weekly_sentiment = weekly_sentiment.reindex(weekday_order)
    
        fig.add_trace(
            go.Heatmap(
                z=weekly_sentiment.values,
                x=weekly_sentiment.columns,
                y=weekly_sentiment.index,
                colorscale='RdYlBu',
                name='Weekly Patterns',
                colorbar=dict(
                    title="Sentiment",
                    thickness=10,
                    len=0.3,
                    yanchor="middle",
                    y=0.5,
                    xanchor="left",
                    x=1.02
                )
            ),
            row=2, col=3
        )
    
        # Sentiment Moving Average
        daily_sentiment = df.groupby('date')['score'].mean().reset_index()
        daily_sentiment['MA7'] = daily_sentiment['score'].rolling(window=7).mean()
        fig.add_trace(
            go.Scatter(
                x=daily_sentiment['date'],
                y=daily_sentiment['MA7'],
                mode='lines',
                name='7-Day Moving Average',
                line=dict(color='purple', width=2)
            ),
            row=3, col=1
        )
    
        # Headline Length vs Sentiment
        df['title_length'] = df['title'].str.len()
        fig.add_trace(
            go.Scatter(
                x=df['title_length'],
                y=df['score'],
                mode='markers',
                name='Length vs Sentiment',
                marker=dict(
                    size=6,
                    color=df['score'],
                    colorscale='Viridis',
                    showscale=False  # Removed redundant colorbar
                )
            ),
            row=3, col=2
        )
    
        # Time of Day Sentiment
        hourly_sentiment = df.groupby('hour')['score'].mean()
        fig.add_trace(
            go.Scatter(
                x=hourly_sentiment.index,
                y=hourly_sentiment.values,
                mode='lines+markers',
                name='Hourly Sentiment',
                line=dict(shape='spline', color='orangered', width=2)
            ),
            row=3, col=3
        )
    
        # Update layout and axes
        fig.update_layout(
            height=1200,
            width=1600,
            showlegend=True,
            legend=dict(
                orientation="h",
                yanchor="bottom",
                y=-0.2,
                xanchor="center",
                x=0.5
            ),
            title_text="Enhanced News Analysis Dashboard",
            template="plotly_white"
        )
    
        # Update all x and y axis labels
        axis_labels = {
            (1,1): ('Date', 'Number of Articles'),
            (1,2): ('Hour of Day', 'Number of Articles'),
            (1,3): ('Date', 'Average Sentiment Score'),
            (2,1): ('Summary Length (characters)', 'Frequency'),
            (2,2): ('Sentiment Score', 'Probability'),
            (2,3): ('Hour of Day', 'Day of Week'),
            (3,1): ('Date', '7-Day Moving Average'),
            (3,2): ('Headline Length (characters)', 'Sentiment Score'),
            (3,3): ('Hour of Day', 'Average Sentiment Score')
        }
    
        for (row, col), (xlabel, ylabel) in axis_labels.items():
            fig.update_xaxes(title_text=xlabel, row=row, col=col)
            fig.update_yaxes(title_text=ylabel, row=row, col=col)
    
        # Show plot
        fig.show()
    
    @staticmethod
    def get_summary_lengths(db_name: str) -> list:
        try:
            with sqlite3.connect(db_name) as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT LENGTH(summary) FROM sentiment_scores")
                return [row[0] for row in cursor.fetchall()]
        except Exception as e:
            logging.error(f"Error getting summary lengths: {e}")
            return []
    

    class NewsFetcher: def init(self, db_manager, sentiment_analyzer): self.db_manager = db_manager self.sentiment_analyzer = sentiment_analyzer self._initialize_processed_titles()

    def _initialize_processed_titles(self):
        """Initialize set of processed titles from database"""
        try:
            with sqlite3.connect(self.db_manager.db_name) as conn:
                cursor = conn.execute("SELECT title FROM sentiment_scores")
                self.processed_titles = {row[0] for row in cursor.fetchall()}
            logging.info(f"Loaded {len(self.processed_titles)} existing titles from database")
        except Exception as e:
            logging.error(f"Error loading existing titles: {e}")
            self.processed_titles = set()
    
    def _is_similar_to_existing(self, title: str, threshold: float = 0.85) -> bool:
        """Check if a title is similar to any existing titles using SequenceMatcher"""
        for existing_title in self.processed_titles:
            similarity = SequenceMatcher(None, title.lower(), existing_title.lower()).ratio()
            if similarity > threshold:
                logging.info(f"Found similar title (similarity: {similarity:.2f})")
                return True
        return False
    
    def _is_valid_entry(self, entry) -> bool:
        """
        Validate entry based on content criteria
        Returns False for entries that should be skipped
        """
        # Check if summary exists and has minimum length
        if not hasattr(entry, 'summary') or len(entry.summary.strip()) < 17:
            logging.debug(f"Skipping entry with insufficient summary: {entry.title[:50]}...")
            return False
    
        # Check if title exists and has minimum length
        if not hasattr(entry, 'title') or len(entry.title.strip()) < 5:
            logging.debug("Skipping entry with insufficient title")
            return False
    
        # Check if published date exists
        if not hasattr(entry, 'published'):
            logging.debug("Skipping entry without published date")
            return False
    
        return True
    
    def fetch_and_process_news(self):
        """Fetch news from RSS feed and process them, avoiding duplicates and invalid entries"""
        try:
            logging.info("Fetching news from RSS feed...")
            feed = feedparser.parse(NEWS_FEED_URL)
            new_entries_count = 0
            skipped_entries_count = 0
    
            for entry in feed.entries:
                # Skip invalid entries
                if not self._is_valid_entry(entry):
                    skipped_entries_count += 1
                    continue
    
                # Skip if title already exists or is similar to existing ones
                if entry.title in self.processed_titles or self._is_similar_to_existing(entry.title):
                    logging.debug(f"Skipping duplicate/similar title: {entry.title[:50]}...")
                    skipped_entries_count += 1
                    continue
    
                try:
                    # Parse date and time
                    published = datetime.strptime(entry.published, DATETIME_FORMAT)
                    date = published.strftime(DATE_FORMAT)
                    time = published.strftime("%H:%M:%S")
    
                    # Analyze sentiment
                    combined_text = f"{entry.title} {entry.summary}"
                    sentiment_score = self.sentiment_analyzer.analyze_sentiment(combined_text)
    
                    # Store in database
                    self.db_manager.store_score(
                        date=date,
                        time=time,
                        title=entry.title,
                        summary=entry.summary,
                        score=sentiment_score
                    )
    
                    # Add to processed titles
                    self.processed_titles.add(entry.title)
                    new_entries_count += 1
    
                    logging.info(f"Processed new article: {entry.title[:50]}...")
    
                except Exception as e:
                    logging.error(f"Error processing entry: {e}")
                    skipped_entries_count += 1
                    continue
    
            logging.info(f"Added {new_entries_count} new articles, skipped {skipped_entries_count} invalid/duplicate entries")
    
        except Exception as e:
            logging.error(f"Error fetching news: {e}")
            raise
    

    Modified run_analysis function

    def run_analysis(): try: logging.info("Starting analysis...")

        # Initialize components
        db_manager = DatabaseManager(DATABASE_NAME)
        sentiment_analyzer = SentimentAnalyzer()
        news_fetcher = NewsFetcher(db_manager, sentiment_analyzer)
    
        # Fetch and process new articles
        logging.info("Fetching and processing news...")
        news_fetcher.fetch_and_process_news()
    
        # Generate visualizations
        logging.info("Generating main dashboard...")
        DataVisualizer.create_visualizations(DATABASE_NAME)
    
        # Analyze headlines
        logging.info("Analyzing headlines...")
        analyze_headlines()
    
        logging.info("Analysis complete!")
    
    except Exception as e:
        logging.error(f"Error during analysis: {e}")
        raise
    finally:
        if 'db_manager' in locals():
            db_manager.close()
    

    def analyze_headlines(): """Analyze and display recent headlines with their sentiment scores""" try: db_manager = DatabaseManager(DATABASE_NAME)

        # Get headline statistics
        stats = db_manager.get_headline_stats()
    
        # Get recent headlines
        recent_headlines = db_manager.get_headlines_with_scores(limit=10)
        most_positive, most_negative = db_manager.get_extreme_sentiment_headlines()
    
        # Create a figure for headlines analysis
        fig = make_subplots(
            rows=3, cols=1,
            subplot_titles=(
                'Recent Headlines with Sentiment Scores',
                'Most Positive Headlines',
                'Most Negative Headlines'
            ),
            specs=[[{"type": "table"}],
                  [{"type": "table"}],
                  [{"type": "table"}]],
            vertical_spacing=0.1
        )
    
        # Add statistics as annotations
        fig.add_annotation(
            text=(f"Total Headlines: {stats.get('total_headlines', 'N/A')} | "
                  f"Average Sentiment: {stats.get('average_sentiment', 0):.2f} | "
                  f"Date Range: {stats.get('date_range', 'N/A')}"),
            xref="paper", yref="paper",
            x=0, y=1.1,
            showarrow=False,
            font=dict(size=12)
        )
    
        # Add recent headlines table
        fig.add_trace(
            go.Table(
                header=dict(
                    values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                    fill_color='paleturquoise',
                    align='left',
                    font=dict(size=12)
                ),
                cells=dict(
                    values=[recent_headlines['date'],
                           recent_headlines['time'],
                           recent_headlines['title'],
                           recent_headlines['score'].round(2)],
                    fill_color='lavender',
                    align='left',
                    font=dict(size=11)
                )
            ),
            row=1, col=1
        )
    
        # Add most positive headlines
        fig.add_trace(
            go.Table(
                header=dict(
                    values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                    fill_color='lightgreen',
                    align='left'
                ),
                cells=dict(
                    values=[most_positive['date'],
                           most_positive['time'],
                           most_positive['title'],
                           most_positive['score'].round(2)],
                    fill_color='honeydew',
                    align='left'
                )
            ),
            row=2, col=1
        )
    
        # Add most negative headlines
        fig.add_trace(
            go.Table(
                header=dict(
                    values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                    fill_color='lightcoral',
                    align='left'
                ),
                cells=dict(
                    values=[most_negative['date'],
                           most_negative['time'],
                           most_negative['title'],
                           most_negative['score'].round(2)],
                    fill_color='mistyrose',
                    align='left'
                )
            ),
            row=3, col=1
        )
    
        # Update layout
        fig.update_layout(
            height=1000,
            title_text="Headlines Analysis Dashboard",
            showlegend=False
        )
    
        # Show plot
        fig.show()
    
    except Exception as e:
        logging.error(f"Error analyzing headlines: {e}")
        raise
    finally:
        db_manager.close()
    

    if name == "main": run_analysis()

    Remove all single line comments from this code and instead add multiple-line comments, one for each function, explaining what that function is doing

  2. Assistant
    # Cell 2
    
    import nltk
    import torch
    import requests
    import feedparser
    import sqlite3
    import pandas as pd
    import plotly.graph_objects as go
    import numpy as np
    from plotly.subplots import make_subplots
    from datetime import datetime, timedelta
    from nltk.sentiment import SentimentIntensityAnalyzer
    from transformers import (
        AutoTokenizer,
        AutoModelForSequenceClassification,
        pipeline
    )
    from concurrent.futures import ThreadPoolExecutor
    from functools import lru_cache
    import logging
    from typing import List, Tuple, Set
    from wordcloud import WordCloud
    import seaborn as sns
    from collections import Counter
    import matplotlib.pyplot as plt
    from nltk.tokenize import word_tokenize
    from nltk.corpus import stopwords
    import warnings
    from difflib import SequenceMatcher
    
    warnings.filterwarnings('ignore')
    
    # Configure logging
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
    
    # Constants
    DATABASE_NAME = 'news_sentiment.db'
    NEWS_FEED_URL = "https://www.thehindu.com/feeder/default.rss"
    DATE_FORMAT = "%Y-%m-%d"
    DATETIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
    
    class DatabaseManager:
        """
        Manages the SQLite database operations for storing and retrieving sentiment scores.
        It handles the creation of the necessary tables, insertion of new data,
        and various queries to retrieve statistics and specific datasets.
        """
        def __init__(self, db_name: str):
            self.db_name = db_name
            self._setup_database()
            self.connection = None
            self._setup_connection()
    
        def _setup_database(self):
            """
            Initializes the database by creating the sentiment_scores table if it does not exist.
            Also creates indexes on date, title, and score columns to optimize query performance.
            """
            with sqlite3.connect(self.db_name) as conn:
                conn.execute('''
                    CREATE TABLE IF NOT EXISTS sentiment_scores (
                        date TEXT,
                        time TEXT,
                        title TEXT,
                        summary TEXT,
                        score REAL
                    )
                ''')
                conn.execute('CREATE INDEX IF NOT EXISTS idx_date ON sentiment_scores(date)')
                conn.execute('CREATE INDEX IF NOT EXISTS idx_title ON sentiment_scores(title)')
                conn.execute('CREATE INDEX IF NOT EXISTS idx_score ON sentiment_scores(score)')
    
        def _setup_connection(self):
            """
            Establishes a persistent connection to the SQLite database with optimized parameters
            for performance. Configures the database to use Write-Ahead Logging and sets cache sizes.
            """
            if not self.connection:
                self.connection = sqlite3.connect(self.db_name, check_same_thread=False)
                self.connection.execute('PRAGMA journal_mode=WAL')  # Write-Ahead Logging
                self.connection.execute('PRAGMA synchronous=NORMAL')
                self.connection.execute('PRAGMA cache_size=-2000')  # 2MB cache
                self.connection.execute('PRAGMA temp_store=MEMORY')
    
        def store_score(self, date: str, time: str, title: str, summary: str, score: float):
            """
            Inserts a new sentiment score entry into the sentiment_scores table.
    
            Parameters:
                date (str): The date of the news article.
                time (str): The time of publication.
                title (str): The headline of the news article.
                summary (str): The summary of the news article.
                score (float): The sentiment score associated with the article.
            """
            try:
                self.connection.execute(
                    "INSERT INTO sentiment_scores VALUES (?, ?, ?, ?, ?)",
                    (date, time, title, summary, score)
                )
                self.connection.commit()
            except Exception as e:
                logging.error(f"Error storing score: {e}")
    
        @lru_cache(maxsize=128)
        def get_daily_average(self, date: str) -> float:
            """
            Retrieves the average sentiment score for a specific date.
    
            Parameters:
                date (str): The date for which to calculate the average sentiment.
    
            Returns:
                float: The average sentiment score, or 0.0 if no entries are found.
            """
            try:
                result = self.connection.execute(
                    "SELECT AVG(score) FROM sentiment_scores WHERE date = ?",
                    (date,)
                ).fetchone()
                return result[0] if result[0] is not None else 0.0
            except Exception as e:
                logging.error(f"Error getting daily average: {e}")
                return 0.0
    
        def get_headlines_with_scores(self, limit: int = 10, min_summary_length: int = 20) -> pd.DataFrame:
            """
            Retrieves recent headlines along with their sentiment scores,
            excluding entries with summaries shorter than the specified length.
    
            Parameters:
                limit (int): The maximum number of headlines to retrieve.
                min_summary_length (int): The minimum length of the summary to include an entry.
    
            Returns:
                pd.DataFrame: A DataFrame containing the selected headlines and their scores.
            """
            try:
                query = """
                    SELECT date, time, title, score
                    FROM sentiment_scores
                    WHERE LENGTH(summary) >= ?
                    ORDER BY date DESC, time DESC
                    LIMIT ?
                """
                return pd.read_sql_query(query, self.connection, params=(min_summary_length, limit))
            except Exception as e:
                logging.error(f"Error retrieving headlines: {e}")
                return pd.DataFrame()
    
        def get_extreme_sentiment_headlines(self) -> Tuple[pd.DataFrame, pd.DataFrame]:
            """
            Retrieves the top five most positive and top five most negative headlines based on sentiment scores.
    
            Returns:
                Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing the most positive and most negative headlines.
            """
            try:
                most_positive = pd.read_sql_query("""
                    SELECT date, time, title, score
                    FROM sentiment_scores
                    ORDER BY score DESC
                    LIMIT 5
                """, self.connection)
    
                most_negative = pd.read_sql_query("""
                    SELECT date, time, title, score
                    FROM sentiment_scores
                    ORDER BY score ASC
                    LIMIT 5
                """, self.connection)
    
                return most_positive, most_negative
            except Exception as e:
                logging.error(f"Error retrieving extreme headlines: {e}")
                return pd.DataFrame(), pd.DataFrame()
    
        def get_headline_stats(self) -> dict:
            """
            Gathers statistical information about the headlines stored in the database,
            including average sentiment, total number of headlines, and the date range covered.
    
            Returns:
                dict: A dictionary containing the statistical metrics.
            """
            try:
                stats = {}
    
                avg_query = "SELECT AVG(score) FROM sentiment_scores"
                stats['average_sentiment'] = self.connection.execute(avg_query).fetchone()[0]
    
                count_query = "SELECT COUNT(*) FROM sentiment_scores"
                stats['total_headlines'] = self.connection.execute(count_query).fetchone()[0]
    
                range_query = """
                    SELECT
                        MIN(date) as earliest_date,
                        MAX(date) as latest_date
                    FROM sentiment_scores
                """
                earliest, latest = self.connection.execute(range_query).fetchone()
                stats['date_range'] = f"{earliest} to {latest}"
    
                return stats
            except Exception as e:
                logging.error(f"Error getting headline stats: {e}")
                return {}
    
        def close(self):
            """
            Closes the database connection if it is open.
            """
            if self.connection:
                self.connection.close()
    
    class SentimentAnalyzer:
        """
        Performs sentiment analysis on text data using multiple NLP models.
        Combines results from VADER, FinBERT, and RoBERTa models to produce a comprehensive sentiment score.
        """
        def __init__(self):
            self._initialize_models()
            self._setup_device()
    
        def _initialize_models(self):
            """
            Initializes the necessary NLP models and tokenizers.
            Downloads required NLTK data if not already present.
            """
            for resource in ['vader_lexicon', 'punkt', 'stopwords']:
                try:
                    nltk.data.find(f'tokenizers/{resource}')
                except LookupError:
                    nltk.download(resource, quiet=True)
    
            self.sia = SentimentIntensityAnalyzer()
    
            self.finbert_tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert")
            self.finbert_model = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert")
            self.roberta_tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment")
            self.roberta_model = AutoModelForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment")
            self.summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
    
        def _setup_device(self):
            """
            Configures the computational device (GPU if available, else CPU)
            and moves the NLP models to the appropriate device.
            """
            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
            self.finbert_model.to(self.device)
            self.roberta_model.to(self.device)
    
        @lru_cache(maxsize=1024)
        def analyze_sentiment(self, text: str) -> float:
            """
            Analyzes the sentiment of the provided text using VADER, FinBERT, and RoBERTa models.
            Combines the scores from each model into a weighted average to produce a final sentiment score.
    
            Parameters:
                text (str): The text to analyze.
    
            Returns:
                float: A sentiment score between 0 and 10.
            """
            if not text or len(text.strip()) < 10:
                return 5.0
    
            try:
                with ThreadPoolExecutor(max_workers=3) as executor:
                    vader_future = executor.submit(self._get_vader_score, text)
                    finbert_future = executor.submit(self._get_finbert_score, text)
                    roberta_future = executor.submit(self._get_roberta_score, text)
    
                    vader_score, vader_confidence = vader_future.result()
                    finbert_score, finbert_confidence = finbert_future.result()
                    roberta_score, roberta_confidence = roberta_future.result()
    
                weights = np.array([vader_confidence, finbert_confidence, roberta_confidence])
                weights = weights / weights.sum()
                scores = np.array([vader_score, finbert_score, roberta_score])
    
                combined_score = np.dot(weights, scores)
                return max(0, min(combined_score, 10))
    
            except Exception as e:
                logging.error(f"Error in sentiment analysis: {e}")
                return 5.0
    
        def _get_vader_score(self, text: str) -> Tuple[float, float]:
            """
            Computes the sentiment score using the VADER model.
    
            Parameters:
                text (str): The text to analyze.
    
            Returns:
                Tuple[float, float]: The sentiment score scaled between 0 and 10, and the confidence score.
            """
            scores = self.sia.polarity_scores(text)
            return (scores['compound'] + 1) * 5, abs(scores['compound'])
    
        def _get_finbert_score(self, text: str) -> Tuple[float, float]:
            """
            Computes the sentiment score using the FinBERT model.
    
            Parameters:
                text (str): The text to analyze.
    
            Returns:
                Tuple[float, float]: The sentiment score scaled between 0 and 10, and the confidence score.
            """
            with torch.no_grad():
                inputs = self.finbert_tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(self.device)
                outputs = self.finbert_model(**inputs)
                probs = torch.nn.functional.softmax(outputs.logits, dim=1)
                sentiment = torch.argmax(probs).item()
                return sentiment * 5, torch.max(probs).item()
    
        def _get_roberta_score(self, text: str) -> Tuple[float, float]:
            """
            Computes the sentiment score using the RoBERTa model.
    
            Parameters:
                text (str): The text to analyze.
    
            Returns:
                Tuple[float, float]: The sentiment score scaled between 0 and 10, and the confidence score.
            """
            with torch.no_grad():
                inputs = self.roberta_tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(self.device)
                outputs = self.roberta_model(**inputs)
                probs = torch.nn.functional.softmax(outputs.logits, dim=1)
                sentiment = torch.argmax(probs).item()
                return sentiment * 5, torch.max(probs).item()
    
    import sqlite3
    import pandas as pd
    import plotly.graph_objects as go
    import logging
    
    class DataVisualizer:
        """
        Generates various visualizations and dashboards based on the sentiment scores stored in the database.
        Creates multiple subplots to display different aspects of the data, such as daily counts, sentiment trends,
        distribution of summaries, and more.
        """
        @staticmethod
        def create_visualizations(db_name: str):
            """
            Creates an enhanced set of visualizations that provide insights into the sentiment analysis data.
            Includes timelines, distributions, heatmaps, and scatter plots to illustrate different metrics.
    
            Parameters:
                db_name (str): The name of the SQLite database to retrieve data from.
            """
            with sqlite3.connect(db_name) as conn:
                df = pd.read_sql_query("SELECT * FROM sentiment_scores", conn)
    
            if df.empty:
                logging.warning("No data available for visualization")
                return
    
            df['date'] = pd.to_datetime(df['date'])
            df['hour'] = pd.to_datetime(df['time']).dt.hour
    
            fig = make_subplots(
                rows=3, cols=3,
                subplot_titles=(
                    'Daily Entry Counts',
                    'Hourly Distribution',
                    'Sentiment Timeline',
                    'Summary Length Distribution',
                    'Sentiment Distribution',
                    'Weekly Patterns',
                    'Sentiment Moving Average',
                    'Headline Length vs Sentiment',
                    'Time of Day Sentiment'
                ),
                specs=[
                    [{'type': 'scatter'}, {'type': 'bar'}, {'type': 'scatter'}],
                    [{'type': 'histogram'}, {'type': 'histogram'}, {'type': 'heatmap'}],
                    [{'type': 'scatter'}, {'type': 'scatter'}, {'type': 'scatter'}]
                ],
                horizontal_spacing=0.12,
                vertical_spacing=0.15
            )
    
            daily_counts = df.groupby('date').size()
            fig.add_trace(
                go.Scatter(
                    x=daily_counts.index,
                    y=daily_counts.values,
                    mode='lines+markers',
                    name='Daily Entries',
                    line=dict(width=2, color='royalblue'),
                    marker=dict(size=6)
                ),
                row=1, col=1
            )
    
            hourly_counts = df['hour'].value_counts().sort_index()
            fig.add_trace(
                go.Bar(
                    x=hourly_counts.index,
                    y=hourly_counts.values,
                    name='Hourly Distribution',
                    marker_color='lightblue'
                ),
                row=1, col=2
            )
    
            daily_sentiment = df.groupby('date')['score'].mean()
            fig.add_trace(
                go.Scatter(
                    x=daily_sentiment.index,
                    y=daily_sentiment.values,
                    mode='lines',
                    name='Daily Sentiment',
                    line=dict(color='green', width=2)
                ),
                row=1, col=3
            )
    
            summary_lengths = DataVisualizer.get_summary_lengths(db_name)
            fig.add_trace(
                go.Histogram(
                    x=summary_lengths,
                    name='Summary Lengths',
                    nbinsx=30,
                    marker_color='lightgreen'
                ),
                row=2, col=1
            )
    
            fig.add_trace(
                go.Histogram(
                    x=df['score'],
                    name='Sentiment Distribution',
                    nbinsx=20,
                    histnorm='probability',
                    marker_color='coral'
                ),
                row=2, col=2
            )
    
            df['weekday'] = pd.to_datetime(df['date']).dt.day_name()
            weekly_sentiment = df.pivot_table(
                values='score',
                index='weekday',
                columns='hour',
                aggfunc='mean'
            )
    
            weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
            weekly_sentiment = weekly_sentiment.reindex(weekday_order)
    
            fig.add_trace(
                go.Heatmap(
                    z=weekly_sentiment.values,
                    x=weekly_sentiment.columns,
                    y=weekly_sentiment.index,
                    colorscale='RdYlBu',
                    name='Weekly Patterns',
                    colorbar=dict(
                        title="Sentiment",
                        thickness=10,
                        len=0.3,
                        yanchor="middle",
                        y=0.5,
                        xanchor="left",
                        x=1.02
                    )
                ),
                row=2, col=3
            )
    
            daily_sentiment = df.groupby('date')['score'].mean().reset_index()
            daily_sentiment['MA7'] = daily_sentiment['score'].rolling(window=7).mean()
            fig.add_trace(
                go.Scatter(
                    x=daily_sentiment['date'],
                    y=daily_sentiment['MA7'],
                    mode='lines',
                    name='7-Day Moving Average',
                    line=dict(color='purple', width=2)
                ),
                row=3, col=1
            )
    
            df['title_length'] = df['title'].str.len()
            fig.add_trace(
                go.Scatter(
                    x=df['title_length'],
                    y=df['score'],
                    mode='markers',
                    name='Length vs Sentiment',
                    marker=dict(
                        size=6,
                        color=df['score'],
                        colorscale='Viridis',
                        showscale=False
                    )
                ),
                row=3, col=2
            )
    
            hourly_sentiment = df.groupby('hour')['score'].mean()
            fig.add_trace(
                go.Scatter(
                    x=hourly_sentiment.index,
                    y=hourly_sentiment.values,
                    mode='lines+markers',
                    name='Hourly Sentiment',
                    line=dict(shape='spline', color='orangered', width=2)
                ),
                row=3, col=3
            )
    
            fig.update_layout(
                height=1200,
                width=1600,
                showlegend=True,
                legend=dict(
                    orientation="h",
                    yanchor="bottom",
                    y=-0.2,
                    xanchor="center",
                    x=0.5
                ),
                title_text="Enhanced News Analysis Dashboard",
                template="plotly_white"
            )
    
            axis_labels = {
                (1,1): ('Date', 'Number of Articles'),
                (1,2): ('Hour of Day', 'Number of Articles'),
                (1,3): ('Date', 'Average Sentiment Score'),
                (2,1): ('Summary Length (characters)', 'Frequency'),
                (2,2): ('Sentiment Score', 'Probability'),
                (2,3): ('Hour of Day', 'Day of Week'),
                (3,1): ('Date', '7-Day Moving Average'),
                (3,2): ('Headline Length (characters)', 'Sentiment Score'),
                (3,3): ('Hour of Day', 'Average Sentiment Score')
            }
    
            for (row, col), (xlabel, ylabel) in axis_labels.items():
                fig.update_xaxes(title_text=xlabel, row=row, col=col)
                fig.update_yaxes(title_text=ylabel, row=row, col=col)
    
            fig.show()
    
        @staticmethod
        def get_summary_lengths(db_name: str) -> list:
            """
            Retrieves the lengths of all summaries stored in the sentiment_scores table.
    
            Parameters:
                db_name (str): The name of the SQLite database to query.
    
            Returns:
                list: A list of summary lengths.
            """
            try:
                with sqlite3.connect(db_name) as conn:
                    cursor = conn.cursor()
                    cursor.execute("SELECT LENGTH(summary) FROM sentiment_scores")
                    return [row[0] for row in cursor.fetchall()]
            except Exception as e:
                logging.error(f"Error getting summary lengths: {e}")
                return []
    
    class NewsFetcher:
        """
        Fetches news articles from an RSS feed, processes them to analyze sentiment,
        and stores the results in the database. Ensures that duplicate or similar
        articles are not processed multiple times.
        """
        def __init__(self, db_manager, sentiment_analyzer):
            self.db_manager = db_manager
            self.sentiment_analyzer = sentiment_analyzer
            self._initialize_processed_titles()
    
        def _initialize_processed_titles(self):
            """
            Loads existing article titles from the database to avoid processing duplicates.
            """
            try:
                with sqlite3.connect(self.db_manager.db_name) as conn:
                    cursor = conn.execute("SELECT title FROM sentiment_scores")
                    self.processed_titles = {row[0] for row in cursor.fetchall()}
                logging.info(f"Loaded {len(self.processed_titles)} existing titles from database")
            except Exception as e:
                logging.error(f"Error loading existing titles: {e}")
                self.processed_titles = set()
    
        def _is_similar_to_existing(self, title: str, threshold: float = 0.85) -> bool:
            """
            Determines if a given title is similar to any already processed titles using SequenceMatcher.
    
            Parameters:
                title (str): The title to compare.
                threshold (float): The similarity threshold above which a title is considered similar.
    
            Returns:
                bool: True if the title is similar to an existing one, False otherwise.
            """
            for existing_title in self.processed_titles:
                similarity = SequenceMatcher(None, title.lower(), existing_title.lower()).ratio()
                if similarity > threshold:
                    logging.info(f"Found similar title (similarity: {similarity:.2f})")
                    return True
            return False
    
        def _is_valid_entry(self, entry) -> bool:
            """
            Validates a news entry based on the presence and length of its summary and title,
            and the existence of a publication date.
    
            Parameters:
                entry: The news entry to validate.
    
            Returns:
                bool: True if the entry is valid, False otherwise.
            """
            if not hasattr(entry, 'summary') or len(entry.summary.strip()) < 17:
                logging.debug(f"Skipping entry with insufficient summary: {entry.title[:50]}...")
                return False
    
            if not hasattr(entry, 'title') or len(entry.title.strip()) < 5:
                logging.debug("Skipping entry with insufficient title")
                return False
    
            if not hasattr(entry, 'published'):
                logging.debug("Skipping entry without published date")
                return False
    
            return True
    
        def fetch_and_process_news(self):
            """
            Fetches news articles from the RSS feed, analyzes their sentiment,
            and stores valid and unique entries into the database.
            """
            try:
                logging.info("Fetching news from RSS feed...")
                feed = feedparser.parse(NEWS_FEED_URL)
                new_entries_count = 0
                skipped_entries_count = 0
    
                for entry in feed.entries:
                    if not self._is_valid_entry(entry):
                        skipped_entries_count += 1
                        continue
    
                    if entry.title in self.processed_titles or self._is_similar_to_existing(entry.title):
                        logging.debug(f"Skipping duplicate/similar title: {entry.title[:50]}...")
                        skipped_entries_count += 1
                        continue
    
                    try:
                        published = datetime.strptime(entry.published, DATETIME_FORMAT)
                        date = published.strftime(DATE_FORMAT)
                        time = published.strftime("%H:%M:%S")
    
                        combined_text = f"{entry.title} {entry.summary}"
                        sentiment_score = self.sentiment_analyzer.analyze_sentiment(combined_text)
    
                        self.db_manager.store_score(
                            date=date,
                            time=time,
                            title=entry.title,
                            summary=entry.summary,
                            score=sentiment_score
                        )
    
                        self.processed_titles.add(entry.title)
                        new_entries_count += 1
    
                        logging.info(f"Processed new article: {entry.title[:50]}...")
    
                    except Exception as e:
                        logging.error(f"Error processing entry: {e}")
                        skipped_entries_count += 1
                        continue
    
                logging.info(f"Added {new_entries_count} new articles, skipped {skipped_entries_count} invalid/duplicate entries")
    
            except Exception as e:
                logging.error(f"Error fetching news: {e}")
                raise
    
    # Modified run_analysis function
    def run_analysis():
        """
        Orchestrates the entire analysis process by initializing components,
        fetching and processing news articles, generating visualizations,
        and performing headline analysis. Ensures that all resources are properly closed.
        """
        try:
            logging.info("Starting analysis...")
    
            db_manager = DatabaseManager(DATABASE_NAME)
            sentiment_analyzer = SentimentAnalyzer()
            news_fetcher = NewsFetcher(db_manager, sentiment_analyzer)
    
            logging.info("Fetching and processing news...")
            news_fetcher.fetch_and_process_news()
    
            logging.info("Generating main dashboard...")
            DataVisualizer.create_visualizations(DATABASE_NAME)
    
            logging.info("Analyzing headlines...")
            analyze_headlines()
    
            logging.info("Analysis complete!")
    
        except Exception as e:
            logging.error(f"Error during analysis: {e}")
            raise
        finally:
            if 'db_manager' in locals():
                db_manager.close()
    
    def analyze_headlines():
        """
        Analyzes and displays recent headlines along with their sentiment scores.
        Retrieves statistical data and extreme sentiment headlines, presenting them in a structured dashboard.
        """
        try:
            db_manager = DatabaseManager(DATABASE_NAME)
    
            stats = db_manager.get_headline_stats()
    
            recent_headlines = db_manager.get_headlines_with_scores(limit=10)
            most_positive, most_negative = db_manager.get_extreme_sentiment_headlines()
    
            fig = make_subplots(
                rows=3, cols=1,
                subplot_titles=(
                    'Recent Headlines with Sentiment Scores',
                    'Most Positive Headlines',
                    'Most Negative Headlines'
                ),
                specs=[[{"type": "table"}],
                       [{"type": "table"}],
                       [{"type": "table"}]],
                vertical_spacing=0.1
            )
    
            fig.add_annotation(
                text=(f"Total Headlines: {stats.get('total_headlines', 'N/A')} | "
                      f"Average Sentiment: {stats.get('average_sentiment', 0):.2f} | "
                      f"Date Range: {stats.get('date_range', 'N/A')}"),
                xref="paper", yref="paper",
                x=0, y=1.1,
                showarrow=False,
                font=dict(size=12)
            )
    
            fig.add_trace(
                go.Table(
                    header=dict(
                        values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                        fill_color='paleturquoise',
                        align='left',
                        font=dict(size=12)
                    ),
                    cells=dict(
                        values=[recent_headlines['date'],
                               recent_headlines['time'],
                               recent_headlines['title'],
                               recent_headlines['score'].round(2)],
                        fill_color='lavender',
                        align='left',
                        font=dict(size=11)
                    )
                ),
                row=1, col=1
            )
    
            fig.add_trace(
                go.Table(
                    header=dict(
                        values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                        fill_color='lightgreen',
                        align='left'
                    ),
                    cells=dict(
                        values=[most_positive['date'],
                               most_positive['time'],
                               most_positive['title'],
                               most_positive['score'].round(2)],
                        fill_color='honeydew',
                        align='left'
                    )
                ),
                row=2, col=1
            )
    
            fig.add_trace(
                go.Table(
                    header=dict(
                        values=['Date', 'Time', 'Headline', 'Sentiment Score'],
                        fill_color='lightcoral',
                        align='left'
                    ),
                    cells=dict(
                        values=[most_negative['date'],
                               most_negative['time'],
                               most_negative['title'],
                               most_negative['score'].round(2)],
                        fill_color='mistyrose',
                        align='left'
                    )
                ),
                row=3, col=1
            )
    
            fig.update_layout(
                height=1000,
                title_text="Headlines Analysis Dashboard",
                showlegend=False
            )
    
            fig.show()
    
        except Exception as e:
            logging.error(f"Error analyzing headlines: {e}")
            raise
        finally:
            db_manager.close()
    
    
    if __name__ == "__main__":
        run_analysis()