Skip to content
The Chat Archive
o1-mini-2024-09-12· OpenAI WildChat-4.8M (AI2)

This is my current code. i need you to make it more professional and remove unnecessary…

  1. User

    This is my current code. i need you to make it more professional and remove unnecessary steps. i need can you help me making it better? what could i do to improve it?you to add more logging info

    import os import re import csv import argparse import logging import nltk import pandas as pd from collections import defaultdict, Counter from nltk.stem import WordNetLemmatizer from gensim.models import KeyedVectors, FastText from sklearn.cluster import AgglomerativeClustering import numpy as np from sklearn.preprocessing import OneHotEncoder from nltk.corpus import wordnet as wn from nltk.corpus import stopwords

    def download_nltk_data(): """ Ensures that required NLTK data packages are downloaded. """ nltk_data_packages = ['wordnet', 'stopwords'] for package in nltk_data_packages: try: nltk.data.find(f'corpora/{package}') except LookupError: nltk.download(package)

    def parse_arguments(): """ Parses command-line arguments. """ parser = argparse.ArgumentParser(description='Process and cluster keywords from a CSV file.') parser.add_argument('--input_file', type=str, help='Path to the input CSV file.') parser.add_argument('--save_lemmatized', action='store_true', help='Save lemmatized keywords and counts.') parser.add_argument('--model_type', type=str, choices=['word2vec', 'glove', 'fasttext'], default='glove', help='Type of model to use for clustering. Default is "glove".') parser.add_argument('--model_path', type=str, default='/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt', required=False, help='Path to the word embedding model file.') parser.add_argument('--frequency_threshold', type=int, default=10, help='Frequency threshold for keywords. Default is 10.') parser.add_argument('--num_clusters', type=int, default=50, help='Number of clusters for infrequent keywords. Default is 50.') parser.add_argument('--distance_threshold', type=float, default=None, help='Distance threshold for clustering. Overrides num_clusters if set.') parser.add_argument('--category_limit', type=int, default=100, help='Limit of top N keywords per category. Default is 100.') parser.add_argument('--remove_duplicates', action='store_true', help='Remove duplicates in clustered keywords per row.') return parser.parse_args()

    def main(): # Parse command-line arguments args = parse_arguments() input_file = args.input_file save_lemmatized = args.save_lemmatized model_type = args.model_type model_path = args.model_path frequency_threshold = args.frequency_threshold num_clusters = args.num_clusters distance_threshold = args.distance_threshold category_limit = args.category_limit remove_duplicates = args.remove_duplicates

    # Check if input file exists
    if not os.path.exists(input_file):
        raise FileNotFoundError(f"Input file '{input_file}' does not exist.")
    
    # Set up logging
    input_dir = os.path.dirname(os.path.abspath(input_file))
    input_filename = os.path.basename(input_file)
    input_name_no_ext = os.path.splitext(input_filename)[0]
    processed_dir = os.path.join(input_dir, 'processed')
    os.makedirs(processed_dir, exist_ok=True)
    log_file = os.path.join(processed_dir, f'log_{input_name_no_ext}.log')
    logging.basicConfig(filename=log_file, level=logging.INFO,
                        format='%(asctime)s %(levelname)s:%(message)s')
    # Also log to stdout
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.INFO)
    console_formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s')
    console_handler.setFormatter(console_formatter)
    logging.getLogger().addHandler(console_handler)
    
    logging.info(f"Processing input file: {input_file}")
    
    # Download NLTK data if necessary
    download_nltk_data()
    
    # Initialize the lemmatizer
    lemmatizer = WordNetLemmatizer()
    # Set of stopwords to filter out
    stop_words = set(stopwords.words('english'))
    
    # Labels (customizable)
    labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style']  # Modify as needed
    
    # Define output file paths
    output_file = os.path.join(processed_dir, f'clustered_{input_filename}')
    one_hot_encoded_file = os.path.join(processed_dir, f'onehot_{input_name_no_ext}.tsv')
    lemmatized_counts_output_file = os.path.join(processed_dir, f'lemmatized_keyword_counts.csv')
    
    # ------------------------------#
    # Step 1: Cleaning #
    # ------------------------------#
    logging.info("Starting CSV cleaning...")
    df_clean = clean_csv(input_file, labels)
    logging.info("CSV cleaning completed.")
    
    # ------------------------------#
    # Step 2: Lemmatization #
    # ------------------------------#
    logging.info("Starting lemmatization...")
    if save_lemmatized:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatized_counts_output_file, lemmatizer)
    else:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, None, lemmatizer)
    # Check if per_row_df is None
    if per_row_df is None:
        logging.error("Error in lemmatization.")
        return
    
    # ------------------------------#
    # Step 3: Load Model #
    # ------------------------------#
    model = load_model(model_type, model_path)
    
    # ------------------------------#
    # Step 4: Clustering #
    # ------------------------------#
    clustered_keyword_mapping = perform_clustering_with_threshold(
        keyword_counts, model, model_type,
        threshold=frequency_threshold, num_clusters=num_clusters,
        distance_threshold=distance_threshold
    )
    
    # ------------------------------#
    # Step 5: Clustered Keywords #
    # ------------------------------#
    per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
        lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
    )
    
    # ------------------------------#
    # Step 6: Categorization #
    # ------------------------------#
    category_keyword_counts = categorize_and_count_clustered_keywords(
        per_row_df, labels=labels, category_limit=category_limit
    )
    
    # ------------------------------#
    # Step 7: Final Processing #
    # ------------------------------#
    per_row_df = generate_clustered_keywords_and_categories(
        per_row_df, clustered_keyword_mapping, category_keyword_counts, labels=labels,
        remove_duplicates=remove_duplicates
    )
    
    # Optionally, write category counts to CSV files
    if save_lemmatized:
        for category, counts in category_keyword_counts.items():
            category_counts_df = pd.DataFrame({'Keyword': list(counts.keys()), 'Count': list(counts.values())})
            category_counts_df = category_counts_df.sort_values(by='Count', ascending=False)
            category_counts_output_file = os.path.join(processed_dir, f'{category}_keywords_counts.csv')
            category_counts_df.to_csv(category_counts_output_file, index=False)
            logging.info(f"{category} keyword counts saved to '{category_counts_output_file}'.")
    
    # ------------------------------#
    # Step 8: Save Results #
    # ------------------------------#
    per_row_df.to_csv(output_file, index=False)
    logging.info(f"Process completed. Output saved to '{output_file}'.")
    
    # ------------------------------#
    # Step 9: One-Hot Encoding #
    # ------------------------------#
    encode_categorical_features(
        per_row_df, one_hot_encoded_file,
        id_column='Image Name', remove_extension='.jpg',
        labels=labels,
        sep='\t', header=False, index=False
    )
    logging.info(f"One-hot encoded features saved to '{one_hot_encoded_file}'.")
    

    def clean_csv(input_file, labels): """ Cleans the input CSV file and returns a DataFrame. """ total_row_count = 0 empty_row_count = 0 na_count = 0 no_comma_count = 0 no_comma_rows = [] cleaned_data = [] # Read the entire content of the file try: with open(input_file, 'r', newline='', encoding='utf-8') as infile: content = infile.read() except Exception as e: logging.error(f"Error reading input file: {e}") return None # Remove newline characters inside quoted strings content = re.sub(r'("[.*?")\n', r'\1 ', content) # Use StringIO to read the cleaned content as a CSV import io temp_file = io.StringIO(content) reader = csv.reader(temp_file)

    # Skip the first line (header)
    try:
        header = next(reader)
    except StopIteration:
        logging.error("Input file is empty.")
        return None
    
    # Process each row
    for row in reader:
        total_row_count += 1
        if not row:
            continue
        jpg_file = row[0]
        if len(row) < 2:
            logging.warning(f"Row {total_row_count} doesn't have enough columns: {row}")
            keywords = []
        else:
            keywords = extract_keywords(row[1], labels)
        # Count occurrences of "N/A" and empty spaces
        na_count += keywords.count("N/A")
        na_count += sum(1 for k in keywords if k.strip() == "")
        # Check if the row is empty or contains only "N/A" or empty spaces
        if not keywords or all(k.strip().upper() == 'N/A' or k.strip() == '' for k in keywords):
            empty_row_count += 1
            continue  # Skip this row
        # Concatenate the keywords into a single string
        keywords_str = ', '.join(keywords)
        # Replace multiple spaces with a comma and a single space
        keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
        # Check if the row has no commas
        if ',' not in keywords_str:
            no_comma_count += 1
            no_comma_rows.append(f'{jpg_file},{keywords_str}')
        # Add to cleaned data
        cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
    # Now, create a DataFrame from the cleaned_data
    df = pd.DataFrame(cleaned_data)
    # Clean the 'Keywords' column further
    df = clean_keywords_in_df(df, labels)
    # Log statistics
    logging.info(f"Total rows processed: {total_row_count}")
    logging.info(f"Number of empty or N/A rows: {empty_row_count}")
    logging.info(f"Occurrences of N/A (including empty spaces): {na_count}")
    logging.info(f"Number of rows without commas: {no_comma_count}")
    if no_comma_rows:
        logging.info("Rows without commas:")
        for row in no_comma_rows:
            logging.info(row)
    return df
    

    def extract_keywords(line, labels): """ Extracts keywords from a line. """ # First, try to extract keywords between curly braces keywords = re.findall(r'{([^}]+)}', line) if keywords: # Further split if multiple keywords are within braces return [kw.strip() for group in keywords for kw in group.split(',')] # Remove newline characters from the keywords section line = line.replace('\n', ' ') # Remove the labels using word boundaries and considering possible formats for label in labels: # Patterns like "Label:", "Label -", "[Label]:", etc. pattern = rf'\b{re.escape(label)}\b[: -]?' line = re.sub(pattern, '', line, flags=re.IGNORECASE) # Remove any '- ' (dash and space) using exact string replacement line = line.replace('- ', '') # Remove single quotes around keywords using regex line = re.sub(r"'([^']+)'", r'\1', line) # Remove any remaining brackets line = line.replace('[', '').replace(']', '').replace('{', '').replace('}', '') # Split keywords by commas and strip whitespace keywords = [keyword.strip() for keyword in line.split(',')] return keywords

    def clean_keywords_in_df(df, labels): """ Cleans the 'Keywords' column in the DataFrame. """ # Construct a regex pattern from the labels with word boundaries labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*' # Replace labels with an empty string using regex df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True) # Remove any '- ' (dash and space) df['Keywords'] = df['Keywords'].str.replace('- ', '', regex=False) # Remove single quotes around keywords df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True) # Replace multiple spaces with a comma and a single space df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True) # Remove leading/trailing commas and spaces df['Keywords'] = df['Keywords'].str.strip(' ,') return df

    def lemmatize_keywords(df, lemmatized_counts_output_file, lemmatizer): """ Lemmatizes the keywords in the DataFrame and returns a DataFrame with lemmatized keywords. Optionally saves the lemmatized keyword counts to a CSV file. """ per_row_data = [] keyword_counts = Counter() # Iterate through each row to extract and lemmatize keywords for index, row in df.iterrows(): image_name = row.get('Image Name', '') keywords_str = row.get('Keywords', '') if pd.isna(keywords_str): keywords_str = '' # Split keywords by comma and strip whitespace extracted_keywords = [kw.strip() for kw in keywords_str.split(',')] # Lemmatize keywords lemmatized_keywords = [lemmatizer.lemmatize(kw.lower()) for kw in extracted_keywords if kw] # Update global counts keyword_counts.update(lemmatized_keywords) # Store per-row data per_row_data.append({ 'Image Name': image_name, 'Keywords': ', '.join(extracted_keywords), 'Lemmatized Keywords': ', '.join(lemmatized_keywords) }) # Create a DataFrame from per_row_data per_row_df = pd.DataFrame(per_row_data) if lemmatized_counts_output_file: # Write the lemmatized keyword counts to CSV lemmatized_counts = [{'Lemmatized Keyword': kw, 'Count': count} for kw, count in keyword_counts.items()] lemmatized_counts_df = pd.DataFrame(lemmatized_counts) # Sort by Count in descending order lemmatized_counts_df = lemmatized_counts_df.sort_values(by='Count', ascending=False) # Write to CSV lemmatized_counts_df.to_csv(lemmatized_counts_output_file, index=False) logging.info(f"Lemmatized keyword counts saved to '{lemmatized_counts_output_file}'.") return per_row_df, keyword_counts

    def load_model(model_type, model_path): """ Loads a word embedding model based on the specified type and path. """ if model_type == 'word2vec': logging.info("Loading Word2Vec model. This may take a few minutes...") model = KeyedVectors.load_word2vec_format(model_path, binary=True) logging.info("Word2Vec model loaded successfully.") elif model_type == 'glove': logging.info("Loading GloVe model. This may take a few minutes...") model = {} with open(model_path, 'r', encoding='utf-8') as f: for line in f: values = line.split() word = values[0] vector = np.asarray(values[1:], dtype='float32') model[word] = vector logging.info("GloVe model loaded successfully.") elif model_type == 'fasttext': logging.info("Loading FastText model. This may take a few minutes...") model = FastText.load_fasttext_format(model_path) logging.info("FastText model loaded successfully.") else: raise ValueError(f"Unsupported model type: {model_type}") return model

    def get_word_vector(word, model, model_type): """ Retrieves the vector representation of a word from the model. """ if word in model: return model[word] # Handle multi-word phrases by averaging their word vectors words = word.split() vectors = [] for w in words: if w in model: vectors.append(model[w]) if vectors: return np.mean(vectors, axis=0) else: return None

    def perform_clustering_with_threshold(keyword_counts, model, model_type, threshold=5, num_clusters=5, distance_threshold=None): """ Clusters infrequent keywords using hierarchical clustering. """ # Separate frequent and infrequent keywords frequent_keywords = [kw for kw, count in keyword_counts.items() if count > threshold] infrequent_keywords = [kw for kw, count in keyword_counts.items() if count <= threshold] logging.info(f"Number of frequent keywords (> {threshold}): {len(frequent_keywords)}") logging.info(f"Number of infrequent keywords (<= {threshold}): {len(infrequent_keywords)}") # Keep frequent keywords as-is clustered = {kw: kw for kw in frequent_keywords} # Cluster infrequent keywords and map them to their representatives if infrequent_keywords: mapping = cluster_infrequent_keywords( infrequent_keywords, keyword_counts, model, model_type, num_clusters=num_clusters, distance_threshold=distance_threshold ) clustered.update(mapping) logging.info(f"Number of cluster representatives: {len(set(mapping.values()))}") else: logging.info("No infrequent keywords to cluster.") return clustered

    def cluster_infrequent_keywords(keywords, keyword_counts, model, model_type, num_clusters=5, distance_threshold=None): """ Clusters infrequent keywords and maps them to cluster representatives. """ vectors = [] valid_keywords = [] for keyword in keywords: vector = get_word_vector(keyword, model, model_type) if vector is not None: vectors.append(vector) valid_keywords.append(keyword) else: logging.warning(f"Vector not found for keyword: '{keyword}'") if not vectors: logging.warning("No valid vectors found for clustering.") return {} vectors = np.array(vectors) # Perform Hierarchical Clustering with distance threshold if distance_threshold is not None: logging.info(f"Performing Agglomerative Clustering with distance_threshold={distance_threshold}") clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold) else: logging.info(f"Performing Agglomerative Clustering with n_clusters={num_clusters}") clustering_model = AgglomerativeClustering(n_clusters=num_clusters) labels = clustering_model.fit_predict(vectors) # Organize keywords by clusters clusters = defaultdict(list) for label, keyword in zip(labels, valid_keywords): clusters[label].append(keyword) # Select the most frequent keyword in each cluster as the representative representative_keywords = {} for label, words in clusters.items(): most_frequent = max(words, key=lambda w: keyword_counts[w]) representative_keywords[label] = most_frequent # Create a mapping from keyword to representative keyword_to_representative = {} for label, words in clusters.items(): rep = representative_keywords[label] for word in words: keyword_to_representative[word] = rep return keyword_to_representative

    def categorize_and_count_clustered_keywords(per_row_df, labels, category_limit=50): """ Categorizes clustered keywords and counts them per category. """ category_keyword_counts = {label: Counter() for label in labels} for idx, row in per_row_df.iterrows(): clustered_keywords = row['Clustered Keywords'].split(', ') for i, label in enumerate(labels): if i < len(clustered_keywords): keyword = clustered_keywords[i] category_keyword_counts[label][keyword] += 1 # Limit to top N keywords per category for label in labels: original_count = len(category_keyword_counts[label]) category_keyword_counts[label] = Counter(dict(category_keyword_counts[label].most_common(category_limit))) logging.info(f"Category '{label}': Reduced from {original_count} to {len(category_keyword_counts[label])} keywords (top {category_limit})")

    return category_keyword_counts
    

    def generate_clustered_keywords_and_categories(per_row_df, clustered_keyword_mapping, category_keyword_counts, labels, remove_duplicates=False): """ Generates the final DataFrame with clustered keywords and category assignments. """ top_keywords_per_category = {label: set(counts.keys()) for label, counts in category_keyword_counts.items()} # Initialize lists for categories and counters for "Other" category_keywords_list = {label: [] for label in labels} clustered_keywords_list = [] # Initialize a counter for the number of rows with specific "Other" counts max_other_count = len(labels) other_count_rows = {i: 0 for i in range(max_other_count + 1)} # from 0 to len(labels) for idx, row in per_row_df.iterrows(): lemmatized_keywords = row['Lemmatized Keywords'].split(', ') clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords] if remove_duplicates: # Remove duplicates while preserving order clustered_keywords = list(dict.fromkeys(clustered_keywords)) clustered_keywords_list.append(', '.join(clustered_keywords)) # Initialize "Other" counter for each row row_other_count = 0 # Assign keywords to categories based on positions for i, label in enumerate(labels): if i < len(clustered_keywords): keyword = clustered_keywords[i] # Map to 'Other' if not in top keywords if keyword not in top_keywords_per_category[label]: keyword = 'Other' row_other_count += 1 # Increase the row counter for "Other" category_keywords_list[label].append(keyword) else: category_keywords_list[label].append('') # or 'Other' # Count the number of rows with the specific number of "Other" keywords if row_other_count <= max_other_count: other_count_rows[row_other_count] += 1 per_row_df['Clustered Keywords'] = clustered_keywords_list # Add the category columns to per_row_df for label in labels: per_row_df[label] = category_keywords_list[label] # Log summary of "Other" counts per category other_count = {label: 0 for label in labels} # Counter for "Other" per category logging.info("\nSummary of 'Other' counts per category:") for label in labels: other_count[label] = sum(1 for keyword in category_keywords_list[label] if keyword == 'Other') logging.info(f"{label}: {other_count[label]}") # Log the summary of rows having specific "Other" counts logging.info("\nSummary of 'Other' count rows:") for count in range(max_other_count + 1): logging.info(f"Rows with {count} 'Other' keywords: {other_count_rows[count]}") return per_row_df

    def encode_categorical_features(input_df, output_tsv_path, id_column='Image Name', remove_extension='.jpg', labels=None, sep='\t', header=False, index=False): """ Encodes categorical features using One-Hot Encoding and saves them to a TSV file. """ if labels is None: raise ValueError("Labels for encoding must be provided.") # Read only the necessary columns columns_to_encode = labels columns_to_read = [id_column] + columns_to_encode df = input_df[columns_to_read] # Initialize OneHotEncoder enc = OneHotEncoder(handle_unknown='ignore', dtype=np.int8) # Fit and transform the categorical columns emb = enc.fit_transform(df[columns_to_encode]) # Retrieve the names of the one-hot encoded features feature_names = enc.get_feature_names_out(columns_to_encode) # Create a DataFrame with the encoded features emb_df = pd.DataFrame(emb.toarray(), columns=feature_names, dtype=np.int8) # Concatenate the id_column with the one-hot encoded features result_df = pd.concat([df[[id_column]].reset_index(drop=True), emb_df.reset_index(drop=True)], axis=1) # Remove specified extension from id_column if remove_extension: result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False) # Save to TSV file result_df.to_csv(output_tsv_path, index=index, header=header, sep=sep)

    if name == "main": main()

  2. Assistant

    Certainly! Improving code for professionalism involves enhancing readability, maintainability, efficiency, and robustness. Below, I've refactored your code with these principles in mind. Key improvements include:

    1. Modularization: Breaking down the code into smaller, reusable functions.
    2. Error Handling: Adding comprehensive error handling to manage unexpected scenarios gracefully.
    3. Logging Enhancements: Incorporating more detailed logging to facilitate easier debugging and monitoring.
    4. Performance Optimizations: Streamlining operations to reduce computational overhead.
    5. Code Cleanup: Removing redundant steps and optimizing existing ones for better performance.

    Refactored Code

    import os
    import re
    import csv
    import argparse
    import logging
    import nltk
    import pandas as pd
    from collections import defaultdict, Counter
    from nltk.stem import WordNetLemmatizer
    from gensim.models import KeyedVectors, FastText
    from sklearn.cluster import AgglomerativeClustering
    import numpy as np
    from sklearn.preprocessing import OneHotEncoder
    from nltk.corpus import stopwords
    import io
    from typing import Tuple, Dict, List, Optional
    
    def setup_logging(input_file: str) -> None:
        """
        Configures logging to file and console.
    
        Args:
            input_file (str): Path to the input CSV file.
        """
        input_dir = os.path.dirname(os.path.abspath(input_file))
        input_filename = os.path.basename(input_file)
        input_name_no_ext = os.path.splitext(input_filename)[0]
        processed_dir = os.path.join(input_dir, 'processed')
        os.makedirs(processed_dir, exist_ok=True)
        log_file = os.path.join(processed_dir, f'log_{input_name_no_ext}.log')
        
        # Configure logging to file
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s %(levelname)s: %(message)s',
            handlers=[
                logging.FileHandler(log_file),
                logging.StreamHandler()
            ]
        )
        logging.info(f"Logging initialized. Log file: {log_file}")
    
    def download_nltk_data() -> None:
        """
        Ensures that required NLTK data packages are downloaded.
        """
        nltk_data_packages = ['wordnet', 'stopwords']
        for package in nltk_data_packages:
            try:
                nltk.data.find(f'corpora/{package}')
                logging.debug(f"NLTK package '{package}' already downloaded.")
            except LookupError:
                logging.info(f"Downloading NLTK package: {package}")
                nltk.download(package)
    
    def parse_arguments() -> argparse.Namespace:
        """
        Parses command-line arguments.
    
        Returns:
            argparse.Namespace: Parsed arguments.
        """
        parser = argparse.ArgumentParser(description='Process and cluster keywords from a CSV file.')
        parser.add_argument('--input_file', type=str, required=True, help='Path to the input CSV file.')
        parser.add_argument('--save_lemmatized', action='store_true', help='Save lemmatized keywords and counts.')
        parser.add_argument('--model_type', type=str, choices=['word2vec', 'glove', 'fasttext'], default='glove',
                            help='Type of model to use for clustering. Default is "glove".')
        parser.add_argument('--model_path', type=str, default='/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt',
                            help='Path to the word embedding model file.')
        parser.add_argument('--frequency_threshold', type=int, default=10,
                            help='Frequency threshold for keywords. Default is 10.')
        parser.add_argument('--num_clusters', type=int, default=50,
                            help='Number of clusters for infrequent keywords. Default is 50.')
        parser.add_argument('--distance_threshold', type=float, default=None,
                            help='Distance threshold for clustering. Overrides num_clusters if set.')
        parser.add_argument('--category_limit', type=int, default=100,
                            help='Limit of top N keywords per category. Default is 100.')
        parser.add_argument('--remove_duplicates', action='store_true',
                            help='Remove duplicates in clustered keywords per row.')
        return parser.parse_args()
    
    def clean_csv(input_file: str, labels: List[str]) -> pd.DataFrame:
        """
        Cleans the input CSV file and returns a DataFrame.
    
        Args:
            input_file (str): Path to the input CSV file.
            labels (List[str]): List of label categories.
    
        Returns:
            pd.DataFrame: Cleaned DataFrame with 'Image Name' and 'Keywords'.
        """
        logging.info("Starting CSV cleaning...")
        total_row_count = 0
        empty_row_count = 0
        na_count = 0
        no_comma_count = 0
        no_comma_rows = []
        cleaned_data = []
    
        try:
            with open(input_file, 'r', newline='', encoding='utf-8') as infile:
                content = infile.read()
        except Exception as e:
            logging.error(f"Error reading input file: {e}")
            raise
    
        # Remove newline characters inside quoted strings
        content = re.sub(r'("\[.*?")\n', r'\1 ', content)
        temp_file = io.StringIO(content)
        reader = csv.reader(temp_file)
    
        try:
            header = next(reader)
            logging.debug(f"CSV Header: {header}")
        except StopIteration:
            logging.error("Input file is empty.")
            raise ValueError("Input CSV is empty.")
    
        for row_number, row in enumerate(reader, start=2):  # Starting at 2 to account for header
            total_row_count += 1
            if not row:
                logging.debug(f"Row {row_number} is empty.")
                empty_row_count += 1
                continue
    
            jpg_file = row[0].strip()
            keywords = extract_keywords(row[1] if len(row) > 1 else '', labels)
    
            na_in_keywords = keywords.count("N/A") + sum(1 for k in keywords if not k)
            na_count += na_in_keywords
    
            if not keywords or all(k.strip().upper() == 'N/A' or not k.strip() for k in keywords):
                empty_row_count += 1
                logging.debug(f"Row {row_number} is empty or contains only 'N/A' or empty keywords.")
                continue
    
            keywords_str = ', '.join(keyword for keyword in keywords if keyword)
            keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
    
            if ',' not in keywords_str:
                no_comma_count += 1
                no_comma_rows.append(f'{jpg_file},{keywords_str}')
                logging.warning(f"Row {row_number} lacks commas: {jpg_file}, {keywords_str}")
    
            cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
    
        df = pd.DataFrame(cleaned_data)
        df = clean_keywords_in_df(df, labels)
    
        logging.info(f"Total rows processed: {total_row_count}")
        logging.info(f"Number of empty or N/A rows: {empty_row_count}")
        logging.info(f"Occurrences of N/A (including empty spaces): {na_count}")
        logging.info(f"Number of rows without commas: {no_comma_count}")
        if no_comma_rows:
            logging.info("Example rows without commas:")
            for row in no_comma_rows[:5]:  # Limit to first 5 for brevity
                logging.info(row)
    
        logging.info("CSV cleaning completed.")
        return df
    
    def extract_keywords(line: str, labels: List[str]) -> List[str]:
        """
        Extracts keywords from a line.
    
        Args:
            line (str): Input line containing keywords.
            labels (List[str]): List of label categories to remove.
    
        Returns:
            List[str]: Extracted keywords.
        """
        # Extract keywords within curly braces
        keywords = re.findall(r'\{([^}]+)\}', line)
        if keywords:
            return [kw.strip() for group in keywords for kw in group.split(',') if kw.strip()]
        
        # Remove label patterns
        for label in labels:
            pattern = rf'\b{re.escape(label)}\b[: -]?'
            line = re.sub(pattern, '', line, flags=re.IGNORECASE)
        
        # Clean up the line
        line = re.sub(r"-\s", "", line)
        line = re.sub(r"'([^']+)'", r'\1', line)
        line = re.sub(r'[\[\]{}]', '', line)
        # Split by comma and strip whitespace
        keywords = [kw.strip() for kw in line.split(',') if kw.strip()]
        return keywords
    
    def clean_keywords_in_df(df: pd.DataFrame, labels: List[str]) -> pd.DataFrame:
        """
        Cleans the 'Keywords' column in the DataFrame.
    
        Args:
            df (pd.DataFrame): DataFrame with 'Keywords' column.
            labels (List[str]): List of label categories to remove.
    
        Returns:
            pd.DataFrame: Cleaned DataFrame.
        """
        labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*'
        df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True)
        df['Keywords'] = df['Keywords'].str.replace(r"-\s", "", regex=True)
        df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True)
        df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True)
        df['Keywords'] = df['Keywords'].str.strip(' ,')
        return df
    
    def lemmatize_keywords(df: pd.DataFrame, lemmatizer: WordNetLemmatizer, save_path: Optional[str]=None
                         ) -> Tuple[pd.DataFrame, Counter]:
        """
        Lemmatizes the keywords in the DataFrame.
    
        Args:
            df (pd.DataFrame): DataFrame with 'Keywords' column.
            lemmatizer (WordNetLemmatizer): NLTK lemmatizer.
            save_path (Optional[str]): Path to save lemmatized keyword counts.
    
        Returns:
            Tuple[pd.DataFrame, Counter]: DataFrame with lemmatized keywords and keyword counts.
        """
        logging.info("Starting lemmatization...")
        per_row_data = []
        keyword_counts = Counter()
    
        for _, row in df.iterrows():
            image_name = row['Image Name']
            keywords = [kw.lower() for kw in row['Keywords'].split(',') if kw]
            lemmatized = [lemmatizer.lemmatize(kw) for kw in keywords]
            keyword_counts.update(lemmatized)
            per_row_data.append({
                'Image Name': image_name,
                'Keywords': ', '.join(keywords),
                'Lemmatized Keywords': ', '.join(lemmatized)
            })
    
        lemmatized_df = pd.DataFrame(per_row_data)
    
        if save_path:
            lemmatized_counts = pd.DataFrame(word_count_to_dict(keyword_counts)).sort_values(by='Count', ascending=False)
            lemmatized_counts.to_csv(save_path, index=False)
            logging.info(f"Lemmatized keyword counts saved to '{save_path}'.")
    
        logging.info("Lemmatization completed.")
        return lemmatized_df, keyword_counts
    
    def word_count_to_dict(counter: Counter) -> List[Dict[str, int]]:
        """
        Converts a Counter object to a list of dictionaries.
    
        Args:
            counter (Counter): Counter object.
    
        Returns:
            List[Dict[str, int]]: List of dictionaries with keyword and count.
        """
        return [{'Lemmatized Keyword': kw, 'Count': cnt} for kw, cnt in counter.items()]
    
    def load_model(model_type: str, model_path: str):
        """
        Loads a word embedding model based on the specified type and path.
    
        Args:
            model_type (str): Type of model ('word2vec', 'glove', 'fasttext').
            model_path (str): Path to the model file.
    
        Returns:
            Model object or dict: Loaded model.
        """
        logging.info(f"Loading {model_type} model from '{model_path}'...")
        if model_type == 'word2vec':
            try:
                model = KeyedVectors.load_word2vec_format(model_path, binary=True)
                logging.info("Word2Vec model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load Word2Vec model: {e}")
                raise
        elif model_type == 'glove':
            try:
                model = {}
                with open(model_path, 'r', encoding='utf-8') as f:
                    for line in f:
                        values = line.strip().split()
                        word = values[0]
                        vector = np.array(values[1:], dtype='float32')
                        model[word] = vector
                logging.info("GloVe model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load GloVe model: {e}")
                raise
        elif model_type == 'fasttext':
            try:
                model = FastText.load_fasttext_format(model_path)
                logging.info("FastText model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load FastText model: {e}")
                raise
        else:
            msg = f"Unsupported model type: {model_type}"
            logging.error(msg)
            raise ValueError(msg)
        return model
    
    def get_word_vector(word: str, model, model_type: str) -> Optional[np.ndarray]:
        """
        Retrieves the vector representation of a word from the model.
    
        Args:
            word (str): The word to retrieve.
            model: The loaded word embedding model.
            model_type (str): The type of the model.
    
        Returns:
            Optional[np.ndarray]: Word vector or None if not found.
        """
        if model_type in ['word2vec', 'fasttext']:
            if word in model:
                return model[word]
        elif model_type == 'glove':
            return model.get(word)
        
        # Handle multi-word phrases by averaging their vectors
        words = word.split()
        vectors = [model[w] for w in words if w in model]
        if vectors:
            return np.mean(vectors, axis=0)
        return None
    
    def perform_clustering_with_threshold(keyword_counts: Counter, model, model_type: str,
                                         threshold: int, num_clusters: int,
                                         distance_threshold: Optional[float]) -> Dict[str, str]:
        """
        Clusters infrequent keywords using hierarchical clustering.
    
        Args:
            keyword_counts (Counter): Counts of lemmatized keywords.
            model: Loaded word embedding model.
            model_type (str): Type of the model.
            threshold (int): Frequency threshold.
            num_clusters (int): Number of clusters for infrequent keywords.
            distance_threshold (Optional[float]): Distance threshold for clustering.
    
        Returns:
            Dict[str, str]: Mapping from keyword to cluster representative.
        """
        logging.info("Starting clustering process...")
        frequent_keywords = [kw for kw, cnt in keyword_counts.items() if cnt > threshold]
        infrequent_keywords = [kw for kw, cnt in keyword_counts.items() if cnt <= threshold]
    
        logging.info(f"Frequent keywords (>{threshold}): {len(frequent_keywords)}")
        logging.info(f"Infrequent keywords (≤{threshold}): {len(infrequent_keywords)}")
    
        clustered = {kw: kw for kw in frequent_keywords}
    
        if infrequent_keywords:
            mapping = cluster_infrequent_keywords(
                infrequent_keywords, keyword_counts, model, model_type,
                num_clusters=num_clusters, distance_threshold=distance_threshold
            )
            clustered.update(mapping)
            logging.info(f"Clustering completed. Total representatives: {len(set(mapping.values()))}")
        else:
            logging.info("No infrequent keywords to cluster.")
    
        return clustered
    
    def cluster_infrequent_keywords(keywords: List[str], keyword_counts: Counter, model, model_type: str,
                                    num_clusters: int, distance_threshold: Optional[float]) -> Dict[str, str]:
        """
        Clusters infrequent keywords and maps them to cluster representatives.
    
        Args:
            keywords (List[str]): List of infrequent keywords.
            keyword_counts (Counter): Counts of lemmatized keywords.
            model: Loaded word embedding model.
            model_type (str): Type of the model.
            num_clusters (int): Number of clusters.
            distance_threshold (Optional[float]): Distance threshold for clustering.
    
        Returns:
            Dict[str, str]: Mapping from keyword to cluster representative.
        """
        logging.info("Clustering infrequent keywords...")
        vectors = []
        valid_keywords = []
    
        for kw in keywords:
            vec = get_word_vector(kw, model, model_type)
            if vec is not None:
                vectors.append(vec)
                valid_keywords.append(kw)
            else:
                logging.warning(f"No vector found for keyword: '{kw}'")
    
        if not vectors:
            logging.warning("No valid vectors found for infrequent keywords. Skipping clustering.")
            return {}
    
        vectors = np.array(vectors)
    
        try:
            if distance_threshold is not None:
                logging.info(f"Using distance_threshold={distance_threshold} for clustering.")
                clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold)
            else:
                logging.info(f"Using num_clusters={num_clusters} for clustering.")
                clustering_model = AgglomerativeClustering(n_clusters=num_clusters)
            labels = clustering_model.fit_predict(vectors)
        except Exception as e:
            logging.error(f"Clustering failed: {e}")
            return {}
    
        clusters = defaultdict(list)
        for label, kw in zip(labels, valid_keywords):
            clusters[label].append(kw)
    
        # Select the most frequent keyword as the representative
        representative_keywords = {}
        for label, kws in clusters.items():
            rep = max(kws, key=lambda k: keyword_counts[k])
            representative_keywords[label] = rep
    
        # Mapping from keyword to representative
        keyword_to_rep = {}
        for label, kws in clusters.items():
            rep = representative_keywords[label]
            for kw in kws:
                keyword_to_rep[kw] = rep
    
        return keyword_to_rep
    
    def categorize_and_count_clustered_keywords(per_row_df: pd.DataFrame, labels: List[str],
                                                category_limit: int) -> Dict[str, Counter]:
        """
        Categorizes clustered keywords and counts them per category.
    
        Args:
            per_row_df (pd.DataFrame): DataFrame with clustered keywords.
            labels (List[str]): List of label categories.
            category_limit (int): Max number of keywords per category.
    
        Returns:
            Dict[str, Counter]: Keyword counts per category.
        """
        logging.info("Starting categorization of clustered keywords...")
        category_keyword_counts = {label: Counter() for label in labels}
    
        for _, row in per_row_df.iterrows():
            clustered_keywords = [kw.strip() for kw in row.get('Clustered Keywords', '').split(',') if kw]
            for i, label in enumerate(labels):
                if i < len(clustered_keywords):
                    keyword = clustered_keywords[i]
                    category_keyword_counts[label][keyword] += 1
    
        # Limit to top N keywords per category
        for label, counter in category_keyword_counts.items():
            original_count = len(counter)
            category_keyword_counts[label] = Counter(dict(counter.most_common(category_limit)))
            logging.info(f"Category '{label}': Reduced from {original_count} to {len(category_keyword_counts[label])} keywords (top {category_limit}).")
    
        logging.info("Categorization completed.")
        return category_keyword_counts
    
    def generate_final_dataframe(per_row_df: pd.DataFrame, clustered_keyword_mapping: Dict[str, str],
                                category_keyword_counts: Dict[str, Counter], labels: List[str],
                                remove_duplicates: bool) -> pd.DataFrame:
        """
        Generates the final DataFrame with clustered keywords and category assignments.
    
        Args:
            per_row_df (pd.DataFrame): DataFrame with lemmatized and clustered keywords.
            clustered_keyword_mapping (Dict[str, str]): Mapping from keyword to representative.
            category_keyword_counts (Dict[str, Counter]): Keyword counts per category.
            labels (List[str]): List of label categories.
            remove_duplicates (bool): Flag to remove duplicate keywords per row.
    
        Returns:
            pd.DataFrame: Final processed DataFrame.
        """
        logging.info("Generating final DataFrame with categorized keywords...")
        top_keywords_per_category = {label: set(counter.keys()) for label, counter in category_keyword_counts.items()}
        category_keywords_list = {label: [] for label in labels}
        clustered_keywords_list = []
        other_count_rows = defaultdict(int)
    
        for _, row in per_row_df.iterrows():
            lemmatized_keywords = [kw.strip() for kw in row.get('Lemmatized Keywords', '').split(',') if kw]
            clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords]
            
            if remove_duplicates:
                clustered_keywords = list(dict.fromkeys(clustered_keywords))
            
            clustered_keywords_list.append(', '.join(clustered_keywords))
    
            row_other_count = 0
            for i, label in enumerate(labels):
                if i < len(clustered_keywords):
                    keyword = clustered_keywords[i]
                    if keyword not in top_keywords_per_category[label]:
                        keyword = 'Other'
                        row_other_count += 1
                    category_keywords_list[label].append(keyword)
                else:
                    category_keywords_list[label].append('')  # Or 'Other' based on requirements
            
            other_count_rows[row_other_count] += 1
    
        per_row_df['Clustered Keywords'] = clustered_keywords_list
    
        for label in labels:
            per_row_df[label] = category_keywords_list[label]
    
        # Logging 'Other' counts
        logging.info("Summary of 'Other' keyword assignments per row:")
        for count, num_rows in sorted(other_count_rows.items()):
            logging.info(f"Rows with {count} 'Other' keywords: {num_rows}")
    
        logging.info("Final DataFrame generation completed.")
        return per_row_df
    
    def encode_categorical_features(input_df: pd.DataFrame, output_tsv_path: str,
                                    id_column: str = 'Image Name', remove_extension: Optional[str] = '.jpg',
                                    labels: Optional[List[str]] = None, sep: str = '\t',
                                    header: bool = False, index: bool = False) -> None:
        """
        Encodes categorical features using One-Hot Encoding and saves them to a TSV file.
    
        Args:
            input_df (pd.DataFrame): Input DataFrame with categorical columns.
            output_tsv_path (str): Path to save the encoded TSV file.
            id_column (str, optional): Column to use as identifier. Defaults to 'Image Name'.
            remove_extension (Optional[str], optional): Extension to remove from ID column. Defaults to '.jpg'.
            labels (Optional[List[str]], optional): List of label columns to encode.
            sep (str, optional): Separator for TSV file. Defaults to tab.
            header (bool, optional): Whether to include header. Defaults to False.
            index (bool, optional): Whether to include index. Defaults to False.
        """
        if labels is None:
            logging.error("Labels for encoding must be provided.")
            raise ValueError("Labels for encoding must be provided.")
    
        logging.info("Starting one-hot encoding of categorical features...")
        try:
            df = input_df[[id_column] + labels].copy()
        except KeyError as e:
            logging.error(f"Missing column in DataFrame: {e}")
            raise
    
        enc = OneHotEncoder(handle_unknown='ignore', sparse=False)
        encoded_features = enc.fit_transform(df[labels])
        feature_names = enc.get_feature_names_out(labels)
        encoded_df = pd.DataFrame(encoded_features, columns=feature_names, dtype=int)
    
        # Concatenate ID column
        result_df = pd.concat([df[[id_column]].reset_index(drop=True), encoded_df.reset_index(drop=True)], axis=1)
    
        # Remove specified extension from ID column
        if remove_extension:
            result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False)
    
        # Save to TSV
        result_df.to_csv(output_tsv_path, sep=sep, index=index, header=header)
        logging.info(f"One-hot encoded features saved to '{output_tsv_path}'.")
    
    def perform_full_processing(args: argparse.Namespace) -> None:
        """
        Coordinates the full processing workflow based on parsed arguments.
    
        Args:
            args (argparse.Namespace): Parsed command-line arguments.
        """
        input_file = args.input_file
        save_lemmatized = args.save_lemmatized
        model_type = args.model_type
        model_path = args.model_path
        frequency_threshold = args.frequency_threshold
        num_clusters = args.num_clusters
        distance_threshold = args.distance_threshold
        category_limit = args.category_limit
        remove_duplicates = args.remove_duplicates
    
        logging.info(f"Processing input file: {input_file}")
    
        labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style']  # Modify as needed
    
        processed_dir = os.path.join(os.path.dirname(os.path.abspath(input_file)), 'processed')
        os.makedirs(processed_dir, exist_ok=True)
    
        output_file = os.path.join(processed_dir, f'clustered_{os.path.basename(input_file)}')
        one_hot_encoded_file = os.path.join(processed_dir, f'onehot_{os.path.splitext(os.path.basename(input_file))[0]}.tsv')
        lemmatized_counts_output_file = os.path.join(processed_dir, f'lemmatized_keyword_counts.csv') if save_lemmatized else None
    
        # Step 1: Cleaning
        df_clean = clean_csv(input_file, labels)
    
        # Step 2: Lemmatization
        lemmatizer = WordNetLemmatizer()
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatizer, lemmatized_counts_output_file)
    
        # Step 3: Load Model
        model = load_model(model_type, model_path)
    
        # Step 4: Clustering
        clustered_keyword_mapping = perform_clustering_with_threshold(
            keyword_counts, model, model_type,
            threshold=frequency_threshold, num_clusters=num_clusters,
            distance_threshold=distance_threshold
        )
    
        # Step 5: Assign Clustered Keywords
        per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
            lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
        )
    
        # Step 6: Categorization
        category_keyword_counts = categorize_and_count_clustered_keywords(
            per_row_df, labels=labels, category_limit=category_limit
        )
    
        # Step 7: Final Processing
        per_row_df = generate_final_dataframe(
            per_row_df, clustered_keyword_mapping, category_keyword_counts,
            labels=labels, remove_duplicates=remove_duplicates
        )
    
        # Optionally, save category counts
        if save_lemmatized:
            for category, counts in category_keyword_counts.items():
                category_counts_df = pd.DataFrame(word_count_to_dict(counts)).sort_values(by='Count', ascending=False)
                category_counts_output_file = os.path.join(processed_dir, f'{category}_keywords_counts.csv')
                category_counts_df.to_csv(category_counts_output_file, index=False)
                logging.info(f"{category} keyword counts saved to '{category_counts_output_file}'.")
    
        # Step 8: Save Results
        per_row_df.to_csv(output_file, index=False)
        logging.info(f"Final processed data saved to '{output_file}'.")
    
        # Step 9: One-Hot Encoding
        encode_categorical_features(
            per_row_df, one_hot_encoded_file,
            id_column='Image Name', remove_extension='.jpg',
            labels=labels,
            sep='\t', header=False, index=False
        )
    
    def main():
        """
        Main function to orchestrate the keyword processing workflow.
        """
        args = parse_arguments()
        setup_logging(args.input_file)
        try:
            download_nltk_data()
        except Exception as e:
            logging.error(f"Failed during NLTK data download: {e}")
            return
    
        try:
            perform_full_processing(args)
        except Exception as e:
            logging.error(f"An error occurred during processing: {e}")
    
    if __name__ == "__main__":
        main()
    

    Key Improvements Explained

    1. Function Documentation: Added docstrings to all functions to clearly explain their purpose, arguments, and return values. This enhances readability and maintainability.

    2. Logging Enhancements:

      • Initialization: The setup_logging function configures both file and console logging with appropriate log levels.
      • Detailed Logs: Added debug and info logs at crucial steps to trace the flow and capture important metrics. Warnings are logged for recoverable issues, and errors are logged before raising exceptions.
    3. Error Handling:

      • Wrapped critical operations in try-except blocks to catch and log exceptions without abruptly terminating the program.
      • Validates the presence of required columns before processing.
      • Provides meaningful error messages to facilitate debugging.
    4. Modularization:

      • Separated the clustering process into distinct functions: perform_clustering_with_threshold and cluster_infrequent_keywords for clarity.
      • Isolated the encoding process into encode_categorical_features for reusability.
    5. Performance Optimizations:

      • Used list comprehensions and generator expressions where appropriate for efficient data processing.
      • Limited the number of logged "rows without commas" to the first five entries to prevent log flooding.
    6. Code Cleanup:

      • Removed redundant or repetitive code segments.
      • Utilized meaningful variable names to enhance clarity.
      • Employed type hints for better code understanding and potential static analysis benefits.
    7. Type Hinting and Optional Parameters:

      • Incorporated type hints for function arguments and return types to improve code clarity and facilitate easier debugging.
      • Used Optional from the typing module for parameters that can accept None.
    8. Utility Functions:

      • Created a helper function word_count_to_dict to convert Counter objects to dictionaries when saving to CSV, promoting code reuse.
    9. Argument Validation:

      • Made the --input_file argument required to prevent the program from running without essential inputs.
      • Validates the presence of label columns during the encoding process.
    10. Optimized One-Hot Encoding:

      • Set sparse=False in OneHotEncoder to directly obtain a dense array, which is then converted to a DataFrame.
      • Handled encoding even if the DataFrame contains rows that were previously skipped during cleaning.

    Additional Recommendations

    1. Configuration Management:

      • Consider externalizing configurations (like default paths and thresholds) using a configuration file (e.g., YAML or JSON). This makes the application more flexible and easier to manage.
    2. Testing:

      • Implement unit tests for critical functions to ensure they work as expected. Libraries like unittest or pytest can be used for this purpose.
    3. Logging Levels Control:

      • Introduce command-line arguments to control the verbosity of logging (e.g., --debug to set log level to DEBUG).
    4. Parallel Processing:

      • For large datasets, consider using parallel processing (e.g., with multiprocessing or joblib) to speed up lemmatization and vector retrieval.
    5. Resource Management:

      • Monitor memory usage, especially when loading large models like GloVe or FastText. Implement mechanisms to handle memory constraints gracefully.
    6. Dependency Management:

      • Use a requirements.txt or Pipfile to specify dependencies, making it easier to set up the environment.

    By implementing these improvements, your code becomes more robust, maintainable, and easier to understand, facilitating future enhancements and collaborations.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.