Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

This is my current code. i need you to make it more professional and remove unnecessary…

  1. User

    This is my current code. i need you to make it more professional and remove unnecessary steps. i need can you help me making it better? what could i do to improve it? add more logging info. moreover, if you find the best way to lemmatize, to cluster the keywords. please do implement it.

    just answer with the entire final corrected code

    import os import re import csv import argparse import logging import nltk import pandas as pd from collections import defaultdict, Counter from nltk.stem import WordNetLemmatizer from gensim.models import KeyedVectors, FastText from sklearn.cluster import AgglomerativeClustering import numpy as np from sklearn.preprocessing import OneHotEncoder from nltk.corpus import wordnet as wn from nltk.corpus import stopwords

    def download_nltk_data(): """ Ensures that required NLTK data packages are downloaded. """ nltk_data_packages = ['wordnet', 'stopwords'] for package in nltk_data_packages: try: nltk.data.find(f'corpora/{package}') except LookupError: nltk.download(package)

    def parse_arguments(): """ Parses command-line arguments. """ parser = argparse.ArgumentParser(description='Process and cluster keywords from a CSV file.') parser.add_argument('--input_file', type=str, help='Path to the input CSV file.') parser.add_argument('--save_lemmatized', action='store_true', help='Save lemmatized keywords and counts.') parser.add_argument('--model_type', type=str, choices=['word2vec', 'glove', 'fasttext'], default='glove', help='Type of model to use for clustering. Default is "glove".') parser.add_argument('--model_path', type=str, default='/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt', required=False, help='Path to the word embedding model file.') parser.add_argument('--frequency_threshold', type=int, default=10, help='Frequency threshold for keywords. Default is 10.') parser.add_argument('--num_clusters', type=int, default=50, help='Number of clusters for infrequent keywords. Default is 50.') parser.add_argument('--distance_threshold', type=float, default=None, help='Distance threshold for clustering. Overrides num_clusters if set.') parser.add_argument('--category_limit', type=int, default=100, help='Limit of top N keywords per category. Default is 100.') parser.add_argument('--remove_duplicates', action='store_true', help='Remove duplicates in clustered keywords per row.') return parser.parse_args()

    def main(): # Parse command-line arguments args = parse_arguments() input_file = args.input_file save_lemmatized = args.save_lemmatized model_type = args.model_type model_path = args.model_path frequency_threshold = args.frequency_threshold num_clusters = args.num_clusters distance_threshold = args.distance_threshold category_limit = args.category_limit remove_duplicates = args.remove_duplicates

    # Check if input file exists
    if not os.path.exists(input_file):
        raise FileNotFoundError(f"Input file '{input_file}' does not exist.")
    
    # Set up logging
    input_dir = os.path.dirname(os.path.abspath(input_file))
    input_filename = os.path.basename(input_file)
    input_name_no_ext = os.path.splitext(input_filename)[0]
    processed_dir = os.path.join(input_dir, 'processed')
    os.makedirs(processed_dir, exist_ok=True)
    log_file = os.path.join(processed_dir, f'log_{input_name_no_ext}.log')
    logging.basicConfig(filename=log_file, level=logging.INFO,
                        format='%(asctime)s %(levelname)s:%(message)s')
    # Also log to stdout
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.INFO)
    console_formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s')
    console_handler.setFormatter(console_formatter)
    logging.getLogger().addHandler(console_handler)
    
    logging.info(f"Processing input file: {input_file}")
    
    # Download NLTK data if necessary
    download_nltk_data()
    
    # Initialize the lemmatizer
    lemmatizer = WordNetLemmatizer()
    # Set of stopwords to filter out
    stop_words = set(stopwords.words('english'))
    
    # Labels (customizable)
    labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style']  # Modify as needed
    
    # Define output file paths
    output_file = os.path.join(processed_dir, f'clustered_{input_filename}')
    one_hot_encoded_file = os.path.join(processed_dir, f'onehot_{input_name_no_ext}.tsv')
    lemmatized_counts_output_file = os.path.join(processed_dir, f'lemmatized_keyword_counts.csv')
    
    # ------------------------------#
    # Step 1: Cleaning #
    # ------------------------------#
    logging.info("Starting CSV cleaning...")
    df_clean = clean_csv(input_file, labels)
    logging.info("CSV cleaning completed.")
    
    # ------------------------------#
    # Step 2: Lemmatization #
    # ------------------------------#
    logging.info("Starting lemmatization...")
    if save_lemmatized:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatized_counts_output_file, lemmatizer)
    else:
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, None, lemmatizer)
    # Check if per_row_df is None
    if per_row_df is None:
        logging.error("Error in lemmatization.")
        return
    
    # ------------------------------#
    # Step 3: Load Model #
    # ------------------------------#
    model = load_model(model_type, model_path)
    
    # ------------------------------#
    # Step 4: Clustering #
    # ------------------------------#
    clustered_keyword_mapping = perform_clustering_with_threshold(
        keyword_counts, model, model_type,
        threshold=frequency_threshold, num_clusters=num_clusters,
        distance_threshold=distance_threshold
    )
    
    # ------------------------------#
    # Step 5: Clustered Keywords #
    # ------------------------------#
    per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
        lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
    )
    
    # ------------------------------#
    # Step 6: Categorization #
    # ------------------------------#
    category_keyword_counts = categorize_and_count_clustered_keywords(
        per_row_df, labels=labels, category_limit=category_limit
    )
    
    # ------------------------------#
    # Step 7: Final Processing #
    # ------------------------------#
    per_row_df = generate_clustered_keywords_and_categories(
        per_row_df, clustered_keyword_mapping, category_keyword_counts, labels=labels,
        remove_duplicates=remove_duplicates
    )
    
    # Optionally, write category counts to CSV files
    if save_lemmatized:
        for category, counts in category_keyword_counts.items():
            category_counts_df = pd.DataFrame({'Keyword': list(counts.keys()), 'Count': list(counts.values())})
            category_counts_df = category_counts_df.sort_values(by='Count', ascending=False)
            category_counts_output_file = os.path.join(processed_dir, f'{category}_keywords_counts.csv')
            category_counts_df.to_csv(category_counts_output_file, index=False)
            logging.info(f"{category} keyword counts saved to '{category_counts_output_file}'.")
    
    # ------------------------------#
    # Step 8: Save Results #
    # ------------------------------#
    per_row_df.to_csv(output_file, index=False)
    logging.info(f"Process completed. Output saved to '{output_file}'.")
    
    # ------------------------------#
    # Step 9: One-Hot Encoding #
    # ------------------------------#
    encode_categorical_features(
        per_row_df, one_hot_encoded_file,
        id_column='Image Name', remove_extension='.jpg',
        labels=labels,
        sep='\t', header=False, index=False
    )
    logging.info(f"One-hot encoded features saved to '{one_hot_encoded_file}'.")
    

    def clean_csv(input_file, labels): """ Cleans the input CSV file and returns a DataFrame. """ total_row_count = 0 empty_row_count = 0 na_count = 0 no_comma_count = 0 no_comma_rows = [] cleaned_data = [] # Read the entire content of the file try: with open(input_file, 'r', newline='', encoding='utf-8') as infile: content = infile.read() except Exception as e: logging.error(f"Error reading input file: {e}") return None # Remove newline characters inside quoted strings content = re.sub(r'("[.*?")\n', r'\1 ', content) # Use StringIO to read the cleaned content as a CSV import io temp_file = io.StringIO(content) reader = csv.reader(temp_file)

    # Skip the first line (header)
    try:
        header = next(reader)
    except StopIteration:
        logging.error("Input file is empty.")
        return None
    
    # Process each row
    for row in reader:
        total_row_count += 1
        if not row:
            continue
        jpg_file = row[0]
        if len(row) < 2:
            logging.warning(f"Row {total_row_count} doesn't have enough columns: {row}")
            keywords = []
        else:
            keywords = extract_keywords(row[1], labels)
        # Count occurrences of "N/A" and empty spaces
        na_count += keywords.count("N/A")
        na_count += sum(1 for k in keywords if k.strip() == "")
        # Check if the row is empty or contains only "N/A" or empty spaces
        if not keywords or all(k.strip().upper() == 'N/A' or k.strip() == '' for k in keywords):
            empty_row_count += 1
            continue  # Skip this row
        # Concatenate the keywords into a single string
        keywords_str = ', '.join(keywords)
        # Replace multiple spaces with a comma and a single space
        keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
        # Check if the row has no commas
        if ',' not in keywords_str:
            no_comma_count += 1
            no_comma_rows.append(f'{jpg_file},{keywords_str}')
        # Add to cleaned data
        cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
    # Now, create a DataFrame from the cleaned_data
    df = pd.DataFrame(cleaned_data)
    # Clean the 'Keywords' column further
    df = clean_keywords_in_df(df, labels)
    # Log statistics
    logging.info(f"Total rows processed: {total_row_count}")
    logging.info(f"Number of empty or N/A rows: {empty_row_count}")
    logging.info(f"Occurrences of N/A (including empty spaces): {na_count}")
    logging.info(f"Number of rows without commas: {no_comma_count}")
    if no_comma_rows:
        logging.info("Rows without commas:")
        for row in no_comma_rows:
            logging.info(row)
    return df
    

    def extract_keywords(line, labels): """ Extracts keywords from a line. """ # First, try to extract keywords between curly braces keywords = re.findall(r'{([^}]+)}', line) if keywords: # Further split if multiple keywords are within braces return [kw.strip() for group in keywords for kw in group.split(',')] # Remove newline characters from the keywords section line = line.replace('\n', ' ') # Remove the labels using word boundaries and considering possible formats for label in labels: # Patterns like "Label:", "Label -", "[Label]:", etc. pattern = rf'\b{re.escape(label)}\b[: -]?' line = re.sub(pattern, '', line, flags=re.IGNORECASE) # Remove any '- ' (dash and space) using exact string replacement line = line.replace('- ', '') # Remove single quotes around keywords using regex line = re.sub(r"'([^']+)'", r'\1', line) # Remove any remaining brackets line = line.replace('[', '').replace(']', '').replace('{', '').replace('}', '') # Split keywords by commas and strip whitespace keywords = [keyword.strip() for keyword in line.split(',')] return keywords

    def clean_keywords_in_df(df, labels): """ Cleans the 'Keywords' column in the DataFrame. """ # Construct a regex pattern from the labels with word boundaries labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*' # Replace labels with an empty string using regex df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True) # Remove any '- ' (dash and space) df['Keywords'] = df['Keywords'].str.replace('- ', '', regex=False) # Remove single quotes around keywords df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True) # Replace multiple spaces with a comma and a single space df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True) # Remove leading/trailing commas and spaces df['Keywords'] = df['Keywords'].str.strip(' ,') return df

    def lemmatize_keywords(df, lemmatized_counts_output_file, lemmatizer): """ Lemmatizes the keywords in the DataFrame and returns a DataFrame with lemmatized keywords. Optionally saves the lemmatized keyword counts to a CSV file. """ per_row_data = [] keyword_counts = Counter() # Iterate through each row to extract and lemmatize keywords for index, row in df.iterrows(): image_name = row.get('Image Name', '') keywords_str = row.get('Keywords', '') if pd.isna(keywords_str): keywords_str = '' # Split keywords by comma and strip whitespace extracted_keywords = [kw.strip() for kw in keywords_str.split(',')] # Lemmatize keywords lemmatized_keywords = [lemmatizer.lemmatize(kw.lower()) for kw in extracted_keywords if kw] # Update global counts keyword_counts.update(lemmatized_keywords) # Store per-row data per_row_data.append({ 'Image Name': image_name, 'Keywords': ', '.join(extracted_keywords), 'Lemmatized Keywords': ', '.join(lemmatized_keywords) }) # Create a DataFrame from per_row_data per_row_df = pd.DataFrame(per_row_data) if lemmatized_counts_output_file: # Write the lemmatized keyword counts to CSV lemmatized_counts = [{'Lemmatized Keyword': kw, 'Count': count} for kw, count in keyword_counts.items()] lemmatized_counts_df = pd.DataFrame(lemmatized_counts) # Sort by Count in descending order lemmatized_counts_df = lemmatized_counts_df.sort_values(by='Count', ascending=False) # Write to CSV lemmatized_counts_df.to_csv(lemmatized_counts_output_file, index=False) logging.info(f"Lemmatized keyword counts saved to '{lemmatized_counts_output_file}'.") return per_row_df, keyword_counts

    def load_model(model_type, model_path): """ Loads a word embedding model based on the specified type and path. """ if model_type == 'word2vec': logging.info("Loading Word2Vec model. This may take a few minutes...") model = KeyedVectors.load_word2vec_format(model_path, binary=True) logging.info("Word2Vec model loaded successfully.") elif model_type == 'glove': logging.info("Loading GloVe model. This may take a few minutes...") model = {} with open(model_path, 'r', encoding='utf-8') as f: for line in f: values = line.split() word = values[0] vector = np.asarray(values[1:], dtype='float32') model[word] = vector logging.info("GloVe model loaded successfully.") elif model_type == 'fasttext': logging.info("Loading FastText model. This may take a few minutes...") model = FastText.load_fasttext_format(model_path) logging.info("FastText model loaded successfully.") else: raise ValueError(f"Unsupported model type: {model_type}") return model

    def get_word_vector(word, model, model_type): """ Retrieves the vector representation of a word from the model. """ if word in model: return model[word] # Handle multi-word phrases by averaging their word vectors words = word.split() vectors = [] for w in words: if w in model: vectors.append(model[w]) if vectors: return np.mean(vectors, axis=0) else: return None

    def perform_clustering_with_threshold(keyword_counts, model, model_type, threshold=5, num_clusters=5, distance_threshold=None): """ Clusters infrequent keywords using hierarchical clustering. """ # Separate frequent and infrequent keywords frequent_keywords = [kw for kw, count in keyword_counts.items() if count > threshold] infrequent_keywords = [kw for kw, count in keyword_counts.items() if count <= threshold] logging.info(f"Number of frequent keywords (> {threshold}): {len(frequent_keywords)}") logging.info(f"Number of infrequent keywords (<= {threshold}): {len(infrequent_keywords)}") # Keep frequent keywords as-is clustered = {kw: kw for kw in frequent_keywords} # Cluster infrequent keywords and map them to their representatives if infrequent_keywords: mapping = cluster_infrequent_keywords( infrequent_keywords, keyword_counts, model, model_type, num_clusters=num_clusters, distance_threshold=distance_threshold ) clustered.update(mapping) logging.info(f"Number of cluster representatives: {len(set(mapping.values()))}") else: logging.info("No infrequent keywords to cluster.") return clustered

    def cluster_infrequent_keywords(keywords, keyword_counts, model, model_type, num_clusters=5, distance_threshold=None): """ Clusters infrequent keywords and maps them to cluster representatives. """ vectors = [] valid_keywords = [] for keyword in keywords: vector = get_word_vector(keyword, model, model_type) if vector is not None: vectors.append(vector) valid_keywords.append(keyword) else: logging.warning(f"Vector not found for keyword: '{keyword}'") if not vectors: logging.warning("No valid vectors found for clustering.") return {} vectors = np.array(vectors) # Perform Hierarchical Clustering with distance threshold if distance_threshold is not None: logging.info(f"Performing Agglomerative Clustering with distance_threshold={distance_threshold}") clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold) else: logging.info(f"Performing Agglomerative Clustering with n_clusters={num_clusters}") clustering_model = AgglomerativeClustering(n_clusters=num_clusters) labels = clustering_model.fit_predict(vectors) # Organize keywords by clusters clusters = defaultdict(list) for label, keyword in zip(labels, valid_keywords): clusters[label].append(keyword) # Select the most frequent keyword in each cluster as the representative representative_keywords = {} for label, words in clusters.items(): most_frequent = max(words, key=lambda w: keyword_counts[w]) representative_keywords[label] = most_frequent # Create a mapping from keyword to representative keyword_to_representative = {} for label, words in clusters.items(): rep = representative_keywords[label] for word in words: keyword_to_representative[word] = rep return keyword_to_representative

    def categorize_and_count_clustered_keywords(per_row_df, labels, category_limit=50): """ Categorizes clustered keywords and counts them per category. """ category_keyword_counts = {label: Counter() for label in labels} for idx, row in per_row_df.iterrows(): clustered_keywords = row['Clustered Keywords'].split(', ') for i, label in enumerate(labels): if i < len(clustered_keywords): keyword = clustered_keywords[i] category_keyword_counts[label][keyword] += 1 # Limit to top N keywords per category for label in labels: original_count = len(category_keyword_counts[label]) category_keyword_counts[label] = Counter(dict(category_keyword_counts[label].most_common(category_limit))) logging.info(f"Category '{label}': Reduced from {original_count} to {len(category_keyword_counts[label])} keywords (top {category_limit})")

    return category_keyword_counts
    

    def generate_clustered_keywords_and_categories(per_row_df, clustered_keyword_mapping, category_keyword_counts, labels, remove_duplicates=False): """ Generates the final DataFrame with clustered keywords and category assignments. """ top_keywords_per_category = {label: set(counts.keys()) for label, counts in category_keyword_counts.items()} # Initialize lists for categories and counters for "Other" category_keywords_list = {label: [] for label in labels} clustered_keywords_list = [] # Initialize a counter for the number of rows with specific "Other" counts max_other_count = len(labels) other_count_rows = {i: 0 for i in range(max_other_count + 1)} # from 0 to len(labels) for idx, row in per_row_df.iterrows(): lemmatized_keywords = row['Lemmatized Keywords'].split(', ') clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords] if remove_duplicates: # Remove duplicates while preserving order clustered_keywords = list(dict.fromkeys(clustered_keywords)) clustered_keywords_list.append(', '.join(clustered_keywords)) # Initialize "Other" counter for each row row_other_count = 0 # Assign keywords to categories based on positions for i, label in enumerate(labels): if i < len(clustered_keywords): keyword = clustered_keywords[i] # Map to 'Other' if not in top keywords if keyword not in top_keywords_per_category[label]: keyword = 'Other' row_other_count += 1 # Increase the row counter for "Other" category_keywords_list[label].append(keyword) else: category_keywords_list[label].append('') # or 'Other' # Count the number of rows with the specific number of "Other" keywords if row_other_count <= max_other_count: other_count_rows[row_other_count] += 1 per_row_df['Clustered Keywords'] = clustered_keywords_list # Add the category columns to per_row_df for label in labels: per_row_df[label] = category_keywords_list[label] # Log summary of "Other" counts per category other_count = {label: 0 for label in labels} # Counter for "Other" per category logging.info("\nSummary of 'Other' counts per category:") for label in labels: other_count[label] = sum(1 for keyword in category_keywords_list[label] if keyword == 'Other') logging.info(f"{label}: {other_count[label]}") # Log the summary of rows having specific "Other" counts logging.info("\nSummary of 'Other' count rows:") for count in range(max_other_count + 1): logging.info(f"Rows with {count} 'Other' keywords: {other_count_rows[count]}") return per_row_df

    def encode_categorical_features(input_df, output_tsv_path, id_column='Image Name', remove_extension='.jpg', labels=None, sep='\t', header=False, index=False): """ Encodes categorical features using One-Hot Encoding and saves them to a TSV file. """ if labels is None: raise ValueError("Labels for encoding must be provided.") # Read only the necessary columns columns_to_encode = labels columns_to_read = [id_column] + columns_to_encode df = input_df[columns_to_read] # Initialize OneHotEncoder enc = OneHotEncoder(handle_unknown='ignore', dtype=np.int8) # Fit and transform the categorical columns emb = enc.fit_transform(df[columns_to_encode]) # Retrieve the names of the one-hot encoded features feature_names = enc.get_feature_names_out(columns_to_encode) # Create a DataFrame with the encoded features emb_df = pd.DataFrame(emb.toarray(), columns=feature_names, dtype=np.int8) # Concatenate the id_column with the one-hot encoded features result_df = pd.concat([df[[id_column]].reset_index(drop=True), emb_df.reset_index(drop=True)], axis=1) # Remove specified extension from id_column if remove_extension: result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False) # Save to TSV file result_df.to_csv(output_tsv_path, index=index, header=header, sep=sep)

    if name == "main": main()

  2. Assistant
    import os
    import re
    import csv
    import argparse
    import logging
    import nltk
    import pandas as pd
    from collections import defaultdict, Counter
    from nltk.stem import WordNetLemmatizer
    from gensim.models import KeyedVectors, FastText
    from gensim.scripts.glove2word2vec import glove2word2vec
    from sklearn.cluster import AgglomerativeClustering
    from sklearn.exceptions import NotFittedError
    import numpy as np
    from sklearn.preprocessing import OneHotEncoder
    from nltk.corpus import wordnet as wn
    from nltk.corpus import stopwords
    from typing import Optional, Tuple, Dict, List
    import time
    import io
    
    def download_nltk_data():
        """
        Ensures that required NLTK data packages are downloaded.
        """
        nltk_data_packages = ['wordnet', 'stopwords', 'averaged_perceptron_tagger']
        for package in nltk_data_packages:
            try:
                nltk.data.find(f'corpora/{package}')
            except LookupError:
                logging.info(f"Downloading NLTK package: {package}")
                nltk.download(package)
    
    def parse_arguments() -> argparse.Namespace:
        """
        Parses command-line arguments.
        """
        parser = argparse.ArgumentParser(description='Process and cluster keywords from a CSV file.')
        parser.add_argument('--input_file', type=str, required=True, help='Path to the input CSV file.')
        parser.add_argument('--save_lemmatized', action='store_true', help='Save lemmatized keywords and counts.')
        parser.add_argument('--model_type', type=str, choices=['word2vec', 'glove', 'fasttext'], default='glove',
                            help='Type of model to use for clustering. Default is "glove".')
        parser.add_argument('--model_path', type=str, default='/home/danilo/Projects/IR_MD/models/glove.6B.300d.word2vec.txt',
                            help='Path to the word embedding model file.')
        parser.add_argument('--frequency_threshold', type=int, default=10,
                            help='Frequency threshold for keywords. Default is 10.')
        parser.add_argument('--num_clusters', type=int, default=50,
                            help='Number of clusters for infrequent keywords. Default is 50.')
        parser.add_argument('--distance_threshold', type=float, default=None,
                            help='Distance threshold for clustering. Overrides num_clusters if set.')
        parser.add_argument('--category_limit', type=int, default=100,
                            help='Limit of top N keywords per category. Default is 100.')
        parser.add_argument('--remove_duplicates', action='store_true',
                            help='Remove duplicates in clustered keywords per row.')
        parser.add_argument('--log_level', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
                            default='INFO', help='Set the logging level. Default is INFO.')
        return parser.parse_args()
    
    def main():
        start_time = time.time()
        # Parse command-line arguments
        args = parse_arguments()
        input_file = args.input_file
        save_lemmatized = args.save_lemmatized
        model_type = args.model_type
        model_path = args.model_path
        frequency_threshold = args.frequency_threshold
        num_clusters = args.num_clusters
        distance_threshold = args.distance_threshold
        category_limit = args.category_limit
        remove_duplicates = args.remove_duplicates
        log_level = args.log_level.upper()
    
        # Set up logging
        if not os.path.exists(input_file):
            raise FileNotFoundError(f"Input file '{input_file}' does not exist.")
    
        input_dir = os.path.dirname(os.path.abspath(input_file))
        input_filename = os.path.basename(input_file)
        input_name_no_ext = os.path.splitext(input_filename)[0]
        processed_dir = os.path.join(input_dir, 'processed')
        os.makedirs(processed_dir, exist_ok=True)
        log_file = os.path.join(processed_dir, f'log_{input_name_no_ext}.log')
    
        # Configure logging
        logging.basicConfig(
            level=getattr(logging, log_level),
            format='%(asctime)s %(levelname)s:%(message)s',
            handlers=[
                logging.FileHandler(log_file, mode='w', encoding='utf-8'),
                logging.StreamHandler()
            ]
        )
    
        logging.info(f"Started processing for input file: {input_file}")
    
        # Download NLTK data if necessary
        download_nltk_data()
    
        # Initialize the lemmatizer
        lemmatizer = WordNetLemmatizer()
        # Set of stopwords to filter out
        stop_words = set(stopwords.words('english'))
    
        # Labels (customizable)
        labels = ['Type', 'Color', 'Wear Location', 'Material', 'Style']  # Modify as needed
    
        # Define output file paths
        output_file = os.path.join(processed_dir, f'clustered_{input_filename}')
        one_hot_encoded_file = os.path.join(processed_dir, f'onehot_{input_name_no_ext}.tsv')
        lemmatized_counts_output_file = os.path.join(processed_dir, f'lemmatized_keyword_counts.csv')
    
        # ------------------------------#
        # Step 1: Cleaning              #
        # ------------------------------#
        logging.info("Step 1: Starting CSV cleaning...")
        df_clean = clean_csv(input_file, labels)
        if df_clean is None:
            logging.error("CSV cleaning failed.")
            return
        logging.info(f"Step 1: CSV cleaning completed. Cleaned data contains {len(df_clean)} rows.")
    
        # ------------------------------#
        # Step 2: Lemmatization         #
        # ------------------------------#
        logging.info("Step 2: Starting lemmatization...")
        per_row_df, keyword_counts = lemmatize_keywords(df_clean, lemmatized_counts_output_file, lemmatizer)
        if per_row_df is None:
            logging.error("Lemmatization failed.")
            return
        logging.info(f"Step 2: Lemmatization completed. Total unique lemmatized keywords: {len(keyword_counts)}.")
    
        # ------------------------------#
        # Step 3: Load Model            #
        # ------------------------------#
        logging.info("Step 3: Loading word embedding model...")
        model = load_model(model_type, model_path)
        logging.info("Step 3: Word embedding model loaded successfully.")
    
        # ------------------------------#
        # Step 4: Clustering            #
        # ------------------------------#
        logging.info("Step 4: Starting clustering of keywords...")
        clustered_keyword_mapping = perform_clustering_with_threshold(
            keyword_counts, model, model_type,
            threshold=frequency_threshold, num_clusters=num_clusters,
            distance_threshold=distance_threshold
        )
        logging.info(f"Step 4: Clustering completed. Total clustered keywords: {len(clustered_keyword_mapping)}.")
    
        # ------------------------------#
        # Step 5: Clustered Keywords    #
        # ------------------------------#
        logging.info("Step 5: Mapping clustered keywords to rows...")
        per_row_df['Clustered Keywords'] = per_row_df['Lemmatized Keywords'].apply(
            lambda kws: ', '.join([clustered_keyword_mapping.get(kw, kw) for kw in kws.split(', ')])
        )
    
        # ------------------------------#
        # Step 6: Categorization        #
        # ------------------------------#
        logging.info("Step 6: Starting categorization of clustered keywords...")
        category_keyword_counts = categorize_and_count_clustered_keywords(
            per_row_df, labels=labels, category_limit=category_limit
        )
        logging.info("Step 6: Categorization completed.")
    
        # ------------------------------#
        # Step 7: Final Processing      #
        # ------------------------------#
        logging.info("Step 7: Generating final clustered keywords and category assignments...")
        per_row_df = generate_clustered_keywords_and_categories(
            per_row_df, clustered_keyword_mapping, category_keyword_counts, labels=labels,
            remove_duplicates=remove_duplicates
        )
        logging.info("Step 7: Final processing completed.")
    
        # Optionally, write category counts to CSV files
        if save_lemmatized:
            logging.info("Saving lemmatized keyword counts...")
            for category, counts in category_keyword_counts.items():
                category_counts_df = pd.DataFrame({'Keyword': list(counts.keys()), 'Count': list(counts.values())})
                category_counts_df = category_counts_df.sort_values(by='Count', ascending=False)
                category_counts_output_file = os.path.join(processed_dir, f'{category}_keywords_counts.csv')
                category_counts_df.to_csv(category_counts_output_file, index=False)
                logging.info(f"Lemmatized keyword counts for category '{category}' saved to '{category_counts_output_file}'.")
    
        # ------------------------------#
        # Step 8: Save Results          #
        # ------------------------------#
        logging.info(f"Step 8: Saving clustered keywords to '{output_file}'...")
        per_row_df.to_csv(output_file, index=False)
        logging.info(f"Step 8: Clustered keywords saved successfully.")
    
        # ------------------------------#
        # Step 9: One-Hot Encoding      #
        # ------------------------------#
        logging.info("Step 9: Starting one-hot encoding of categorical features...")
        encode_categorical_features(
            per_row_df, one_hot_encoded_file,
            id_column='Image Name', remove_extension='.jpg',
            labels=labels,
            sep='\t', header=False, index=False
        )
        logging.info(f"Step 9: One-hot encoded features saved to '{one_hot_encoded_file}'.")
    
        end_time = time.time()
        elapsed_time = end_time - start_time
        logging.info(f"Processing completed in {elapsed_time:.2f} seconds.")
    
    def clean_csv(input_file: str, labels: List[str]) -> Optional[pd.DataFrame]:
        """
        Cleans the input CSV file and returns a DataFrame.
        """
        total_row_count = 0
        empty_row_count = 0
        na_count = 0
        no_comma_count = 0
        no_comma_rows = []
        cleaned_data = []
        start_time = time.time()
    
        try:
            with open(input_file, 'r', newline='', encoding='utf-8') as infile:
                content = infile.read()
        except Exception as e:
            logging.error(f"Error reading input file: {e}")
            return None
    
        # Remove newline characters inside quoted strings
        content = re.sub(r'("\[.*?")\n', r'\1 ', content)
        temp_file = io.StringIO(content)
        reader = csv.reader(temp_file)
    
        # Skip the header
        try:
            header = next(reader)
        except StopIteration:
            logging.error("Input file is empty.")
            return None
    
        for row in reader:
            total_row_count += 1
            if not row:
                continue
            jpg_file = row[0]
            if len(row) < 2:
                logging.warning(f"Row {total_row_count} doesn't have enough columns: {row}")
                keywords = []
            else:
                keywords = extract_keywords(row[1], labels)
            # Count occurrences of "N/A" and empty spaces
            na_count += keywords.count("N/A")
            na_count += sum(1 for k in keywords if k.strip() == "")
            # Check if the row is empty or contains only "N/A" or empty spaces
            if not keywords or all(k.strip().upper() == 'N/A' or k.strip() == '' for k in keywords):
                empty_row_count += 1
                continue  # Skip this row
            # Concatenate the keywords into a single string
            keywords_str = ', '.join(keywords)
            # Replace multiple spaces with a comma and a single space
            keywords_str = re.sub(r'\s{2,}', ', ', keywords_str)
            # Check if the row has no commas
            if ',' not in keywords_str:
                no_comma_count += 1
                no_comma_rows.append(f'{jpg_file},{keywords_str}')
            # Add to cleaned data
            cleaned_data.append({'Image Name': jpg_file, 'Keywords': keywords_str})
    
        df = pd.DataFrame(cleaned_data)
        df = clean_keywords_in_df(df, labels)
    
        end_time = time.time()
        elapsed_time = end_time - start_time
    
        # Log statistics
        logging.info(f"CSV Cleaning Statistics:")
        logging.info(f"  Total rows processed: {total_row_count}")
        logging.info(f"  Number of empty or N/A rows: {empty_row_count}")
        logging.info(f"  Total occurrences of N/A (including empty spaces): {na_count}")
        logging.info(f"  Number of rows without commas: {no_comma_count}")
        if no_comma_rows:
            logging.info("Rows without commas:")
            for row in no_comma_rows:
                logging.debug(row)
        logging.info(f"  Time taken for CSV cleaning: {elapsed_time:.2f} seconds.")
    
        return df
    
    def extract_keywords(line: str, labels: List[str]) -> List[str]:
        """
        Extracts keywords from a line.
        """
        # Extract keywords within curly braces
        keywords = re.findall(r'\{([^}]+)\}', line)
        if keywords:
            return [kw.strip() for group in keywords for kw in group.split(',')]
        # Remove newline characters
        line = line.replace('\n', ' ')
        # Remove labels
        for label in labels:
            pattern = rf'\b{re.escape(label)}\b[: -]?'
            line = re.sub(pattern, '', line, flags=re.IGNORECASE)
        # Remove specific patterns
        line = line.replace('- ', '')
        line = re.sub(r"'([^']+)'", r'\1', line)
        # Remove remaining brackets
        line = re.sub(r'[\[\]\{\}]', '', line)
        # Split by commas
        keywords = [kw.strip() for kw in line.split(',')]
        return keywords
    
    def clean_keywords_in_df(df: pd.DataFrame, labels: List[str]) -> pd.DataFrame:
        """
        Cleans the 'Keywords' column in the DataFrame.
        """
        # Remove labels using regex
        labels_pattern = r'\b(' + '|'.join(map(re.escape, labels)) + r')\b\s*[:\-]?'
        df['Keywords'] = df['Keywords'].str.replace(labels_pattern, '', regex=True, flags=re.IGNORECASE)
        # Remove specific patterns
        df['Keywords'] = df['Keywords'].str.replace(r'-\s', '', regex=True)
        df['Keywords'] = df['Keywords'].str.replace(r"'([^']+)'", r'\1', regex=True)
        # Replace multiple spaces with a comma and space
        df['Keywords'] = df['Keywords'].str.replace(r'\s{2,}', ', ', regex=True)
        # Strip leading/trailing commas and spaces
        df['Keywords'] = df['Keywords'].str.strip(' ,')
        return df
    
    def lemmatize_keywords(df: pd.DataFrame, lemmatized_counts_output_file: Optional[str],
                          lemmatizer: WordNetLemmatizer) -> Tuple[pd.DataFrame, Counter]:
        """
        Lemmatizes the keywords in the DataFrame and returns a DataFrame with lemmatized keywords.
        Optionally saves the lemmatized keyword counts to a CSV file.
        """
        per_row_data = []
        keyword_counts = Counter()
        start_time = time.time()
    
        for index, row in df.iterrows():
            image_name = row.get('Image Name', '')
            keywords_str = row.get('Keywords', '')
            if pd.isna(keywords_str):
                keywords_str = ''
            # Split keywords by comma and strip whitespace
            extracted_keywords = [kw.strip() for kw in keywords_str.split(',')]
            # POS tagging for better lemmatization
            pos_tags = nltk.pos_tag(extracted_keywords)
            # Lemmatize with appropriate POS
            lemmatized_keywords = [
                lemmatizer.lemmatize(kw.lower(), pos=get_wordnet_pos(tag)) 
                for kw, tag in pos_tags 
                if kw and kw.lower() not in stopwords.words('english')
            ]
            # Update global counts
            keyword_counts.update(lemmatized_keywords)
            # Store per-row data
            per_row_data.append({
                'Image Name': image_name,
                'Keywords': ', '.join(extracted_keywords),
                'Lemmatized Keywords': ', '.join(lemmatized_keywords)
            })
    
        per_row_df = pd.DataFrame(per_row_data)
    
        if lemmatized_counts_output_file:
            # Write the lemmatized keyword counts to CSV
            lemmatized_counts_df = pd.DataFrame({
                'Lemmatized Keyword': list(keyword_counts.keys()),
                'Count': list(keyword_counts.values())
            }).sort_values(by='Count', ascending=False)
            per_row_df.to_csv(lemmatized_counts_output_file, index=False)
            logging.info(f"Lemmatized keyword counts saved to '{lemmatized_counts_output_file}'.")
    
        end_time = time.time()
        elapsed_time = end_time - start_time
        logging.info(f"Lemmatization completed in {elapsed_time:.2f} seconds.")
    
        return per_row_df, keyword_counts
    
    def get_wordnet_pos(nltk_pos_tag: str) -> str:
        """
        Maps NLTK POS tags to WordNet POS tags.
        """
        if nltk_pos_tag.startswith('J'):
            return wn.ADJ
        elif nltk_pos_tag.startswith('V'):
            return wn.VERB
        elif nltk_pos_tag.startswith('N'):
            return wn.NOUN
        elif nltk_pos_tag.startswith('R'):
            return wn.ADV
        else:
            return wn.NOUN
    
    def load_model(model_type: str, model_path: str) -> KeyedVectors:
        """
        Loads a word embedding model based on the specified type and path.
        """
        if model_type == 'word2vec':
            logging.info("Loading Word2Vec model. This may take a few minutes...")
            try:
                model = KeyedVectors.load_word2vec_format(model_path, binary=True)
                logging.info("Word2Vec model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load Word2Vec model: {e}")
                raise
        elif model_type == 'glove':
            logging.info("Loading GloVe model. This may take a few minutes...")
            try:
                # Convert GloVe to word2vec format if not already converted
                word2vec_output_file = model_path + '.word2vec'
                if not os.path.isfile(word2vec_output_file):
                    logging.info("Converting GloVe format to word2vec format...")
                    glove2word2vec(model_path, word2vec_output_file)
                    logging.info("Conversion completed.")
                model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)
                logging.info("GloVe model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load GloVe model: {e}")
                raise
        elif model_type == 'fasttext':
            logging.info("Loading FastText model. This may take a few minutes...")
            try:
                model = FastText.load(model_path)
                logging.info("FastText model loaded successfully.")
            except Exception as e:
                logging.error(f"Failed to load FastText model: {e}")
                raise
        else:
            raise ValueError(f"Unsupported model type: {model_type}")
        return model
    
    def get_word_vector(word: str, model: KeyedVectors, model_type: str) -> Optional[np.ndarray]:
        """
        Retrieves the vector representation of a word from the model.
        """
        try:
            return model[word]
        except KeyError:
            # Handle multi-word phrases by averaging their word vectors
            words = word.split()
            vectors = []
            for w in words:
                if w in model:
                    vectors.append(model[w])
            if vectors:
                return np.mean(vectors, axis=0)
            else:
                logging.debug(f"Vector not found for keyword: '{word}'")
                return None
    
    def perform_clustering_with_threshold(keyword_counts: Counter, model: KeyedVectors, model_type: str,
                                         threshold: int = 5, num_clusters: int = 5,
                                         distance_threshold: Optional[float] = None) -> Dict[str, str]:
        """
        Clusters infrequent keywords using hierarchical clustering.
        """
        frequent_keywords = [kw for kw, count in keyword_counts.items() if count > threshold]
        infrequent_keywords = [kw for kw, count in keyword_counts.items() if count <= threshold]
        logging.info(f"Number of frequent keywords (> {threshold}): {len(frequent_keywords)}")
        logging.info(f"Number of infrequent keywords (<= {threshold}): {len(infrequent_keywords)}")
    
        # Keep frequent keywords as-is
        clustered = {kw: kw for kw in frequent_keywords}
    
        if infrequent_keywords:
            mapping = cluster_infrequent_keywords(
                infrequent_keywords, keyword_counts, model, model_type,
                num_clusters=num_clusters, distance_threshold=distance_threshold
            )
            clustered.update(mapping)
            logging.info(f"Number of cluster representatives: {len(set(mapping.values()))}")
        else:
            logging.info("No infrequent keywords to cluster.")
    
        return clustered
    
    def cluster_infrequent_keywords(keywords: List[str], keyword_counts: Counter,
                                    model: KeyedVectors, model_type: str,
                                    num_clusters: int = 5, distance_threshold: Optional[float] = None) -> Dict[str, str]:
        """
        Clusters infrequent keywords and maps them to cluster representatives.
        """
        vectors = []
        valid_keywords = []
        for keyword in keywords:
            vector = get_word_vector(keyword, model, model_type)
            if vector is not None:
                vectors.append(vector)
                valid_keywords.append(keyword)
            else:
                logging.warning(f"Vector not found for keyword: '{keyword}'")
    
        if not vectors:
            logging.warning("No valid vectors found for clustering.")
            return {}
    
        vectors = np.array(vectors)
    
        # Perform Hierarchical Clustering
        if distance_threshold is not None:
            logging.info(f"Performing Agglomerative Clustering with distance_threshold={distance_threshold}")
            clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold)
        else:
            logging.info(f"Performing Agglomerative Clustering with n_clusters={num_clusters}")
            clustering_model = AgglomerativeClustering(n_clusters=num_clusters)
    
        try:
            labels = clustering_model.fit_predict(vectors)
        except Exception as e:
            logging.error(f"Clustering failed: {e}")
            return {}
    
        # Organize keywords by clusters
        clusters = defaultdict(list)
        for label, keyword in zip(labels, valid_keywords):
            clusters[label].append(keyword)
    
        # Select the most frequent keyword in each cluster as the representative
        representative_keywords = {}
        for label, words in clusters.items():
            most_frequent = max(words, key=lambda w: keyword_counts[w])
            representative_keywords[label] = most_frequent
    
        # Create a mapping from keyword to representative
        keyword_to_representative = {}
        for label, words in clusters.items():
            rep = representative_keywords[label]
            for word in words:
                keyword_to_representative[word] = rep
    
        return keyword_to_representative
    
    def categorize_and_count_clustered_keywords(per_row_df: pd.DataFrame, labels: List[str],
                                               category_limit: int = 50) -> Dict[str, Counter]:
        """
        Categorizes clustered keywords and counts them per category.
        """
        category_keyword_counts = {label: Counter() for label in labels}
    
        for _, row in per_row_df.iterrows():
            clustered_keywords = row['Clustered Keywords'].split(', ')
            for i, label in enumerate(labels):
                if i < len(clustered_keywords):
                    keyword = clustered_keywords[i]
                    category_keyword_counts[label][keyword] += 1
    
        # Limit to top N keywords per category
        for label in labels:
            original_count = len(category_keyword_counts[label])
            category_keyword_counts[label] = Counter(dict(category_keyword_counts[label].most_common(category_limit)))
            logging.info(f"Category '{label}': Reduced from {original_count} to {len(category_keyword_counts[label])} keywords (top {category_limit})")
    
        return category_keyword_counts
    
    def generate_clustered_keywords_and_categories(per_row_df: pd.DataFrame, clustered_keyword_mapping: Dict[str, str],
                                                    category_keyword_counts: Dict[str, Counter], labels: List[str],
                                                    remove_duplicates: bool = False) -> pd.DataFrame:
        """
        Generates the final DataFrame with clustered keywords and category assignments.
        """
        top_keywords_per_category = {label: set(counts.keys()) for label, counts in category_keyword_counts.items()}
        category_keywords_list = {label: [] for label in labels}
        clustered_keywords_list = []
        other_count_rows = {i: 0 for i in range(len(labels) + 1)}  # from 0 to len(labels)
    
        for _, row in per_row_df.iterrows():
            lemmatized_keywords = row['Lemmatized Keywords'].split(', ')
            clustered_keywords = [clustered_keyword_mapping.get(kw, kw) for kw in lemmatized_keywords]
            if remove_duplicates:
                clustered_keywords = list(dict.fromkeys(clustered_keywords))
            clustered_keywords_list.append(', '.join(clustered_keywords))
    
            row_other_count = 0
            for i, label in enumerate(labels):
                if i < len(clustered_keywords):
                    keyword = clustered_keywords[i]
                    if keyword not in top_keywords_per_category[label]:
                        keyword = 'Other'
                        row_other_count += 1
                    category_keywords_list[label].append(keyword)
                else:
                    category_keywords_list[label].append('')  # or 'Other'
    
            if row_other_count <= len(labels):
                other_count_rows[row_other_count] += 1
    
        per_row_df['Clustered Keywords'] = clustered_keywords_list
    
        # Add category columns
        for label in labels:
            per_row_df[label] = category_keywords_list[label]
    
        # Log summary of "Other" counts per category
        logging.info("\nSummary of 'Other' counts per category:")
        for label in labels:
            other_count = sum(1 for kw in category_keywords_list[label] if kw == 'Other')
            logging.info(f"  {label}: {other_count}")
    
        # Log the summary of rows having specific "Other" counts
        logging.info("\nSummary of 'Other' count rows:")
        for count, num_rows in other_count_rows.items():
            logging.info(f"  Rows with {count} 'Other' keywords: {num_rows}")
    
        return per_row_df
    
    def encode_categorical_features(input_df: pd.DataFrame, output_tsv_path: str, id_column: str = 'Image Name',
                                    remove_extension: Optional[str] = '.jpg', labels: Optional[List[str]] = None,
                                    sep: str = '\t', header: bool = False, index: bool = False):
        """
        Encodes categorical features using One-Hot Encoding and saves them to a TSV file.
        """
        if labels is None:
            raise ValueError("Labels for encoding must be provided.")
    
        columns_to_encode = labels
        columns_to_read = [id_column] + columns_to_encode
    
        df = input_df[columns_to_read].copy()
    
        # Initialize OneHotEncoder
        enc = OneHotEncoder(handle_unknown='ignore', sparse=False, dtype=np.int8)
        enc.fit(df[columns_to_encode])
        emb = enc.transform(df[columns_to_encode])
        feature_names = enc.get_feature_names_out(columns_to_encode)
        emb_df = pd.DataFrame(emb, columns=feature_names, dtype=np.int8)
    
        # Concatenate the id_column with the one-hot encoded features
        result_df = pd.concat([df[[id_column]].reset_index(drop=True), emb_df.reset_index(drop=True)], axis=1)
    
        # Remove specified extension from id_column
        if remove_extension:
            result_df[id_column] = result_df[id_column].str.replace(remove_extension, '', regex=False)
    
        # Save to TSV file
        result_df.to_csv(output_tsv_path, index=index, header=header, sep=sep)
        logging.info(f"One-hot encoded features saved to '{output_tsv_path}'.")
    
    if __name__ == "__main__":
        main()