Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

from pyspark.sql.functions import udf, col from pyspark.ml.feature import Tokenizer,…

  1. User

    from pyspark.sql.functions import udf, col from pyspark.ml.feature import Tokenizer, StopWordsRemover, HashingTF, IDF from pyspark.sql.types import DoubleType import numpy as np import re import logging

    class EnhancedMonteCarloTextCategorizerSpark: def init(self, categories_df, tfidf_num_features=500, n_simulations=1000, confidence_threshold=0.6): self.categories_df = categories_df # Espera-se que já seja um DataFrame do PySpark self.tfidf_num_features = tfidf_num_features self.n_simulations = n_simulations self.confidence_threshold = confidence_threshold

    @staticmethod
    def preprocess_text(text):
        """Pré-processa o texto para minúsculas e remove caracteres não desejados."""
        return ' '.join(re.findall(r'\w+', text.lower()))
    
    def create_tfidf_features(self, texts_df, input_col, output_col):
        """Cria features TF-IDF usando tokenização e remoção de stopwords."""
        tokenizer = Tokenizer(inputCol=input_col, outputCol="words")
        words_data = tokenizer.transform(texts_df)
        
        remover = StopWordsRemover(inputCol="words", outputCol="filtered")
        filtered_words_data = remover.transform(words_data)
        
        hashing_tf = HashingTF(inputCol="filtered", outputCol="rawFeatures", numFeatures=self.tfidf_num_features)
        featurized_data = hashing_tf.transform(filtered_words_data)
        
        idf = IDF(inputCol="rawFeatures", outputCol=output_col)
        idf_model = idf.fit(featurized_data)
        rescaled_data = idf_model.transform(featurized_data)
        
        return rescaled_data
    
    def categorize_texts(self, text_df):
        """Categoriza cada texto no DataFrame."""
        prepared_categories = self.create_tfidf_features(self.categories_df, "palavras_chave", "tfidfFeaturesCategory")
    
        text_tfidf_results = self.create_tfidf_features(text_df, "tx_trsc_segm", "tfidfFeaturesText")
        
        category_joined_df = text_tfidf_results.crossJoin(prepared_categories)
    
        def monte_carlo_udf(cat_vector, text_vector):
            """Calcula a similaridade TF-IDF entre texto e categorias."""
            similarities = float(text_vector.dot(cat_vector))
            return similarities  # Retorna a similaridade como um valor float
    
        similarity_udf = udf(monte_carlo_udf, DoubleType())
        
        labeled_df = category_joined_df.withColumn(
            "similarity", 
            similarity_udf(col("tfidfFeaturesCategory"), col("tfidfFeaturesText"))
        )
    
        from pyspark.sql.window import Window
        from pyspark.sql.functions import row_number
        
        window_spec = Window.partitionBy("tx_trsc_segm").orderBy(col("similarity").desc())
    
        ranked_df = labeled_df.withColumn("rank", row_number().over(window_spec))
        
        result_df = ranked_df.filter(col("rank") == 1).select("tx_trsc_segm", "categoria")
    
        return result_df
    

    Supõe-se que categories_df e sampled_df sejam DataFrames do PySpark em um ambiente que fornece uma SparkSession

    categorizer = EnhancedMonteCarloTextCategorizerSpark(categories_df) final_df = categorizer.categorize_texts(sampled_df)

    logging.info("Resultados finais:") final_df.show() use cach para tudo

  2. Assistant

    To improve the performance of the EnhancedMonteCarloTextCategorizerSpark class, we can cache intermediate DataFrames. Caching is particularly useful when the same DataFrame is used multiple times in a workflow, which helps minimize data recomputation or re-reading from disk. Here's how you can incorporate caching into your existing code:

    from pyspark.sql.functions import udf, col, row_number
    from pyspark.ml.feature import Tokenizer, StopWordsRemover, HashingTF, IDF
    from pyspark.sql.types import DoubleType
    from pyspark.sql.window import Window
    import numpy as np
    import re
    import logging
    
    class EnhancedMonteCarloTextCategorizerSpark:
        def __init__(self, categories_df, tfidf_num_features=500, n_simulations=1000, confidence_threshold=0.6):
            self.categories_df = categories_df  # Expect a PySpark DataFrame
            self.tfidf_num_features = tfidf_num_features
            self.n_simulations = n_simulations
            self.confidence_threshold = confidence_threshold
    
        @staticmethod
        def preprocess_text(text):
            """Preprocess text to lowercase and remove unwanted characters."""
            return ' '.join(re.findall(r'\w+', text.lower()))
    
        def create_tfidf_features(self, texts_df, input_col, output_col):
            """Create TF-IDF features using tokenization and stopword removal."""
            tokenizer = Tokenizer(inputCol=input_col, outputCol="words")
            words_data = tokenizer.transform(texts_df)
            
            remover = StopWordsRemover(inputCol="words", outputCol="filtered")
            filtered_words_data = remover.transform(words_data)
            
            hashing_tf = HashingTF(inputCol="filtered", outputCol="rawFeatures", numFeatures=self.tfidf_num_features)
            featurized_data = hashing_tf.transform(filtered_words_data)
            
            idf = IDF(inputCol="rawFeatures", outputCol=output_col)
            idf_model = idf.fit(featurized_data)
            rescaled_data = idf_model.transform(featurized_data)
            
            return rescaled_data
    
        def categorize_texts(self, text_df):
            """Categorize each text in the DataFrame."""
            prepared_categories = self.create_tfidf_features(self.categories_df, "palavras_chave", "tfidfFeaturesCategory")
            prepared_categories = prepared_categories.cache()  # Cache the categories DataFrame
    
            text_tfidf_results = self.create_tfidf_features(text_df, "tx_trsc_segm", "tfidfFeaturesText")
            text_tfidf_results = text_tfidf_results.cache()  # Cache the text DataFrame
            
            category_joined_df = text_tfidf_results.crossJoin(prepared_categories)
            category_joined_df = category_joined_df.cache()  # Cache the joined DataFrame
    
            def monte_carlo_udf(cat_vector, text_vector):
                """Calculate TF-IDF similarity between text and categories."""
                similarities = float(text_vector.dot(cat_vector))
                return similarities  # Return similarity as a float value
    
            similarity_udf = udf(monte_carlo_udf, DoubleType())
            
            labeled_df = category_joined_df.withColumn(
                "similarity", 
                similarity_udf(col("tfidfFeaturesCategory"), col("tfidfFeaturesText"))
            )
    
            window_spec = Window.partitionBy("tx_trsc_segm").orderBy(col("similarity").desc())
    
            ranked_df = labeled_df.withColumn("rank", row_number().over(window_spec))
            
            result_df = ranked_df.filter(col("rank") == 1).select("tx_trsc_segm", "categoria")
    
            return result_df
    
    # Assumes categories_df and sampled_df are PySpark DataFrames within an environment providing a SparkSession
    
    categorizer = EnhancedMonteCarloTextCategorizerSpark(categories_df)
    final_df = categorizer.categorize_texts(sampled_df)
    
    logging.info("Final Results:")
    final_df.show()
    

    Key Changes:

    1. Caching is applied using the .cache() method to intermediate DataFrames such as prepared_categories, text_tfidf_results, and category_joined_df. This ensures that these DataFrames are stored in memory, reducing the need to recompute or re-read data from disk.
    2. Placement of cache(): Cache is applied right after transformations that would benefit from being reused in subsequent operations, especially before operations that are computationally intensive or involve large joins.

    Important Considerations:

    • Resource Management: Ensure you have enough memory resources allocated if caching large DataFrames.
    • Performance Testing: Time the operations before and after caching to make sure improvements are significant.
    • Unpersist the DataFrames: If resources are a constraint and DataFrames won't be used further in the script, consider using .unpersist() after their final usage to free up resources.
  3. User

    Exibir as primeiras 30 linhas do DataFrame final_df sem truncar colunas

    final_df.display() exibir somente 5 linhas de maneira rapida

  4. Assistant

    To display only the first 5 rows of your DataFrame without truncating the columns in PySpark, you can use the show() method with specific arguments. Specifically, you want to set the number of rows to display and turn off column truncation by setting the truncate parameter to False. Here's how you can do it:

    # Display the first 5 rows of the DataFrame without truncating columns
    final_df.show(5, truncate=False)
    

    Explanation:

    • show(n): Displays the specified number of rows (n).
    • truncate=False: Ensures that column contents are not truncated, allowing you to see the full data in each column.

    This method will quickly show the first 5 entries of final_df with the complete data of each column, assuming the DataFrame isn't excessively large. This is a practical approach when you want to inspect or debug data without having to page through large datasets.