from pyspark.sql.functions import udf, col from pyspark.ml.feature import Tokenizer,…
- User
from pyspark.sql.functions import udf, col from pyspark.ml.feature import Tokenizer, StopWordsRemover, HashingTF, IDF from pyspark.sql.types import DoubleType import numpy as np import re import logging
class EnhancedMonteCarloTextCategorizerSpark: def init(self, categories_df, tfidf_num_features=500, n_simulations=1000, confidence_threshold=0.6): self.categories_df = categories_df # Espera-se que já seja um DataFrame do PySpark self.tfidf_num_features = tfidf_num_features self.n_simulations = n_simulations self.confidence_threshold = confidence_threshold
@staticmethod def preprocess_text(text): """Pré-processa o texto para minúsculas e remove caracteres não desejados.""" return ' '.join(re.findall(r'\w+', text.lower())) def create_tfidf_features(self, texts_df, input_col, output_col): """Cria features TF-IDF usando tokenização e remoção de stopwords.""" tokenizer = Tokenizer(inputCol=input_col, outputCol="words") words_data = tokenizer.transform(texts_df) remover = StopWordsRemover(inputCol="words", outputCol="filtered") filtered_words_data = remover.transform(words_data) hashing_tf = HashingTF(inputCol="filtered", outputCol="rawFeatures", numFeatures=self.tfidf_num_features) featurized_data = hashing_tf.transform(filtered_words_data) idf = IDF(inputCol="rawFeatures", outputCol=output_col) idf_model = idf.fit(featurized_data) rescaled_data = idf_model.transform(featurized_data) return rescaled_data def categorize_texts(self, text_df): """Categoriza cada texto no DataFrame.""" prepared_categories = self.create_tfidf_features(self.categories_df, "palavras_chave", "tfidfFeaturesCategory") text_tfidf_results = self.create_tfidf_features(text_df, "tx_trsc_segm", "tfidfFeaturesText") category_joined_df = text_tfidf_results.crossJoin(prepared_categories) def monte_carlo_udf(cat_vector, text_vector): """Calcula a similaridade TF-IDF entre texto e categorias.""" similarities = float(text_vector.dot(cat_vector)) return similarities # Retorna a similaridade como um valor float similarity_udf = udf(monte_carlo_udf, DoubleType()) labeled_df = category_joined_df.withColumn( "similarity", similarity_udf(col("tfidfFeaturesCategory"), col("tfidfFeaturesText")) ) from pyspark.sql.window import Window from pyspark.sql.functions import row_number window_spec = Window.partitionBy("tx_trsc_segm").orderBy(col("similarity").desc()) ranked_df = labeled_df.withColumn("rank", row_number().over(window_spec)) result_df = ranked_df.filter(col("rank") == 1).select("tx_trsc_segm", "categoria") return result_dfSupõe-se que categories_df e sampled_df sejam DataFrames do PySpark em um ambiente que fornece uma SparkSession
categorizer = EnhancedMonteCarloTextCategorizerSpark(categories_df) final_df = categorizer.categorize_texts(sampled_df)
logging.info("Resultados finais:") final_df.show() use cach para tudo
- Assistant
To improve the performance of the
EnhancedMonteCarloTextCategorizerSparkclass, we can cache intermediate DataFrames. Caching is particularly useful when the same DataFrame is used multiple times in a workflow, which helps minimize data recomputation or re-reading from disk. Here's how you can incorporate caching into your existing code:from pyspark.sql.functions import udf, col, row_number from pyspark.ml.feature import Tokenizer, StopWordsRemover, HashingTF, IDF from pyspark.sql.types import DoubleType from pyspark.sql.window import Window import numpy as np import re import logging class EnhancedMonteCarloTextCategorizerSpark: def __init__(self, categories_df, tfidf_num_features=500, n_simulations=1000, confidence_threshold=0.6): self.categories_df = categories_df # Expect a PySpark DataFrame self.tfidf_num_features = tfidf_num_features self.n_simulations = n_simulations self.confidence_threshold = confidence_threshold @staticmethod def preprocess_text(text): """Preprocess text to lowercase and remove unwanted characters.""" return ' '.join(re.findall(r'\w+', text.lower())) def create_tfidf_features(self, texts_df, input_col, output_col): """Create TF-IDF features using tokenization and stopword removal.""" tokenizer = Tokenizer(inputCol=input_col, outputCol="words") words_data = tokenizer.transform(texts_df) remover = StopWordsRemover(inputCol="words", outputCol="filtered") filtered_words_data = remover.transform(words_data) hashing_tf = HashingTF(inputCol="filtered", outputCol="rawFeatures", numFeatures=self.tfidf_num_features) featurized_data = hashing_tf.transform(filtered_words_data) idf = IDF(inputCol="rawFeatures", outputCol=output_col) idf_model = idf.fit(featurized_data) rescaled_data = idf_model.transform(featurized_data) return rescaled_data def categorize_texts(self, text_df): """Categorize each text in the DataFrame.""" prepared_categories = self.create_tfidf_features(self.categories_df, "palavras_chave", "tfidfFeaturesCategory") prepared_categories = prepared_categories.cache() # Cache the categories DataFrame text_tfidf_results = self.create_tfidf_features(text_df, "tx_trsc_segm", "tfidfFeaturesText") text_tfidf_results = text_tfidf_results.cache() # Cache the text DataFrame category_joined_df = text_tfidf_results.crossJoin(prepared_categories) category_joined_df = category_joined_df.cache() # Cache the joined DataFrame def monte_carlo_udf(cat_vector, text_vector): """Calculate TF-IDF similarity between text and categories.""" similarities = float(text_vector.dot(cat_vector)) return similarities # Return similarity as a float value similarity_udf = udf(monte_carlo_udf, DoubleType()) labeled_df = category_joined_df.withColumn( "similarity", similarity_udf(col("tfidfFeaturesCategory"), col("tfidfFeaturesText")) ) window_spec = Window.partitionBy("tx_trsc_segm").orderBy(col("similarity").desc()) ranked_df = labeled_df.withColumn("rank", row_number().over(window_spec)) result_df = ranked_df.filter(col("rank") == 1).select("tx_trsc_segm", "categoria") return result_df # Assumes categories_df and sampled_df are PySpark DataFrames within an environment providing a SparkSession categorizer = EnhancedMonteCarloTextCategorizerSpark(categories_df) final_df = categorizer.categorize_texts(sampled_df) logging.info("Final Results:") final_df.show()Key Changes:
- Caching is applied using the
.cache()method to intermediate DataFrames such asprepared_categories,text_tfidf_results, andcategory_joined_df. This ensures that these DataFrames are stored in memory, reducing the need to recompute or re-read data from disk. - Placement of
cache(): Cache is applied right after transformations that would benefit from being reused in subsequent operations, especially before operations that are computationally intensive or involve large joins.
Important Considerations:
- Resource Management: Ensure you have enough memory resources allocated if caching large DataFrames.
- Performance Testing: Time the operations before and after caching to make sure improvements are significant.
- Unpersist the DataFrames: If resources are a constraint and DataFrames won't be used further in the script, consider using
.unpersist()after their final usage to free up resources.
- Caching is applied using the
- User
Exibir as primeiras 30 linhas do DataFrame
final_dfsem truncar colunasfinal_df.display() exibir somente 5 linhas de maneira rapida
- Assistant
To display only the first 5 rows of your DataFrame without truncating the columns in PySpark, you can use the
show()method with specific arguments. Specifically, you want to set the number of rows to display and turn off column truncation by setting thetruncateparameter toFalse. Here's how you can do it:# Display the first 5 rows of the DataFrame without truncating columns final_df.show(5, truncate=False)Explanation:
show(n): Displays the specified number of rows (n).truncate=False: Ensures that column contents are not truncated, allowing you to see the full data in each column.
This method will quickly show the first 5 entries of
final_dfwith the complete data of each column, assuming the DataFrame isn't excessively large. This is a practical approach when you want to inspect or debug data without having to page through large datasets.