from IPython.display import clearoutput def converttoenglishfloat(value: str): return…
- User
from IPython.display import clear_output
def convert_to_english_float(value: str): return float(value.split(".")[0].replace(".", "").replace(",", ".").strip())
codigos_processos = df_gerais_principal["Código Processo"].to_list()
criar coluna dos valores lá
df_gerais_principal['valor_total_menos_transferidos'] = 0.0 for i, codigo_processo in enumerate(codigos_processos): print("-" * 8) print(codigo_processo) rows_com_valor_transferido = df_movimentos_orgao[ (df_movimentos_orgao["Código Processo"].astype(str) == str(codigo_processo)) & (df_movimentos_orgao["Descrição"].str.contains("valor transferido")) ] valor_total_dos_transferidos = sum( [ convert_to_english_float(value.split(": r$")[1]) for value in list( rows_com_valor_transferido.to_dict()["Descrição"].values() ) ] ) row_codigo_atual = df_gerais_principal[ df_gerais_principal["Código Processo"].astype(str) == str(codigo_processo) ].iloc[0] valor_total_codigo_atual = row_codigo_atual['Valor'] valor_total = convert_to_english_float(valor_total_codigo_atual) valor_total_menos_transferido = valor_total - valor_total_dos_transferidos valor_total_menos_transferido print( f"{valor_total} - {valor_total_dos_transferidos} = {valor_total_menos_transferido}" ) df_gerais_principal.loc[df_gerais_principal['Código Processo'].astype(str) == str(codigo_processo), 'valor_total_menos_transferidos'] = valor_total_menos_transferido print(df_gerais_principal[df_gerais_principal["Código Processo"].astype(str) == str(codigo_processo)])
how i make this but much faster using pandas native things
Assistant
Optimizing Your Code with Pandas Native Operations
Here's how you can make your code much faster by leveraging pandas' vectorized operations and avoiding row-by-row iteration:
def convert_to_english_float(value: str): return value.str.split(".").str[0].str.replace(".", "").str.replace(",", ".").str.strip().astype(float) # Preprocess the data df_gerais_principal['Código Processo'] = df_gerais_principal['Código Processo'].astype(str) df_movimentos_orgao['Código Processo'] = df_movimentos_orgao['Código Processo'].astype(str) # Filter rows with "valor transferido" and extract the values transfer_rows = df_movimentos_orgao[df_movimentos_orgao['Descrição'].str.contains('valor transferido', na=False)] transfer_rows['transfer_value'] = transfer_rows['Descrição'].str.extract(r': r\$([\d\.,]+)')[0] transfer_rows['transfer_value'] = convert_to_english_float(transfer_rows['transfer_value']) # Group by process code and sum the transfer values transfer_sums = transfer_rows.groupby('Código Processo')['transfer_value'].sum().reset_index() transfer_sums.columns = ['Código Processo', 'total_transferred'] # Convert main values to float df_gerais_principal['valor_total'] = convert_to_english_float(df_gerais_principal['Valor']) # Merge and calculate df_gerais_principal = df_gerais_principal.merge( transfer_sums, on='Código Processo', how='left' ).fillna({'total_transferred': 0}) df_gerais_principal['valor_total_menos_transferidos'] = ( df_gerais_principal['valor_total'] - df_gerais_principal['total_transferred'] ) # Drop intermediate columns if needed df_gerais_principal = df_gerais_principal.drop(['total_transferred'], axis=1)Key Optimizations:
- Vectorized operations: Replaced row-by-row iteration with pandas' built-in string operations
- Groupby aggregation: Used
groupby()to sum transfer values by process code - Merge operation: Joined the data in one operation instead of looking up each code
- Regex extraction: Used
str.extract()to get transfer values in one step - Bulk conversion: Converted all values at once instead of one at a time
This approach should be significantly faster, especially for large datasets, as it avoids Python-level loops and leverages pandas' optimized C-based operations.
If you need to keep the debugging prints, you can add them after the merge operation to inspect a sample of the results.
- User
do a map here: transfer_rows['transfer_value'] = transfer_rows['Descrição'].str.extract(r': r$([\d.,]+)')[0] instead of extract, extract n the same way i did
Assistant
You want to modify the extraction of transfer values to match your original approach of splitting the string at
": r$"and converting the number. Here's how to do it with a vectorized approach usingstr.split()andmap():# Extract the value after ": r$" and convert to float transfer_rows['transfer_value'] = ( transfer_rows['Descrição'] .str.split(": r$") .str[1] # Get the part after ": r$" .map(convert_to_english_float) # Apply your conversion function )However, since your
convert_to_english_floatfunction was originally designed for single strings, we should modify it slightly to work with this vectorized approach. Here's the complete optimized solution:# Modified conversion function that works with both strings and Series def convert_to_english_float(value): if isinstance(value, str): return float(value.split(".")[0].replace(".", "").replace(",", ".").strip()) else: # Assuming it's a pandas Series return ( value.str.split(".").str[0] .str.replace(".", "") .str.replace(",", ".") .str.strip() .astype(float) ) # Preprocess the data df_gerais_principal['Código Processo'] = df_gerais_principal['Código Processo'].astype(str) df_movimentos_orgao['Código Processo'] = df_movimentos_orgao['Código Processo'].astype(str) # Filter rows with "valor transferido" and extract the values transfer_rows = df_movimentos_orgao[ df_movimentos_orgao['Descrição'].str.contains('valor transferido', na=False) ].copy() # Extract values using your original approach but vectorized transfer_rows['transfer_value'] = ( transfer_rows['Descrição'] .str.split(": r$") .str[1] # Get the part after ": r$" .pipe(convert_to_english_float) # Apply conversion ) # Group by process code and sum the transfer values transfer_sums = transfer_rows.groupby('Código Processo')['transfer_value'].sum().reset_index() transfer_sums.columns = ['Código Processo', 'total_transferred'] # Convert main values to float df_gerais_principal['valor_total'] = convert_to_english_float(df_gerais_principal['Valor']) # Merge and calculate df_gerais_principal = df_gerais_principal.merge( transfer_sums, on='Código Processo', how='left' ).fillna({'total_transferred': 0}) df_gerais_principal['valor_total_menos_transferidos'] = ( df_gerais_principal['valor_total'] - df_gerais_principal['total_transferred'] ) # Drop intermediate columns if needed df_gerais_principal = df_gerais_principal.drop(['total_transferred'], axis=1)Key changes:
- Made
convert_to_english_floatwork with both individual strings and pandas Series - Used
str.split(": r$").str[1]to match your original extraction approach - Used
pipe()to apply the conversion function to the Series - Kept all other optimizations from the previous version
This maintains the exact same logic as your original code but performs the operations in a vectorized way for better performance.
- Made
- User
only use .str acessor wiht string values!
Extract values using your original approach but vectorized
transfer_rows['transfer_value'] = ( transfer_rows['Descrição'] .str.split(": r") .str[1] # Get the part after ": r" .pipe(convert_to_english_float) # Apply conversion )
Assistant
Got it! You want to ensure we only use
.straccessor with string values and keep the originalconvert_to_english_floatfunction unchanged (operating on individual strings). Here's the optimized solution usingmap()instead of.pipe():def convert_to_english_float(value: str): """Original function - operates on individual strings""" return float(value.split(".")[0].replace(".", "").replace(",", ".").strip()) # Preprocess the data df_gerais_principal['Código Processo'] = df_gerais_principal['Código Processo'].astype(str) df_movimentos_orgao['Código Processo'] = df_movimentos_orgao['Código Processo'].astype(str) # Filter rows with "valor transferido" and extract the values transfer_rows = df_movimentos_orgao[ df_movimentos_orgao['Descrição'].str.contains('valor transferido', na=False) ].copy() # Extract values using .str accessor only for string operations transfer_rows['transfer_value'] = ( transfer_rows['Descrição'] .str.split(": r$") .str[1] # Get the part after ": r$" (this is still string operation) .map(convert_to_english_float) # Then map the conversion function to each string ) # Group by process code and sum the transfer values transfer_sums = transfer_rows.groupby('Código Processo')['transfer_value'].sum().reset_index() transfer_sums.columns = ['Código Processo', 'total_transferred'] # Convert main values to float - using map since convert_to_english_float works on strings df_gerais_principal['valor_total'] = df_gerais_principal['Valor'].map(convert_to_english_float) # Merge and calculate df_gerais_principal = df_gerais_principal.merge( transfer_sums, on='Código Processo', how='left' ).fillna({'total_transferred': 0}) df_gerais_principal['valor_total_menos_transferidos'] = ( df_gerais_principal['valor_total'] - df_gerais_principal['total_transferred'] ) # Drop intermediate columns if needed df_gerais_principal = df_gerais_principal.drop(['total_transferred'], axis=1)Key points:
- Kept the original
convert_to_english_floatfunction unchanged - Used
.straccessor only for string operations (splitting) - Used
.map()to apply the conversion function to each string value - Maintained all other optimizations from the previous version
This approach is clean and follows your requirement to only use
.strfor string operations while keeping the conversion logic in the original function.- Kept the original
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.