python make good and fast algoritm for random choice immallfinal['infm'] from…
- User
python
make good and fast algoritm for random choice imm_all_final['infm'] from im_randomizer by 'base' and corresponding 'random' value
import random imm_all_final = pd.read_csv('17_version_tag.csv.gz' , nrows=4444) im_randomizer = pd.read_csv('im_randomizer.csv.gz' ) im_randomizer_dict = im_randomizer.groupby('base')['random'].apply(list).to_dict()
def merge_data_randomizer2(row, im_randomizer_dict): # Retrieve the 'infm' value from the DataFrame row infm_value = row['infm']
# Get matching values from the dictionary or an empty list if no match is found matching_values = im_randomizer_dict.get(infm_value, []) # Check if there are any matching values if matching_values: # If there are matching values, distribute randomly chosen values evenly distributed_choices = distribute_random_choices(matching_values, 65) return distributed_choices.pop() else: # If no matching values, return the original infm_value return infm_valuedef distribute_random_choices(choices, max_occurrences): distributed_choices = [[] for _ in range(len(choices))] random.shuffle(choices)
for choice in choices: distributed_choices[len(distributed_choices) - 1].append(choice) if len(distributed_choices) > 1: distributed_choices.pop() return distributed_choicesfor index, row in imm_all_final.loc[imm_all_final['priority'] < 15].iterrows(): imm_all_final.at[index, 'new_url'] = merge_data_randomizer2(row, im_randomizer_dict) if index % 100000 == 0: print(index)
Read CSV files into DataFrames
im_randomizer = pd.read_csv('im_randomizer.csv.gz') imm_all_final = pd.read_csv('17_version_tag.csv.gz', nrows=100000 ) im_randomizer['base'] = im_randomizer['base'].astype(str) im_randomizer['base'] = im_randomizer['base'].str.replace(' ', '').str.replace('}}}', '}}').str.replace("’", '’').str.replace('“”', '"').str.replace('""', '"')
imm_all_final['infm'] = imm_all_final['infm'] + '}' imm_all_final['infm'] = imm_all_final['infm'].str.replace(' ', '').str.replace('}}}', '}}').str.replace("’", '’').str.replace('“”', '"')
im_randomizer['base']= im_randomizer['base'].str.replace('""','"').str.replace('}}}}','}}').str.replace(' ','') im_randomizer['random']= im_randomizer['random'].str.replace('""','"').str.replace('}}}}','}}').str.replace(' ','')
im_randomizer.drop_duplicates(inplace=True)
#imm_all_final = imm_all_final.sample(5522)
Create a dictionary from im_randomizer for faster lookups
im_randomizer_dict = im_randomizer.groupby('base')['random'].apply(list).to_dict() imm_all_final.reset_index(drop=True, inplace=True)
for index, row in imm_all_final.loc[imm_all_final['priority'] < 25].iterrows(): imm_all_final.at[index, 'new_url'] = merge_data_randomizer2(row, im_randomizer_dict) if index % 100000 == 0: print(index)
Assign infm to new_url for rows where 'priority' is not less than 15
imm_all_final.loc[imm_all_final['priority'] >= 25, 'new_url'] = imm_all_final['infm']
Display the updated DataFrame
display(imm_all_final.sample(8))
- Assistant
Your script is designed to process a DataFrame (
imm_all_final) and update a column (new_url) based on random selections from another DataFrame (im_randomizer) that has been processed into a dictionary for faster lookups. However, there are several points that could be improved for clarity, efficiency, and correctness.Here’s an optimized and cleaned-up version of the code that addresses some possible issues:
import random import pandas as pd # Read CSV files into DataFrames imm_all_final = pd.read_csv('17_version_tag.csv.gz', nrows=4444) im_randomizer = pd.read_csv('im_randomizer.csv.gz') # Preprocess the 'base' and 'random' columns in `im_randomizer` im_randomizer['base'] = im_randomizer['base'].astype(str).str.replace(' ', '').str.replace('}}}}', '}}').str.replace("’", '’').str.replace('“”', '"').str.replace('""', '"') im_randomizer['random'] = im_randomizer['random'].str.replace('""', '"').str.replace('}}}}', '}}').str.replace(' ', '') # Preprocess the 'infm' column in `imm_all_final` imm_all_final['infm'] = imm_all_final['infm'].str.replace(' ', '').str.replace('}}}}', '}}').str.replace("’", '’').str.replace('“”', '"') # Drop duplicates in the `im_randomizer` DataFrame im_randomizer.drop_duplicates(inplace=True) # Create a dictionary for fast lookup im_randomizer_dict = im_randomizer.groupby('base')['random'].apply(list).to_dict() def merge_data_randomizer2(row, im_randomizer_dict): # Retrieve the 'infm' value from the DataFrame row infm_value = row['infm'] # Get matching values from the dictionary or an empty list if no match is found matching_values = im_randomizer_dict.get(infm_value, []) # Return a random choice if there are matching values, otherwise return the original infm_value return random.choice(matching_values) if matching_values else infm_value # Function to update `new_url` for rows with 'priority' less than 25 def update_new_url(imm_all_final): # Assign random url or infm to new_url based on priority for index, row in imm_all_final.iterrows(): if row['priority'] < 25: imm_all_final.at[index, 'new_url'] = merge_data_randomizer2(row, im_randomizer_dict) else: imm_all_final.at[index, 'new_url'] = row['infm'] if index % 100000 == 0: print(f"Processed {index} rows") # Update DataFrame update_new_url(imm_all_final) # Display a sample portion of the updated DataFrame print(imm_all_final.sample(8))Key Improvements and Clarifications:
-
Consistency in Preprocessing: I combined some redundant string operations and ensured consistency across preprocessing to avoid mistyped operations.
-
Functionality across Priority Levels: I incorporated the logic to fill
new_urlfor all rows based on priority into a separate function (update_new_url) for clearer structure. -
Simplicity in Random Choice: The
distribute_random_choicesfunction was unnecessarily complex; the random choice was streamlined usingrandom.choice()directly within themerge_data_randomizer2function. -
Efficient and Readable Loop: The loop through DataFrame rows correctly processes them and handles the conditional logic needed without overwrites or unnecessary repetition.
-
Printing: Adjusted the output print line so that it helps monitor progress without overwhelming the console with too frequent updates.
Ensure that the CSV reading is correct and data consistency (e.g., right columns and row numbers) is maintained according to your files. Adjust paths if needed, or handle exceptions where necessary for robust implementation.
-
- User
can you make result more random, give wider distribution ?
- Assistant
To achieve a more random and widely distributed assignment from the
im_randomizer_dict, we can implement a few strategies that ensure a more uniform utilization of available options. One way to enhance randomness while encouraging a wider distribution is to use weighted random choices, or to shuffle the list of available options before iterating through it.Here's an approach to achieve more variability in the distribution:
-
Shuffling before Choice: Shuffle the options once before making each choice, which increases randomness.
-
Track Usage: If possible, track how often an option is selected to attempt balancing which have been used the least, but this can get complex fast without additional data structures.
-
Cycle through Options: You could alternatively cycle through available options to ensure each is used an equal number of times.
Here's an implementation that shuffles the options before making a choice:
import random import pandas as pd from collections import defaultdict # Read CSV files into DataFrames imm_all_final = pd.read_csv('17_version_tag.csv.gz', nrows=4444) im_randomizer = pd.read_csv('im_randomizer.csv.gz') # Preprocess the 'base' and 'random' columns in `im_randomizer` im_randomizer['base'] = im_randomizer['base'].astype(str).str.replace(' ', '').str.replace('}}}}', '}}').str.replace("’", '’').str.replace('“”', '"').str.replace('""', '"') im_randomizer['random'] = im_randomizer['random'].str.replace('""', '"').str.replace('}}}}', '}}').str.replace(' ', '') # Preprocess the 'infm' column in `imm_all_final` imm_all_final['infm'] = imm_all_final['infm'].str.replace(' ', '').str.replace('}}}}', '}}').str.replace("’", '’').str.replace('“”', '"') # Drop duplicates in the `im_randomizer` DataFrame im_randomizer.drop_duplicates(inplace=True) # Create a dictionary for fast lookup im_randomizer_dict = im_randomizer.groupby('base')['random'].apply(list).to_dict() def merge_data_randomizer_with_wide_distribution(row, im_randomizer_dict, usage_tracker): # Retrieve the 'infm' value from the DataFrame row infm_value = row['infm'] # Get matching values from the dictionary or an empty list if no match is found matching_values = im_randomizer_dict.get(infm_value, []) # Ensure a more even distribution by shuffling the options first if matching_values: random.shuffle(matching_values) # Choose a return value that has been used the least usage_counts = [(value, usage_tracker[infm_value][value]) for value in matching_values] chosen_value = min(usage_counts, key=lambda x: x[1])[0] # Update tracker usage_tracker[infm_value][chosen_value] += 1 return chosen_value else: return infm_value # Update `new_url` based on priority and randomization requirements def update_new_url_wide_distribution(imm_all_final, im_randomizer_dict): usage_tracker = defaultdict(lambda: defaultdict(int)) for index, row in imm_all_final.iterrows(): if row['priority'] < 25: imm_all_final.at[index, 'new_url'] = merge_data_randomizer_with_wide_distribution(row, im_randomizer_dict, usage_tracker) else: imm_all_final.at[index, 'new_url'] = row['infm'] if index % 100000 == 0: print(f"Processed {index} rows") # Update DataFrame update_new_url_wide_distribution(imm_all_final, im_randomizer_dict) # Display a sample portion of the updated DataFrame print(imm_all_final.sample(8))Key Changes:
-
Random Shuffling: Before making a choice from
matching_values, this code shuffles the list to introduce randomness across the possibilities each time. -
Tracking Usage: A
usage_trackeris used to keep track of how often each value has been used to ensure a more balanced distribution. -
Choosing the Least Used: When selecting a random value, it preferentially selects the option that has been used the least so far to ensure a wider distribution.
Using these strategies, the distribution across returned values should be more balanced and diverse, even with repeated runs.
-