Write a function that calculates how the 5 versions of density correlate with each…
- User
Write a function that calculates how the 5 versions of density correlate with each other. Plot a heat map. The 5 versions are the 4 + judge_score
""" from time import sleep import pandas as pd import os from openai import OpenAI import json from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt from scipy.stats import pearsonr, spearmanr
from prompts.participation import PARTICIPATION_SYS from prompts.responsivity import RESPONSIVITY_SYS from prompts.internal_cohesion import IN_COHESION_SYS from prompts.social_impact import SOCIAL_IMPACT_SYS from prompts.newness import NEWNESS_SYS from prompts.communication_density import COMMUNICATION_DENSE_SYS
""" Evaluate each GCA aspect using LLM as a judge.
First, for each discussion and participant, create a text file in the following
""" API_KEY_OPENAI = "sk-proj-XXX" os.environ[ "OPENAI_API_KEY"] = API_KEY_OPENAI
SEED = 30 TEST_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/group_discussion_analysis_rep/evalGCA/test_data.csv" DATA_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/data"
MODELS = {"GPT4_1": "gpt-4.1-2025-04-14", }
PROMPTS = { "participation": PARTICIPATION_SYS, "responsivity": RESPONSIVITY_SYS, "internal_cohesion": IN_COHESION_SYS, "social_impact": SOCIAL_IMPACT_SYS, "newness": NEWNESS_SYS, "communication_density": COMMUNICATION_DENSE_SYS }
def extract_score(response_text: str): """ Parse model response and extract rationale + score. If parsing fails, fall back to raw text. """ try: # Try to parse JSON data = json.loads(response_text) score = int(data.get("score")) rationale = data.get("rationale", "") return score, rationale
except Exception as e: print(f"Response in unexpected format: {response_text}.") return None, response_text.strip()def load_data(path: str) -> pd.DataFrame: """ load the csv with all discussion turns :param path: :return: """ return pd.read_csv(path)
def create_judge_files(df: pd.DataFrame, output_dir: str): """ Create text files for each discussion and participant for LLM judging in the format: Discussion: <discussion_name> Participant: <participant_name> Text:
:param df: :param output_dir: :return: """ # create directory if it does not exist if not os.path.exists(output_dir): os.makedirs(output_dir) discussions = df['discussion'].unique() for discussion in discussions: df_disc = df[df['discussion'] == discussion] participants = df_disc['author'].unique() for participant in participants: filename = f"{output_dir}/{discussion}_{participant}.txt" with open(filename, 'w') as f: f.write(f"Discussion: {discussion}\n") f.write(f"Participant: {participant}\n") f.write(f"Content:\n") # for each turn, write "<author>: <text>" for idx, row in df_disc.iterrows(): f.write(f"\n{row['author']}: {row['text']}\n") passdef main_preprocess(): data = load_data("/home/esther/PycharmProjects/Dialogos/role_discovery/data/turn_all_contributions.csv") create_judge_files(data, "/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files")
def main_run_judge(judge_data_dir: str, test_data: pd.DataFrame, split: str, aspect: str, model: str): """ Run the LLM judge on the test data for the given aspect. For each discussion and participant in the test data, load the corresponding text file and run the LLM judge prompt for the given aspect. Save the results in a csv file.
:param split: :param model: :param test_data: :param judge_data_dir: :param aspect: :return: """ # ✅ Save after every iteration (safe checkpointing) out_path = f"outs/{split}/{model}/{aspect}/judge_results_{aspect}.csv" # Load history if file exists if os.path.exists(out_path): existing = pd.read_csv(out_path).dropna(subset=["score"]) done_pairs = set(zip(existing["discussion"], existing["participant"])) results = existing.to_dict("records") print(f"Loaded history: {len(done_pairs)} completed evaluations") else: # create directory if it does not exist dir = "/".join(out_path.split("/")[:-1]) os.makedirs(dir, exist_ok=True) done_pairs = set() results = [] print("No history found, starting fresh") # Go through unique (discussion, participant) pairs in the test data discussion_author_pairs = test_data[["discussion", "author"]].drop_duplicates() for _, row in discussion_author_pairs.iterrows(): discussion = row["discussion"] participant = row["author"] if (discussion, participant) in done_pairs: # todo only skip if score is not null - search results for this pair and check score if any(r for r in results if r["discussion"] == discussion and r["participant"] == participant and r["score"] is not None and r["score"] > 0): continue # Skip already completed filename = f"{judge_data_dir}/{discussion}_{participant}.txt" if not os.path.exists(filename): print(f"Warning: missing file {filename}") continue with open(filename, "r") as f: user_prompt = f.read() try: client = OpenAI() response = client.responses.create( model=MODELS[model], input=[ {"role": "system", "content": [{"type": "input_text", "text": PROMPTS[aspect]}]}, {"role": "user", "content": [{"type": "input_text", "text": user_prompt}]}, ], text={}, reasoning={}, tools=[], temperature=0, max_output_tokens=3000, top_p=1, store=True, ) res = response.output[0].content[0].text score, rationale = extract_score(res) except Exception as e: print(f"Error for {discussion}-{participant}: {e}") score = None rationale = None results.append({ "discussion": discussion, "participant": participant, "aspect": aspect, "score": score, "rationale": rationale }) sleep(0.3) # to avoid rate limits pd.DataFrame(results).to_csv(out_path, index=False) print(f"Progress saved: {discussion}-{participant}") # sort results by discussion, participant final_df = pd.DataFrame(results).sort_values(by=["discussion", "participant"]) final_df.to_csv(out_path, index=False) print(f"Finished. Results saved to {out_path}")def create_test_split(blocks: str, size: float) -> pd.DataFrame: """ If the test data already exists, just load it. Otherwise,load the data and create a test split of the given size
All discussions selected must have at least 3 participants and at least 15 contributions. :param blocks: "etu", "du" or "turn" :param size: :return: """ # If it already exists, just load if size < 1. and os.path.exists(TEST_PATH): print(f"Loading existing test split") return pd.read_csv(TEST_PATH) # Map block type to file path block2file = { "etu": f"{DATA_PATH}/all_contributions.csv", "du": f"{DATA_PATH}/du_all_contributions.csv", "turn": f"{DATA_PATH}/turn_all_contributions.csv", } if blocks not in block2file: raise ValueError(f"Unknown blocks type: {blocks}") # Load data df = pd.read_csv(block2file[blocks]) # filter out discussions with at least 3 participants and at least 15 contributions valid_discussions = [] for disc, g in df.groupby("discussion"): n_participants = g["author"].nunique() n_cont = len(g) if n_participants >= 3 and n_cont >= 15: valid_discussions.append(disc) df_valid = df[df["discussion"].isin(valid_discussions)] # if size == 1.0, return all valid data if size == 1.0: return df_valid # Sample discussions for test set discussions = df_valid["discussion"].unique() print("Number of unique valid discussions:", df_valid["discussion"].nunique()) _, test_idxs = train_test_split( discussions, test_size=size, random_state=SEED ) print("Size of test split:", len(test_idxs)) test_df = df_valid[df_valid["discussion"].isin(test_idxs)] # Save for reuse test_df.to_csv(TEST_PATH, index=False) print(f"Saved test split") return test_dfdef judge_main(split="test"): if split == "test": judge_data = create_test_split(blocks="du", size=.20) else: judge_data = create_test_split(blocks="du", size=1.)
for aspect in PROMPTS.keys(): print(f"\nEvaluating aspect: {aspect}\n") main_run_judge(test_data=judge_data, aspect=aspect, split=split, model="GPT4_1", judge_data_dir="/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files/")def judge_evaluation(): """ Evaluate the judge results against the GCA scores. 1. Load the judge results for each aspect 2. Load the GCA scores 3. For each aspect, - compute the correlation between the judge scores and the GCA scores - plot the distributions of the judge scores and the GCA scores - plot the scatter plot of the judge scores vs the GCA scores 4. Print the results :return: """ model = "GPT4_1" base_out_dir = f"outs/all/{model}" os.makedirs(base_out_dir, exist_ok=True)
gca_df = pd.read_csv(f"{DATA_PATH}/du_results_6_aspects_129_discussions.csv") # normalize column names for merge if "discussion_id" in gca_df.columns: gca_df = gca_df.rename(columns={"discussion_id": "discussion"}) # Map aspects → GCA score column aspect2col = { "participation": "participation_score", "responsivity": "overall_responsivity_score", "internal_cohesion": "internal_cohesion_score", "social_impact": "social_impact_score", "newness": "newness_score", "communication_density": "communication_density_score", } summary_records = [] for aspect, gca_col in aspect2col.items(): judge_path = f"{base_out_dir}/{aspect}/judge_results_{aspect}.csv" if not os.path.exists(judge_path): print(f"Skipping {aspect}: no judge file at {judge_path}") continue judge_df = pd.read_csv(judge_path) merged = pd.merge( judge_df, gca_df[["discussion", "participant", gca_col]], on=["discussion", "participant"], how="inner", ) merged = merged.rename(columns={"score": "judge_score", gca_col: "gca_score"}) merged = merged.dropna(subset=["judge_score", "gca_score"]) # correlations pearson_r, pearson_p = pearsonr(merged["judge_score"], merged["gca_score"]) n = len(merged) print(f"\nAspect: {aspect} | N={n}") print(f" Pearson r = {pearson_r:.3f} (p={pearson_p:.3g})") summary_records.append({ "aspect": aspect, "n": n, "pearson_r": pearson_r, "pearson_p": pearson_p, }) # plot dir plot_dir = f"{base_out_dir}/{aspect}/plots" os.makedirs(plot_dir, exist_ok=True) max_judge_score = merged["judge_score"].max() max_gca_score = merged["gca_score"].max() min_judge_score = merged["judge_score"].min() min_gca_score = merged["gca_score"].min() # merged["judge_norm"] = merged["judge_score"].apply(lambda x: x/max_judge_score) # merged["gca_norm"] = merged["gca_score"].apply(lambda x: x/max_gca_score) merged["judge_norm"] = merged["judge_score"].apply( lambda x: (x - min_judge_score) / (max_judge_score - min_judge_score) ) merged["gca_norm"] = merged["gca_score"].apply( lambda x: (x - min_gca_score) / (max_gca_score - min_gca_score) ) # ---- Overlay normalized distributions ---- plt.figure() plt.hist(merged["judge_norm"], bins=5, alpha=0.6, # density=True, label="Judge (normalized)", edgecolor="grey" # outlines the bars ) plt.hist(merged["gca_norm"], bins=20, alpha=0.6, # density=True, label="GCA (normalized)" , edgecolor="grey") # outlines the bars plt.xlabel("Normalized score (0–1)") plt.ylabel("Density") plt.title(f"{aspect} – Normalized distributions") plt.legend() plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_normalized_overlay.png") plt.show() plt.close() # # histograms # plt.figure() # plt.hist(merged["judge_score"], bins=10) # plt.title(f"{aspect} – Judge scores") # plt.savefig(f"{plot_dir}/{aspect}_judge_hist.png") # plt.show() # plt.close() # # plt.figure() # plt.hist(merged["gca_score"], bins=10) # plt.title(f"{aspect} – GCA scores") # plt.savefig(f"{plot_dir}/{aspect}_gca_hist.png") # plt.show() # plt.close() # scatter plt.figure() plt.scatter( merged["gca_score"], merged["judge_score"], s=10, # small dots alpha=0.7, # transparency so overlaps look darker ) plt.title(f"{aspect} – Judge vs GCA") # make axis limits match min and max of both scores acordingly plt.xlim(min_gca_score, max_gca_score) plt.ylim(min_judge_score - .1, max_judge_score + .1) # y values should be integers plt.yticks(range(0, 6)) plt.xlabel("GCA score") plt.ylabel("Judge score") plt.savefig(f"{plot_dir}/{aspect}_scatter.png") plt.show() plt.close() # save merged data merged_out = f"{base_out_dir}/{aspect}/merged_judge_gca_{aspect}.csv" merged.to_csv(merged_out, index=False) # summary table if summary_records: summary_df = pd.DataFrame(summary_records) summary_path = f"{base_out_dir}/judge_gca_summary.csv" summary_df.to_csv(summary_path, index=False) print("\n=== Overall Summary ===") print(summary_df.to_string(index=False))def density_eval(): """ Evaluate the 4 versions of communication density. 1. Load the judge results for 9 aspects 2. Filter the columns for the 4 density versions 3. For each version, - compute the correlation between the judge scores and the GCA scores - plot the distributions of the judge scores and the GCA scores - plot the scatter plot of the judge scores vs the GCA scores 4. Print the results
:return: """ model = "GPT4_1" base_out_dir = f"outs/all/{model}" os.makedirs(base_out_dir, exist_ok=True) # Load GCA df gca_path = f"{DATA_PATH}/du_results_9_aspects_129_discussions.csv" if not os.path.exists(gca_path): print(f"GCA file not found at {gca_path}. Aborting density_eval.") return gca_df = pd.read_csv(gca_path) # normalize column names if "discussion_id" in gca_df.columns: gca_df = gca_df.rename(columns={"discussion_id": "discussion"}) # density columns density2col = { "communication_density": "tf_idf_norm_cd", "tf_idf_sum_communication_density": "tf_idf_sum_cd", "pos_communication_density": "pos_cd", "llm_concise_communication_density": "llm_concise_cd", } summary_records = [] # compare each density column to the judge scores for "communication_density" judge_path = f"{base_out_dir}/communication_density/judge_results_communication_density.csv" if not os.path.exists(judge_path): print(f"Judge file not found at {judge_path}. Aborting density_eval.") return judge_df = pd.read_csv(judge_path) # for each density version for density_col in density2col.keys(): merged = pd.merge( judge_df, gca_df[["discussion", "participant", density_col]], on=["discussion", "participant"], how="inner", ) merged = merged.rename(columns={"score": "judge_score", density_col: density2col[density_col]}) col = density2col[density_col] merged = merged.dropna(subset=["judge_score", col]) # correlations pearson_r, pearson_p = pearsonr(merged["judge_score"], merged[col]) n = len(merged) print(f"\nDensity version: {col} | N={n}") print(f" Pearson r = {pearson_r:.3f} (p={pearson_p:.3g})") summary_records.append({ "version": col, "n": n, "pearson_r": pearson_r, "pearson_p": pearson_p, }) # plot dir plot_dir = f"{base_out_dir}/{col}/plots" os.makedirs(plot_dir, exist_ok=True) max_judge_score = merged["judge_score"].max() max_gca_score = merged[col].max() min_judge_score = merged["judge_score"].min() min_gca_score = merged[col].min() # merged["judge_norm"] = merged["judge_score"].apply(lambda x: x / max_judge_score) # merged["gca_norm"] = merged["gca_score"].apply(lambda x: x / max_gca_score) merged["judge_norm"] = (merged["judge_score"] - min_judge_score) / (max_judge_score - min_judge_score) merged["gca_norm"] = (merged[col] - min_gca_score) / (max_gca_score - min_gca_score) # ---- Overlay normalized distributions ---- plt.figure() plt.hist(merged["judge_norm"], bins=5, alpha=0.6, # density=True, label="Judge (normalized)", edgecolor="grey" # outlines the bars ) plt.hist(merged["gca_norm"], bins=20, alpha=0.6, # density=True, label="GCA (normalized)" , edgecolor="grey") # outlines the bars plt.xlabel("Normalized score (0–1)") plt.ylabel("Density") plt.title(f"{col} – Normalized distributions") plt.legend() plt.tight_layout() plt.savefig(f"{plot_dir}/{col}_normalized_overlay.png") plt.show() plt.close() # scatter plt.figure() plt.scatter( merged[col], merged["judge_score"], s=10, # small dots alpha=0.7, # transparency so overlaps look darker ) plt.title(f"{col} – Judge vs GCA") # make axis limits match min and max of both scores acordingly plt.xlim(min_gca_score, max_gca_score) plt.ylim(min_judge_score - .1, max_judge_score + .1) # y values should be integers plt.yticks(range(0, 6)) plt.xlabel("GCA score") plt.ylabel("Judge score") plt.savefig(f"{plot_dir}/{col}_scatter.png") plt.show() plt.close() # save merged data merged_out = f"{base_out_dir}/{col}/merged_judge_gca_{col}.csv" merged.to_csv(merged_out, index=False) # summary table if summary_records: summary_df = pd.DataFrame(summary_records) summary_path = f"{base_out_dir}/density_summary.csv" summary_df.to_csv(summary_path, index=False) print("\n=== Overall Summary ===") print(summary_df.to_string(index=False))if name == 'main': """
""" # judge_main(split="all") # judge_evaluation() density_eval()"""
- Assistant
Got it ✅ You want a function that calculates how the five versions of communication density (the four GCA variants + the judge score) correlate with each other, and then plots a heatmap. Here’s how you can extend your code: pythonCopy codeimport seaborn as sns import matplotlib.pyplot as plt
def density_correlation_heatmap(): """ Compute correlations among the 5 versions of communication density (the 4 GCA variants + judge_score). Plot a heatmap of Pearson correlations. """ model = "GPT4_1" base_out_dir = f"outs/all/{model}"
# Load GCA df gca_path = f"{DATA_PATH}/du_results_9_aspects_129_discussions.csv" judge_path = f"{base_out_dir}/communication_density/judge_results_communication_density.csv" if not os.path.exists(gca_path) or not os.path.exists(judge_path): print("Required files not found. Aborting density_correlation_heatmap.") return gca_df = pd.read_csv(gca_path) judge_df = pd.read_csv(judge_path) # Normalize column names if "discussion_id" in gca_df.columns: gca_df = gca_df.rename(columns={"discussion_id": "discussion"}) # Map the density versions density2col = { "tf_idf_norm_cd": "TF-IDF Norm", "tf_idf_sum_cd": "TF-IDF Sum", "pos_cd": "POS-based", "llm_concise_cd": "LLM Concise", } # Merge judge + GCA data merged = judge_df.rename(columns={"score": "judge_score"}) merged = pd.merge( merged, gca_df[["discussion", "participant"] + list(density2col.keys())], on=["discussion", "participant"], how="inner" ).dropna(subset=["judge_score"] + list(density2col.keys())) # Select only the relevant columns corr_df = merged[["judge_score"] + list(density2col.keys())] corr_df = corr_df.rename(columns={"judge_score": "Judge"}) # Compute correlation matrix corr_matrix = corr_df.corr(method="pearson") # Plot heatmap plt.figure(figsize=(8, 6)) sns.heatmap(corr_matrix, annot=True, cmap="coolwarm", center=0, xticklabels=corr_matrix.columns, yticklabels=corr_matrix.columns, fmt=".2f") plt.title("Correlation Heatmap – Communication Density Versions", fontsize=14) plt.tight_layout() plt.savefig(f"{base_out_dir}/density_correlation_heatmap.png") plt.show() return corr_matrixThis function will:
Load the judge results and the four GCA communication density versions.
Merge them into one dataframe.
Compute Pearson correlations between all 5.
Plot a seaborn heatmap with annotated values.
Save the figure under outs/all/GPT4_1/density_correlation_heatmap.png.
Do you also want me to add Spearman correlation as an alternative matrix (maybe in a second heatmap)?