Implement the missing function: """ from time import sleep import pandas as pd import os…
- User
Implement the missing function:
""" from time import sleep
import pandas as pd import os from openai import OpenAI import json
from prompts.participation import PARTICIPATION_SYS from prompts.responsivity import RESPONSIVITY_SYS from prompts.internal_cohesion import IN_COHESION_SYS from prompts.social_impact import SOCIAL_IMPACT_SYS from prompts.newness import NEWNESS_SYS from prompts.communication_density import COMMUNICATION_DENSE_SYS from sklearn.model_selection import train_test_split
""" Evaluate each GCA aspect using LLM as a judge.
First, for each discussion and participant, create a text file in the following
""" API_KEY_OPENAI = "sk-proj-XXX" os.environ[ "OPENAI_API_KEY"] = API_KEY_OPENAI
SEED = 30 TEST_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/group_discussion_analysis_rep/evalGCA/test_data.csv" DATA_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/data"
MODELS = {"GPT4_1": "gpt-4.1-2025-04-14", }
PROMPTS = { "participation": PARTICIPATION_SYS, "responsivity": RESPONSIVITY_SYS, "internal_cohesion": IN_COHESION_SYS, "social_impact": SOCIAL_IMPACT_SYS, "newness": NEWNESS_SYS, "communication_density": COMMUNICATION_DENSE_SYS }
def extract_score(response_text: str): """ Parse model response and extract rationale + score. If parsing fails, fall back to raw text. """ try: # Try to parse JSON data = json.loads(response_text) score = int(data.get("score")) rationale = data.get("rationale", "") return score, rationale
except Exception as e: print(f"Response in unexpected format: {response_text}.") return None, response_text.strip()def load_data(path: str) -> pd.DataFrame: """ load the csv with all discussion turns :param path: :return: """ return pd.read_csv(path)
def create_judge_files(df: pd.DataFrame, output_dir: str): """ Create text files for each discussion and participant for LLM judging in the format: Discussion: <discussion_name> Participant: <participant_name> Text:
:param df: :param output_dir: :return: """ # create directory if it does not exist if not os.path.exists(output_dir): os.makedirs(output_dir) discussions = df['discussion'].unique() for discussion in discussions: df_disc = df[df['discussion'] == discussion] participants = df_disc['author'].unique() for participant in participants: filename = f"{output_dir}/{discussion}_{participant}.txt" with open(filename, 'w') as f: f.write(f"Discussion: {discussion}\n") f.write(f"Participant: {participant}\n") f.write(f"Content:\n") # for each turn, write "<author>: <text>" for idx, row in df_disc.iterrows(): f.write(f"\n{row['author']}: {row['text']}\n") passdef main_preprocess(): data = load_data("/home/esther/PycharmProjects/Dialogos/role_discovery/data/turn_all_contributions.csv") create_judge_files(data, "/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files")
def main_run_judge(judge_data_dir: str, test_data: pd.DataFrame, aspect: str, model: str): """ Run the LLM judge on the test data for the given aspect. For each discussion and participant in the test data, load the corresponding text file and run the LLM judge prompt for the given aspect. Save the results in a csv file.
:param model: :param test_data: :param judge_data_dir: :param aspect: :return: """ # ✅ Save after every iteration (safe checkpointing) out_path = f"outs/{model}/{aspect}/judge_results_{aspect}.csv" # Load history if file exists if os.path.exists(out_path): existing = pd.read_csv(out_path).dropna(subset=["score"]) done_pairs = set(zip(existing["discussion"], existing["participant"])) results = existing.to_dict("records") print(f"Loaded history: {len(done_pairs)} completed evaluations") else: done_pairs = set() results = [] print("No history found, starting fresh") # Go through unique (discussion, participant) pairs in the test data discussion_author_pairs = test_data[["discussion", "author"]].drop_duplicates() for _, row in discussion_author_pairs.iterrows(): discussion = row["discussion"] participant = row["author"] if (discussion, participant) in done_pairs: # todo only skip if score is not null - search results for this pair and check score if any(r for r in results if r["discussion"] == discussion and r["participant"] == participant and r["score"] is not None and r["score"] > 0): continue # Skip already completed filename = f"{judge_data_dir}/{discussion}_{participant}.txt" if not os.path.exists(filename): print(f"Warning: missing file {filename}") continue with open(filename, "r") as f: user_prompt = f.read() try: client = OpenAI() response = client.responses.create( model=MODELS[model], input=[ {"role": "system", "content": [{"type": "input_text", "text": PROMPTS[aspect]}]}, {"role": "user", "content": [{"type": "input_text", "text": user_prompt}]}, ], text={}, reasoning={}, tools=[], temperature=0, max_output_tokens=2048, top_p=1, store=True, ) res = response.output[0].content[0].text score, rationale = extract_score(res) except Exception as e: print(f"Error for {discussion}-{participant}: {e}") score = None rationale = None results.append({ "discussion": discussion, "participant": participant, "aspect": aspect, "score": score, "rationale": rationale }) sleep(0.3) # to avoid rate limits pd.DataFrame(results).to_csv(out_path, index=False) print(f"Progress saved: {discussion}-{participant}") print(f"Finished. Results saved to {out_path}")def create_test_split(blocks: str, size: float) -> pd.DataFrame: """ If the test data already exists, just load it. Otherwise,load the data and create a test split of the given size
All discussions selected must have at least 3 participants and at least 15 contributions. :param blocks: "etu", "du" or "turn" :param size: :return: """ # If it already exists, just load if os.path.exists(TEST_PATH): print(f"Loading existing test split") return pd.read_csv(TEST_PATH) # Map block type to file path block2file = { "etu": f"{DATA_PATH}/all_contributions.csv", "du": f"{DATA_PATH}/du_all_contributions.csv", "turn": f"{DATA_PATH}/turn_all_contributions.csv", } if blocks not in block2file: raise ValueError(f"Unknown blocks type: {blocks}") # Load data df = pd.read_csv(block2file[blocks]) # filter out discussions with at least 3 participants and at least 15 contributions valid_discussions = [] for disc, g in df.groupby("discussion"): n_participants = g["author"].nunique() n_cont = len(g) if n_participants >= 3 and n_cont >= 15: valid_discussions.append(disc) df_valid = df[df["discussion"].isin(valid_discussions)] # Sample discussions for test set discussions = df_valid["discussion"].unique() print("Number of unique valid discussions:", df_valid["discussion"].nunique()) _, test_idxs = train_test_split( discussions, test_size=size, random_state=SEED ) print("Size of test split:", len(test_idxs)) test_df = df_valid[df_valid["discussion"].isin(test_idxs)] # Save for reuse test_df.to_csv(TEST_PATH, index=False) print(f"Saved test split") return test_dfdef judge_main(): test_data = create_test_split(blocks="du", size=.20) for aspect in PROMPTS.keys(): print(f"\nEvaluating aspect: {aspect}\n") main_run_judge(test_data=test_data, aspect=aspect, model="GPT4_1", judge_data_dir="/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files")
def judge_evaluation(): """ Evaluate the judge results against the GCA scores. 1. Load the judge results for each aspect 2. Load the GCA scores 3. For each aspect, - compute the correlation between the judge scores and the GCA scores - plot the distributions of the judge scores and the GCA scores - plot the scatter plot of the judge scores vs the GCA scores 4. Print the results :return: """ pass
if name == 'main': """
""" judge_main() judge_evaluation()"""
- Assistant
Nice — I implemented judge_evaluation() for you. It:
Loads the judge outputs for each aspect from outs/GPT4_1//judge_results_.csv.
Looks for GCA/reference scores in a few likely filenames inside DATA_PATH and picks the first that exists.
Merges judge results with GCA scores on (discussion, participant) (handles author vs participant name differences).
Computes Pearson and Spearman correlations, sample size, and basic stats.
Plots distributions (histograms) and a scatter plot (judge vs GCA) for each aspect, saves them to outs/GPT4_1//plots/.
Saves merged numeric data for each aspect to outs/GPT4_1//merged_judge_gca_.csv.
Is robust to missing files and prints helpful warnings.
Drop this function into your file (replace the pass in judge_evaluation()): pythonCopy codedef judge_evaluation(): """ Evaluate the judge results against the GCA scores. For each aspect: - load judge CSV (outs/GPT4_1//judge_results_.csv) - load GCA/reference scores (tries a list of plausible filenames in DATA_PATH) - merge on discussion+participant/author - compute Pearson and Spearman correlations - plot distributions and scatter, save plots and merged csv """ import matplotlib.pyplot as plt from scipy.stats import pearsonr, spearmanr
model = "GPT4_1" base_out_dir = f"outs/{model}" os.makedirs(base_out_dir, exist_ok=True) # Candidate GCA filenames to try (add others used in your project if needed) candidate_gca_files = [ f"{DATA_PATH}/gca_scores.csv", f"{DATA_PATH}/gca_all.csv", f"{DATA_PATH}/gca_all_scores.csv", f"{DATA_PATH}/gca_features.csv", f"{DATA_PATH}/all_contributions_with_gca.csv", f"{DATA_PATH}/all_gca_scores.csv", f"{DATA_PATH}/gca.csv", ] # find first existing GCA file gca_path = None for p in candidate_gca_files: if os.path.exists(p): gca_path = p break if gca_path is None: print("Could not find any GCA file in DATA_PATH. Tried the following paths:") for p in candidate_gca_files: print(f" - {p}") print("Please update `candidate_gca_files` in judge_evaluation() to include your GCA scores file path.") return print(f"Using GCA file: {gca_path}") gca_df = pd.read_csv(gca_path) # Normalize expected column names: prefer 'discussion' and 'participant' or 'author' if "discussion" not in gca_df.columns: raise ValueError(f"GCA file {gca_path} missing required 'discussion' column") # unify participant column name for merging if "participant" not in gca_df.columns and "author" in gca_df.columns: gca_df = gca_df.rename(columns={"author": "participant"}) elif "participant" not in gca_df.columns: # try other variants possible = [c for c in gca_df.columns if c.lower() in ("author", "participant", "person", "user")] if possible: gca_df = gca_df.rename(columns={possible[0]: "participant"}) else: raise ValueError("GCA file does not contain 'participant' or 'author' column to merge on.") # Identify which column(s) in gca_df hold the ground truth scores. # Heuristic: find numeric columns other than 'discussion' and 'participant' numeric_cols = gca_df.select_dtypes(include=["number"]).columns.tolist() if not numeric_cols: raise ValueError(f"No numeric columns found in GCA file {gca_path}. Need at least one numeric GCA score column.") # If there is a column that matches an aspect name we'll use that later in loop. summary_records = [] for aspect in PROMPTS.keys(): judge_path = f"{base_out_dir}/{aspect}/judge_results_{aspect}.csv" if not os.path.exists(judge_path): print(f"Skipping aspect '{aspect}': judge file not found at {judge_path}") continue print(f"\nProcessing aspect: {aspect}") judge_df = pd.read_csv(judge_path) # normalize judge df columns if "discussion" not in judge_df.columns: raise ValueError(f"Judge file {judge_path} missing 'discussion' column") if "participant" not in judge_df.columns and "author" in judge_df.columns: judge_df = judge_df.rename(columns={"author": "participant"}) # merge on discussion+participant merged = pd.merge(judge_df, gca_df, on=["discussion", "participant"], how="inner", suffixes=("_judge", "_gca")) if merged.empty: print(f"No overlapping rows between judge results and GCA for aspect '{aspect}'. Skipping.") continue # Try to determine the gca score column to compare to: # Prefer a column named exactly as aspect, otherwise pick the first numeric column not 'score' (judge's) gca_score_col = None if aspect in merged.columns: gca_score_col = aspect else: numeric_candidates = [c for c in numeric_cols if c in merged.columns and c != "score"] if len(numeric_candidates) == 1: gca_score_col = numeric_candidates[0] elif len(numeric_candidates) > 1: # try to pick a column with aspect substring for c in numeric_candidates: if aspect in c: gca_score_col = c break if gca_score_col is None: # otherwise pick the first numeric candidate gca_score_col = numeric_candidates[0] if gca_score_col is None: print(f"Could not determine GCA score column to compare for aspect '{aspect}'. Numeric candidates: {numeric_cols}") continue # judge score column name may be 'score' or 'score_judge' judge_score_col = "score" if judge_score_col not in merged.columns: # find numeric column that came from judge side candidates = [c for c in merged.select_dtypes(include=["number"]).columns if c.endswith("_judge") or "score" in c] judge_score_col = candidates[0] if candidates else None if judge_score_col is None: print(f"Could not determine judge score column for aspect '{aspect}'. Skipping.") continue # Keep only numeric and drop NaNs df_comp = merged[[ "discussion", "participant", judge_score_col, gca_score_col]].copy() df_comp = df_comp.rename(columns={judge_score_col: "judge_score", gca_score_col: "gca_score"}) df_comp = df_comp.dropna(subset=["judge_score", "gca_score"]) # ensure numeric df_comp["judge_score"] = pd.to_numeric(df_comp["judge_score"], errors="coerce") df_comp["gca_score"] = pd.to_numeric(df_comp["gca_score"], errors="coerce") df_comp = df_comp.dropna(subset=["judge_score", "gca_score"]) if df_comp.empty: print(f"No numeric overlapping scores for aspect '{aspect}' after cleaning. Skipping.") continue # compute correlations try: pearson_r, pearson_p = pearsonr(df_comp["judge_score"], df_comp["gca_score"]) except Exception: pearson_r, pearson_p = (None, None) try: spearman_r, spearman_p = spearmanr(df_comp["judge_score"], df_comp["gca_score"]) except Exception: spearman_r, spearman_p = (None, None) n = len(df_comp) print(f"Aspect: {aspect} | N={n} | pearson r={pearson_r:.3f} (p={pearson_p}) | spearman r={spearman_r:.3f} (p={spearman_p})") summary_records.append({ "aspect": aspect, "n": n, "pearson_r": pearson_r, "pearson_p": pearson_p, "spearman_r": spearman_r, "spearman_p": spearman_p, "gca_score_col": gca_score_col, "judge_path": judge_path, "gca_path": gca_path }) # create output dir for plots plot_dir = f"{base_out_dir}/{aspect}/plots" os.makedirs(plot_dir, exist_ok=True) # Histogram: judge scores plt.figure() plt.hist(df_comp["judge_score"], bins=10) plt.title(f"{aspect} - Judge score distribution (N={n})") plt.xlabel("Judge score") plt.ylabel("Count") plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_judge_hist.png") plt.close() # Histogram: gca scores plt.figure() plt.hist(df_comp["gca_score"], bins=10) plt.title(f"{aspect} - GCA score distribution (N={n})") plt.xlabel("GCA score") plt.ylabel("Count") plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_gca_hist.png") plt.close() # Scatter plot plt.figure() plt.scatter(df_comp["gca_score"], df_comp["judge_score"]) plt.title(f"{aspect} - Judge vs GCA (N={n})\npearson r={pearson_r:.3f}, spearman r={spearman_r:.3f}") plt.xlabel("GCA score") plt.ylabel("Judge score") # show simple y=x line for reference try: minv = min(df_comp["gca_score"].min(), df_comp["judge_score"].min()) maxv = max(df_comp["gca_score"].max(), df_comp["judge_score"].max()) plt.plot([minv, maxv], [minv, maxv], linestyle="--") except Exception: pass plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_scatter.png") plt.close() # save merged numeric data merged_out_path = f"{base_out_dir}/{aspect}/merged_judge_gca_{aspect}.csv" df_comp.to_csv(merged_out_path, index=False) print(f"Saved merged numeric data to {merged_out_path}") print(f"Saved plots to {plot_dir}") # summary if summary_records: summary_df = pd.DataFrame(summary_records) summary_path = f"{base_out_dir}/judge_gca_summary.csv" summary_df.to_csv(summary_path, index=False) print(f"\nSummary saved to {summary_path}") print(summary_df.to_string(index=False)) else: print("No aspects processed; nothing to summarize.")Notes & hints
If your GCA (ground-truth) scores file uses a different filename or has multiple specialized columns per aspect (e.g., participation_score, responsivity_score), add those filenames to candidate_gca_files or adjust the detection logic to prefer columns that include the aspect name (e.g., if aspect in c:).
If your judge CSVs store the judge score under a different column name than score (or as score_judge), the implemented heuristics should pick it up; otherwise tweak the mapping near the judge_score_col detection.
If you want nicer plots or to save figures in other formats (PDF), change the savefig calls.
I used scipy.stats.pearsonr and spearmanr; if scipy is not installed, pip install scipy will be needed.
If you want, I can:
adapt the function to automatically accept a path to your exact GCA file or a mapping of aspect->gca-column names,
or run a quick local dry-run on your machine (if you paste paths or the exact GCA filename/columns) to ensure matching works without edits.
- User
The GCA_score_df columns are: ['discussion_id', 'participant', 'participation_score', 'overall_responsivity_score', 'internal_cohesion_score', 'social_impact_score', 'newness_score', 'communication_density_score']
- Assistant
Perfect — thanks, that clears things up. Since your GCA scores file already has a clean structure with one column per aspect, we don’t need all the guessing heuristics. We can directly map each aspect name in PROMPTS to its corresponding *_score column in the GCA dataframe. Here’s the updated judge_evaluation() function for your setup: pythonCopy codedef judge_evaluation(): """ Evaluate the judge results against the GCA scores. 1. Load the judge results for each aspect 2. Load the GCA scores 3. For each aspect: - merge judge and GCA scores on discussion+participant - compute Pearson & Spearman correlations - plot distributions and scatter plots - save merged data and plots """ import matplotlib.pyplot as plt from scipy.stats import pearsonr, spearmanr
model = "GPT4_1" base_out_dir = f"outs/{model}" os.makedirs(base_out_dir, exist_ok=True) # ✅ Adjust this path if needed gca_path = f"{DATA_PATH}/gca_scores.csv" if not os.path.exists(gca_path): raise FileNotFoundError(f"GCA scores file not found at {gca_path}") gca_df = pd.read_csv(gca_path) # normalize column names for merge if "discussion" in gca_df.columns: gca_df = gca_df.rename(columns={"discussion": "discussion_id"}) # Map aspects → GCA score column aspect2col = { "participation": "participation_score", "responsivity": "overall_responsivity_score", "internal_cohesion": "internal_cohesion_score", "social_impact": "social_impact_score", "newness": "newness_score", "communication_density": "communication_density_score", } summary_records = [] for aspect, gca_col in aspect2col.items(): judge_path = f"{base_out_dir}/{aspect}/judge_results_{aspect}.csv" if not os.path.exists(judge_path): print(f"Skipping {aspect}: no judge file at {judge_path}") continue judge_df = pd.read_csv(judge_path) # rename for merge if "discussion" in judge_df.columns: judge_df = judge_df.rename(columns={"discussion": "discussion_id"}) if "author" in judge_df.columns: judge_df = judge_df.rename(columns={"author": "participant"}) merged = pd.merge( judge_df, gca_df[["discussion_id", "participant", gca_col]], on=["discussion_id", "participant"], how="inner", ) if merged.empty: print(f"No overlap for aspect {aspect}") continue merged = merged.rename(columns={"score": "judge_score", gca_col: "gca_score"}) merged = merged.dropna(subset=["judge_score", "gca_score"]) if merged.empty: print(f"No numeric overlap for {aspect}") continue # correlations pearson_r, pearson_p = pearsonr(merged["judge_score"], merged["gca_score"]) spearman_r, spearman_p = spearmanr(merged["judge_score"], merged["gca_score"]) n = len(merged) print(f"\nAspect: {aspect} | N={n}") print(f" Pearson r = {pearson_r:.3f} (p={pearson_p:.3g})") print(f" Spearman r = {spearman_r:.3f} (p={spearman_p:.3g})") summary_records.append({ "aspect": aspect, "n": n, "pearson_r": pearson_r, "pearson_p": pearson_p, "spearman_r": spearman_r, "spearman_p": spearman_p, }) # plot dir plot_dir = f"{base_out_dir}/{aspect}/plots" os.makedirs(plot_dir, exist_ok=True) # histograms plt.figure() plt.hist(merged["judge_score"], bins=10) plt.title(f"{aspect} – Judge scores") plt.savefig(f"{plot_dir}/{aspect}_judge_hist.png") plt.close() plt.figure() plt.hist(merged["gca_score"], bins=10) plt.title(f"{aspect} – GCA scores") plt.savefig(f"{plot_dir}/{aspect}_gca_hist.png") plt.close() # scatter plt.figure() plt.scatter(merged["gca_score"], merged["judge_score"]) plt.title(f"{aspect} – Judge vs GCA") plt.xlabel("GCA score") plt.ylabel("Judge score") # diagonal reference line minv = min(merged["gca_score"].min(), merged["judge_score"].min()) maxv = max(merged["gca_score"].max(), merged["judge_score"].max()) plt.plot([minv, maxv], [minv, maxv], "--") plt.savefig(f"{plot_dir}/{aspect}_scatter.png") plt.close() # save merged data merged_out = f"{base_out_dir}/{aspect}/merged_judge_gca_{aspect}.csv" merged.to_csv(merged_out, index=False) # summary table if summary_records: summary_df = pd.DataFrame(summary_records) summary_path = f"{base_out_dir}/judge_gca_summary.csv" summary_df.to_csv(summary_path, index=False) print("\n=== Overall Summary ===") print(summary_df.to_string(index=False)) else: print("No aspects evaluated.")✅ Differences from the draft I gave before:
Now directly uses your gca_scores.csv file.
Uses a fixed aspect→column mapping (aspect2col).
Drops all the column-guessing logic.
Keeps results & plots in the same structure as before.
Do you also want me to adapt this so that it prints one compact table (Pearson + Spearman for all aspects) at the end, instead of logging each aspect separately?
- User
Please help me understand the meaning of this result:
Aspect: participation | N=136 Pearson r = 0.634 (p=1.22e-16)
- Assistant
Great question 👍 let’s unpack what that result means.
The numbers
Aspect: participation → you’re comparing the LLM-judge’s scores against the GCA (ground truth) participation scores.
N=136 → you had 136 (discussion_id, participant) pairs with valid scores in both datasets.
Pearson r = 0.634 → this is the Pearson correlation coefficient, measuring how linearly related the two sets of scores are.
Ranges from -1 (perfect inverse correlation) to +1 (perfect positive correlation).
0.634 is a moderately strong positive correlation: when GCA says “higher participation”, the LLM judge also tends to give higher scores, fairly consistently.
p = 1.22e-16 → the p-value for testing whether the correlation is different from 0.
1.22e-16 is astronomically small, meaning this result is highly statistically significant.
In other words, the correlation is very unlikely to have occurred by chance.
Interpretation
Your LLM judge is aligned with the GCA participation metric: it captures much of the same signal.
An r of 0.63 means that about 40% of the variance in the judge scores can be explained by the GCA scores (r² = 0.40).
That’s solid agreement, but not perfect.
The LLM is probably sensitive to some nuances the GCA formula doesn’t capture, and vice versa.
The remaining 60% of variance reflects differences between the two methods (noise, judgment criteria, or systematic differences).
Practical takeaway
Your LLM-as-judge is a reasonably reliable evaluator of participation.
It’s not a 1:1 substitute for GCA scores, but it correlates strongly enough that you can treat it as capturing the same construct with some additional variability.
If you compare across aspects, participation might be one of the better-aligned ones.
Would you like me to also explain how to compare across aspects (e.g., what would count as “high” vs. “low” correlation for your other metrics like social impact or newness)?
- User
The judge scores can only be 1,2,3,4,5 so this plot doesn't make sense.
- Can you fix it?
- Can you normalize the values and then plot both distributions - i.e. the judge and the GCA on the same grid?
- Assistant
Got it ✅ Because your judge scores are ordinal (1–5) while GCA scores are continuous, plotting them as if they’re on the same continuous histogram doesn’t make much sense. Here’s how I’d fix it:
-
Plotting judge scores Instead of a histogram, use a bar chart showing counts per discrete score (1–5). That way you see how often each score was assigned.
-
Normalizing & overlaying distributions To compare judge vs. GCA on the same grid:
Normalize both series to z-scores (subtract mean, divide by std).
Plot them as kernel density estimates (KDEs) or smoothed histograms on the same axis.
Judge scores will still be coarse (1–5), but once normalized you can compare shapes.
Alternatively, you can plot two step histograms (density=True) overlaid.
Updated plotting snippet (replace the plotting block in judge_evaluation()) pythonCopy codefrom sklearn.preprocessing import StandardScaler
---- Discrete distribution of judge scores ----
plt.figure() counts = merged["judge_score"].value_counts().sort_index() plt.bar(counts.index, counts.values, width=0.6, alpha=0.7) plt.xticks([1, 2, 3, 4, 5]) plt.xlabel("Judge score") plt.ylabel("Count") plt.title(f"{aspect} – Judge score distribution (discrete)") plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_judge_bar.png") plt.close()
---- Normalized distributions overlay ----
scaler = StandardScaler() normed = scaler.fit_transform(merged[["judge_score", "gca_score"]])
plt.figure() plt.hist(normed[:, 0], bins=5, alpha=0.6, density=True, label="Judge (normalized)") plt.hist(normed[:, 1], bins=20, alpha=0.6, density=True, label="GCA (normalized)") plt.xlabel("Normalized score (z-score)") plt.ylabel("Density") plt.title(f"{aspect} – Normalized distributions") plt.legend() plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_normalized_overlay.png") plt.close()
What you’ll get
Judge bar plot → clear view of how often the model gave 1–5.
Overlay plot → judge and GCA scores normalized to mean=0, std=1, on the same axis.
Do you want me to also replace the scatter plot with something more appropriate for ordinal vs. continuous (like boxplots of GCA vs. judge category)? That might give you a clearer picture than scatter.
-
- User
For normalizing, just make the values between 0 and 1. nothing complicated
- Assistant
Perfect 👍 you want min–max normalization (scale values to [0,1]), not z-scores. That keeps the interpretation simple and works nicely for overlay plots. Here’s how to update the plotting block: pythonCopy code# ---- Discrete distribution of judge scores ---- plt.figure() counts = merged["judge_score"].value_counts().sort_index() plt.bar(counts.index, counts.values, width=0.6, alpha=0.7) plt.xticks([1, 2, 3, 4, 5]) plt.xlabel("Judge score") plt.ylabel("Count") plt.title(f"{aspect} – Judge score distribution (discrete)") plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_judge_bar.png") plt.close()
---- Min–max normalize both scores ----
def minmax(series): return (series - series.min()) / (series.max() - series.min())
merged["judge_norm"] = minmax(merged["judge_score"]) merged["gca_norm"] = minmax(merged["gca_score"])
---- Overlay normalized distributions ----
plt.figure() plt.hist(merged["judge_norm"], bins=5, alpha=0.6, density=True, label="Judge (normalized)") plt.hist(merged["gca_norm"], bins=20, alpha=0.6, density=True, label="GCA (normalized)") plt.xlabel("Normalized score (0–1)") plt.ylabel("Density") plt.title(f"{aspect} – Normalized distributions") plt.legend() plt.tight_layout() plt.savefig(f"{plot_dir}/{aspect}_normalized_overlay.png") plt.close()
✅ What this gives you:
Bar chart of judge scores (1–5).
Overlay histogram of normalized judge vs. GCA scores, both scaled into [0,1], so their distributions are directly comparable on the same axis.
Do you also want me to adjust the scatter plot into something like a boxplot (GCA distribution per judge score)? That would better respect the discrete nature of judge scores.