Skip to content
TheChatArchiveThe Chat Archive
ChatGPT (model unspecified)· OpenAIHugging Face ShareLM (browser-extension contributions)

Write a function that calculates how the 5 versions of density correlate with each…

  1. User

    Write a function that calculates how the 5 versions of density correlate with each other. Plot a heat map. The 5 versions are the 4 + judge_score

    """ from time import sleep import pandas as pd import os from openai import OpenAI import json from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt from scipy.stats import pearsonr, spearmanr

    from prompts.participation import PARTICIPATION_SYS from prompts.responsivity import RESPONSIVITY_SYS from prompts.internal_cohesion import IN_COHESION_SYS from prompts.social_impact import SOCIAL_IMPACT_SYS from prompts.newness import NEWNESS_SYS from prompts.communication_density import COMMUNICATION_DENSE_SYS

    """ Evaluate each GCA aspect using LLM as a judge.

    First, for each discussion and participant, create a text file in the following

    """ API_KEY_OPENAI = "sk-proj-XXX" os.environ[ "OPENAI_API_KEY"] = API_KEY_OPENAI

    SEED = 30 TEST_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/group_discussion_analysis_rep/evalGCA/test_data.csv" DATA_PATH = "/home/esther/PycharmProjects/Dialogos/role_discovery/data"

    MODELS = {"GPT4_1": "gpt-4.1-2025-04-14", }

    PROMPTS = { "participation": PARTICIPATION_SYS, "responsivity": RESPONSIVITY_SYS, "internal_cohesion": IN_COHESION_SYS, "social_impact": SOCIAL_IMPACT_SYS, "newness": NEWNESS_SYS, "communication_density": COMMUNICATION_DENSE_SYS }

    def extract_score(response_text: str): """ Parse model response and extract rationale + score. If parsing fails, fall back to raw text. """ try: # Try to parse JSON data = json.loads(response_text) score = int(data.get("score")) rationale = data.get("rationale", "") return score, rationale

    except Exception as e:
        print(f"Response in unexpected format: {response_text}.")
        return None, response_text.strip()
    

    def load_data(path: str) -> pd.DataFrame: """ load the csv with all discussion turns :param path: :return: """ return pd.read_csv(path)

    def create_judge_files(df: pd.DataFrame, output_dir: str): """ Create text files for each discussion and participant for LLM judging in the format: Discussion: <discussion_name> Participant: <participant_name> Text:

    :param df:
    :param output_dir:
    :return:
    """
    # create directory if it does not exist
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    discussions = df['discussion'].unique()
    for discussion in discussions:
        df_disc = df[df['discussion'] == discussion]
        participants = df_disc['author'].unique()
        for participant in participants:
            filename = f"{output_dir}/{discussion}_{participant}.txt"
            with open(filename, 'w') as f:
                f.write(f"Discussion: {discussion}\n")
                f.write(f"Participant: {participant}\n")
                f.write(f"Content:\n")
                # for each turn, write "<author>: <text>"
                for idx, row in df_disc.iterrows():
                    f.write(f"\n{row['author']}: {row['text']}\n")
    
    pass
    

    def main_preprocess(): data = load_data("/home/esther/PycharmProjects/Dialogos/role_discovery/data/turn_all_contributions.csv") create_judge_files(data, "/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files")

    def main_run_judge(judge_data_dir: str, test_data: pd.DataFrame, split: str, aspect: str, model: str): """ Run the LLM judge on the test data for the given aspect. For each discussion and participant in the test data, load the corresponding text file and run the LLM judge prompt for the given aspect. Save the results in a csv file.

    :param split:
    :param model:
    :param test_data:
    :param judge_data_dir:
    :param aspect:
    :return:
    """
    # ✅ Save after every iteration (safe checkpointing)
    out_path = f"outs/{split}/{model}/{aspect}/judge_results_{aspect}.csv"
    
    # Load history if file exists
    if os.path.exists(out_path):
        existing = pd.read_csv(out_path).dropna(subset=["score"])
        done_pairs = set(zip(existing["discussion"], existing["participant"]))
        results = existing.to_dict("records")
        print(f"Loaded history: {len(done_pairs)} completed evaluations")
    else:
        # create directory if it does not exist
        dir = "/".join(out_path.split("/")[:-1])
        os.makedirs(dir, exist_ok=True)
        done_pairs = set()
        results = []
        print("No history found, starting fresh")
    
    # Go through unique (discussion, participant) pairs in the test data
    discussion_author_pairs = test_data[["discussion", "author"]].drop_duplicates()
    
    for _, row in discussion_author_pairs.iterrows():
        discussion = row["discussion"]
        participant = row["author"]
    
        if (discussion, participant) in done_pairs:
            # todo only skip if score is not null - search results for this pair and check score
            if any(r for r in results if r["discussion"] == discussion and
                                         r["participant"] == participant and
                                         r["score"] is not None and r["score"] > 0):
                continue  # Skip already completed
    
        filename = f"{judge_data_dir}/{discussion}_{participant}.txt"
        if not os.path.exists(filename):
            print(f"Warning: missing file {filename}")
            continue
    
        with open(filename, "r") as f:
            user_prompt = f.read()
    
        try:
            client = OpenAI()
            response = client.responses.create(
                model=MODELS[model],
                input=[
                    {"role": "system", "content": [{"type": "input_text", "text": PROMPTS[aspect]}]},
                    {"role": "user", "content": [{"type": "input_text", "text": user_prompt}]},
                ],
                text={},
                reasoning={},
                tools=[],
                temperature=0,
                max_output_tokens=3000,
                top_p=1,
                store=True,
            )
            res = response.output[0].content[0].text
            score, rationale = extract_score(res)
    
        except Exception as e:
            print(f"Error for {discussion}-{participant}: {e}")
            score = None
            rationale = None
    
        results.append({
            "discussion": discussion,
            "participant": participant,
            "aspect": aspect,
            "score": score,
            "rationale": rationale
        })
        sleep(0.3)  # to avoid rate limits
    
        pd.DataFrame(results).to_csv(out_path, index=False)
        print(f"Progress saved: {discussion}-{participant}")
    
    # sort results by discussion, participant
    final_df = pd.DataFrame(results).sort_values(by=["discussion", "participant"])
    final_df.to_csv(out_path, index=False)
    print(f"Finished. Results saved to {out_path}")
    

    def create_test_split(blocks: str, size: float) -> pd.DataFrame: """ If the test data already exists, just load it. Otherwise,load the data and create a test split of the given size

    All discussions selected must have at least 3 participants and at least 15 contributions.
    
    :param blocks: "etu", "du" or "turn"
    :param size:
    :return:
    """
    # If it already exists, just load
    if size < 1. and os.path.exists(TEST_PATH):
        print(f"Loading existing test split")
        return pd.read_csv(TEST_PATH)
    
    # Map block type to file path
    block2file = {
        "etu": f"{DATA_PATH}/all_contributions.csv",
        "du": f"{DATA_PATH}/du_all_contributions.csv",
        "turn": f"{DATA_PATH}/turn_all_contributions.csv",
    }
    if blocks not in block2file:
        raise ValueError(f"Unknown blocks type: {blocks}")
    
    # Load data
    df = pd.read_csv(block2file[blocks])
    
    # filter out discussions with at least 3 participants and at least 15 contributions
    valid_discussions = []
    for disc, g in df.groupby("discussion"):
        n_participants = g["author"].nunique()
        n_cont = len(g)
        if n_participants >= 3 and n_cont >= 15:
            valid_discussions.append(disc)
    
    df_valid = df[df["discussion"].isin(valid_discussions)]
    
    # if size == 1.0, return all valid data
    if size == 1.0:
        return df_valid
    
    # Sample discussions for test set
    discussions = df_valid["discussion"].unique()
    print("Number of unique valid discussions:", df_valid["discussion"].nunique())
    _, test_idxs = train_test_split(
        discussions, test_size=size, random_state=SEED
    )
    print("Size of test split:", len(test_idxs))
    
    test_df = df_valid[df_valid["discussion"].isin(test_idxs)]
    
    # Save for reuse
    test_df.to_csv(TEST_PATH, index=False)
    print(f"Saved test split")
    
    return test_df
    

    def judge_main(split="test"): if split == "test": judge_data = create_test_split(blocks="du", size=.20) else: judge_data = create_test_split(blocks="du", size=1.)

    for aspect in PROMPTS.keys():
        print(f"\nEvaluating aspect: {aspect}\n")
        main_run_judge(test_data=judge_data,
                       aspect=aspect,
                       split=split,
                       model="GPT4_1",
                       judge_data_dir="/home/esther/PycharmProjects/Dialogos/role_discovery/data/judge_files/")
    

    def judge_evaluation(): """ Evaluate the judge results against the GCA scores. 1. Load the judge results for each aspect 2. Load the GCA scores 3. For each aspect, - compute the correlation between the judge scores and the GCA scores - plot the distributions of the judge scores and the GCA scores - plot the scatter plot of the judge scores vs the GCA scores 4. Print the results :return: """ model = "GPT4_1" base_out_dir = f"outs/all/{model}" os.makedirs(base_out_dir, exist_ok=True)

    gca_df = pd.read_csv(f"{DATA_PATH}/du_results_6_aspects_129_discussions.csv")
    # normalize column names for merge
    if "discussion_id" in gca_df.columns:
        gca_df = gca_df.rename(columns={"discussion_id": "discussion"})
    
    # Map aspects → GCA score column
    aspect2col = {
        "participation": "participation_score",
        "responsivity": "overall_responsivity_score",
        "internal_cohesion": "internal_cohesion_score",
        "social_impact": "social_impact_score",
        "newness": "newness_score",
        "communication_density": "communication_density_score",
    }
    
    summary_records = []
    
    for aspect, gca_col in aspect2col.items():
        judge_path = f"{base_out_dir}/{aspect}/judge_results_{aspect}.csv"
        if not os.path.exists(judge_path):
            print(f"Skipping {aspect}: no judge file at {judge_path}")
            continue
    
        judge_df = pd.read_csv(judge_path)
    
        merged = pd.merge(
            judge_df,
            gca_df[["discussion", "participant", gca_col]],
            on=["discussion", "participant"],
            how="inner",
        )
    
        merged = merged.rename(columns={"score": "judge_score", gca_col: "gca_score"})
        merged = merged.dropna(subset=["judge_score", "gca_score"])
    
        # correlations
        pearson_r, pearson_p = pearsonr(merged["judge_score"], merged["gca_score"])
        n = len(merged)
    
        print(f"\nAspect: {aspect} | N={n}")
        print(f" Pearson r = {pearson_r:.3f} (p={pearson_p:.3g})")
    
        summary_records.append({
            "aspect": aspect,
            "n": n,
            "pearson_r": pearson_r,
            "pearson_p": pearson_p,
        })
    
        # plot dir
        plot_dir = f"{base_out_dir}/{aspect}/plots"
        os.makedirs(plot_dir, exist_ok=True)
    
        max_judge_score = merged["judge_score"].max()
        max_gca_score = merged["gca_score"].max()
        min_judge_score = merged["judge_score"].min()
        min_gca_score = merged["gca_score"].min()
        # merged["judge_norm"] = merged["judge_score"].apply(lambda x: x/max_judge_score)
        # merged["gca_norm"] = merged["gca_score"].apply(lambda x: x/max_gca_score)
        merged["judge_norm"] = merged["judge_score"].apply(
            lambda x: (x - min_judge_score) / (max_judge_score - min_judge_score)
        )
        merged["gca_norm"] = merged["gca_score"].apply(
            lambda x: (x - min_gca_score) / (max_gca_score - min_gca_score)
        )
    
        # ---- Overlay normalized distributions ----
        plt.figure()
        plt.hist(merged["judge_norm"], bins=5, alpha=0.6,
                 # density=True,
                 label="Judge (normalized)",
                 edgecolor="grey"  # outlines the bars
                 )
        plt.hist(merged["gca_norm"], bins=20, alpha=0.6,
                 # density=True,
                 label="GCA (normalized)"
                 , edgecolor="grey")  # outlines the bars
        plt.xlabel("Normalized score (0–1)")
        plt.ylabel("Density")
        plt.title(f"{aspect} – Normalized distributions")
        plt.legend()
        plt.tight_layout()
        plt.savefig(f"{plot_dir}/{aspect}_normalized_overlay.png")
        plt.show()
        plt.close()
    
        # # histograms
        # plt.figure()
        # plt.hist(merged["judge_score"], bins=10)
        # plt.title(f"{aspect} – Judge scores")
        # plt.savefig(f"{plot_dir}/{aspect}_judge_hist.png")
        # plt.show()
        # plt.close()
        #
        # plt.figure()
        # plt.hist(merged["gca_score"], bins=10)
        # plt.title(f"{aspect} – GCA scores")
        # plt.savefig(f"{plot_dir}/{aspect}_gca_hist.png")
        # plt.show()
        # plt.close()
    
        # scatter
        plt.figure()
        plt.scatter(
            merged["gca_score"],
            merged["judge_score"],
            s=10,  # small dots
            alpha=0.7,  # transparency so overlaps look darker
        )
        plt.title(f"{aspect} – Judge vs GCA")
        # make axis limits match min and max of both scores acordingly
        plt.xlim(min_gca_score, max_gca_score)
        plt.ylim(min_judge_score - .1, max_judge_score + .1)
        # y values should be integers
        plt.yticks(range(0, 6))
        plt.xlabel("GCA score")
        plt.ylabel("Judge score")
        plt.savefig(f"{plot_dir}/{aspect}_scatter.png")
        plt.show()
        plt.close()
    
        # save merged data
        merged_out = f"{base_out_dir}/{aspect}/merged_judge_gca_{aspect}.csv"
        merged.to_csv(merged_out, index=False)
    
    # summary table
    if summary_records:
        summary_df = pd.DataFrame(summary_records)
        summary_path = f"{base_out_dir}/judge_gca_summary.csv"
        summary_df.to_csv(summary_path, index=False)
        print("\n=== Overall Summary ===")
        print(summary_df.to_string(index=False))
    

    def density_eval(): """ Evaluate the 4 versions of communication density. 1. Load the judge results for 9 aspects 2. Filter the columns for the 4 density versions 3. For each version, - compute the correlation between the judge scores and the GCA scores - plot the distributions of the judge scores and the GCA scores - plot the scatter plot of the judge scores vs the GCA scores 4. Print the results

    :return:
    """
    model = "GPT4_1"
    base_out_dir = f"outs/all/{model}"
    os.makedirs(base_out_dir, exist_ok=True)
    
    # Load GCA df
    gca_path = f"{DATA_PATH}/du_results_9_aspects_129_discussions.csv"
    if not os.path.exists(gca_path):
        print(f"GCA file not found at {gca_path}. Aborting density_eval.")
        return
    
    gca_df = pd.read_csv(gca_path)
    # normalize column names
    if "discussion_id" in gca_df.columns:
        gca_df = gca_df.rename(columns={"discussion_id": "discussion"})
    
    # density columns
    density2col = {
        "communication_density": "tf_idf_norm_cd",
        "tf_idf_sum_communication_density": "tf_idf_sum_cd",
        "pos_communication_density": "pos_cd",
        "llm_concise_communication_density": "llm_concise_cd",
    }
    summary_records = []
    
    # compare each density column to the judge scores for "communication_density"
    judge_path = f"{base_out_dir}/communication_density/judge_results_communication_density.csv"
    if not os.path.exists(judge_path):
        print(f"Judge file not found at {judge_path}. Aborting density_eval.")
        return
    
    judge_df = pd.read_csv(judge_path)
    
    # for each density version
    for density_col in density2col.keys():
        merged = pd.merge(
            judge_df,
            gca_df[["discussion", "participant", density_col]],
            on=["discussion", "participant"],
            how="inner",
        )
    
        merged = merged.rename(columns={"score": "judge_score",
                                        density_col: density2col[density_col]})
        col = density2col[density_col]
        merged = merged.dropna(subset=["judge_score", col])
    
        # correlations
        pearson_r, pearson_p = pearsonr(merged["judge_score"], merged[col])
        n = len(merged)
    
        print(f"\nDensity version: {col} | N={n}")
        print(f" Pearson r = {pearson_r:.3f} (p={pearson_p:.3g})")
    
        summary_records.append({
            "version": col,
            "n": n,
            "pearson_r": pearson_r,
            "pearson_p": pearson_p,
        })
    
        # plot dir
        plot_dir = f"{base_out_dir}/{col}/plots"
        os.makedirs(plot_dir, exist_ok=True)
    
        max_judge_score = merged["judge_score"].max()
        max_gca_score = merged[col].max()
        min_judge_score = merged["judge_score"].min()
        min_gca_score = merged[col].min()
        # merged["judge_norm"] = merged["judge_score"].apply(lambda x: x / max_judge_score)
        # merged["gca_norm"] = merged["gca_score"].apply(lambda x: x / max_gca_score)
        merged["judge_norm"] = (merged["judge_score"] - min_judge_score) / (max_judge_score - min_judge_score)
        merged["gca_norm"] = (merged[col] - min_gca_score) / (max_gca_score - min_gca_score)
    
        # ---- Overlay normalized distributions ----
        plt.figure()
        plt.hist(merged["judge_norm"], bins=5, alpha=0.6,
                 # density=True,
                 label="Judge (normalized)",
                 edgecolor="grey"  # outlines the bars
                 )
        plt.hist(merged["gca_norm"], bins=20, alpha=0.6,
                 # density=True,
                 label="GCA (normalized)"
                 , edgecolor="grey")  # outlines the bars
        plt.xlabel("Normalized score (0–1)")
        plt.ylabel("Density")
        plt.title(f"{col} – Normalized distributions")
        plt.legend()
        plt.tight_layout()
        plt.savefig(f"{plot_dir}/{col}_normalized_overlay.png")
        plt.show()
        plt.close()
    
        # scatter
        plt.figure()
        plt.scatter(
            merged[col],
            merged["judge_score"],
            s=10,  # small dots
            alpha=0.7,  # transparency so overlaps look darker
        )
        plt.title(f"{col} – Judge vs GCA")
        # make axis limits match min and max of both scores acordingly
        plt.xlim(min_gca_score, max_gca_score)
        plt.ylim(min_judge_score - .1, max_judge_score + .1)
        # y values should be integers
        plt.yticks(range(0, 6))
        plt.xlabel("GCA score")
        plt.ylabel("Judge score")
        plt.savefig(f"{plot_dir}/{col}_scatter.png")
        plt.show()
        plt.close()
    
        # save merged data
        merged_out = f"{base_out_dir}/{col}/merged_judge_gca_{col}.csv"
        merged.to_csv(merged_out, index=False)
    
    # summary table
    if summary_records:
        summary_df = pd.DataFrame(summary_records)
        summary_path = f"{base_out_dir}/density_summary.csv"
        summary_df.to_csv(summary_path, index=False)
        print("\n=== Overall Summary ===")
        print(summary_df.to_string(index=False))
    

    if name == 'main': """

    """
    # judge_main(split="all")
    # judge_evaluation()
    density_eval()
    

    """

  2. Assistant

    Got it ✅ You want a function that calculates how the five versions of communication density (the four GCA variants + the judge score) correlate with each other, and then plots a heatmap. Here’s how you can extend your code: pythonCopy codeimport seaborn as sns import matplotlib.pyplot as plt

    def density_correlation_heatmap(): """ Compute correlations among the 5 versions of communication density (the 4 GCA variants + judge_score). Plot a heatmap of Pearson correlations. """ model = "GPT4_1" base_out_dir = f"outs/all/{model}"

    # Load GCA df
    gca_path = f"{DATA_PATH}/du_results_9_aspects_129_discussions.csv"
    judge_path = f"{base_out_dir}/communication_density/judge_results_communication_density.csv"
    
    if not os.path.exists(gca_path) or not os.path.exists(judge_path):
        print("Required files not found. Aborting density_correlation_heatmap.")
        return
    
    gca_df = pd.read_csv(gca_path)
    judge_df = pd.read_csv(judge_path)
    
    # Normalize column names
    if "discussion_id" in gca_df.columns:
        gca_df = gca_df.rename(columns={"discussion_id": "discussion"})
    
    # Map the density versions
    density2col = {
        "tf_idf_norm_cd": "TF-IDF Norm",
        "tf_idf_sum_cd": "TF-IDF Sum",
        "pos_cd": "POS-based",
        "llm_concise_cd": "LLM Concise",
    }
    
    # Merge judge + GCA data
    merged = judge_df.rename(columns={"score": "judge_score"})
    merged = pd.merge(
        merged,
        gca_df[["discussion", "participant"] + list(density2col.keys())],
        on=["discussion", "participant"],
        how="inner"
    ).dropna(subset=["judge_score"] + list(density2col.keys()))
    
    # Select only the relevant columns
    corr_df = merged[["judge_score"] + list(density2col.keys())]
    corr_df = corr_df.rename(columns={"judge_score": "Judge"})
    
    # Compute correlation matrix
    corr_matrix = corr_df.corr(method="pearson")
    
    # Plot heatmap
    plt.figure(figsize=(8, 6))
    sns.heatmap(corr_matrix, annot=True, cmap="coolwarm", center=0,
                xticklabels=corr_matrix.columns,
                yticklabels=corr_matrix.columns,
                fmt=".2f")
    plt.title("Correlation Heatmap – Communication Density Versions", fontsize=14)
    plt.tight_layout()
    plt.savefig(f"{base_out_dir}/density_correlation_heatmap.png")
    plt.show()
    
    return corr_matrix
    

    This function will:

    Load the judge results and the four GCA communication density versions.

    Merge them into one dataframe.

    Compute Pearson correlations between all 5.

    Plot a seaborn heatmap with annotated values.

    Save the figure under outs/all/GPT4_1/density_correlation_heatmap.png.

    Do you also want me to add Spearman correlation as an alternative matrix (maybe in a second heatmap)?