Help me use classes in this script to prevent calculating the same values multiple…
- User
Help me use classes in this script to prevent calculating the same values multiple times.
""" from collections import defaultdict
import numpy as np import pandas as pd from scipy.linalg import orth from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity from sklearn.decomposition import TruncatedSVD
""" This script provides a framework for analyzing group discussions. Calculates the aspects from: https://link.springer.com/article/10.3758/s13428-018-1102-z """
TODO decide if to implement a class for variable storage
SEED = 100
for tf-idf vectorization
TODO wrap this into a class
all_data = pd.read_csv("/home/esther/PycharmProjects/Dialogos/role_discovery/data/all_contributions.csv", index_col=0) # replace with your data source vectorizer = TfidfVectorizer(ngram_range=(1, 2), min_df=1) # tune X_tfidf = vectorizer.fit_transform(all_data['text'].tolist()) # shape (n, vocab) svd = TruncatedSVD(n_components=100, random_state=SEED) LSA_DOC_VECTORS = svd.fit_transform(X_tfidf) # shape (n, dim) -> LSA-style document vectors
Build once (global) — mapping text -> list of row indices in all_data
TEXT_IDX_MAP = defaultdict(list) for idx, txt in enumerate(all_data['text'].tolist()): TEXT_IDX_MAP[txt].append(idx)
def calculate_participation(discussion_data: pd.DataFrame, participant: str) -> dict[float, list[int]]: """ Calculate participation score for a specific participant in a discussion. The logic: the discussion is split into contributions: c1, c2, ..., cn t is referred to as time: 1 <= t <= n k is the number of participants in the discussion. The participation function: p_a(t) = 1 if participant contributes at time t else 0 The participation function for a participant is the the sequence of contributions: p_a = (p_a(1), p_a(2), ..., p_a(n)) The number of contributions a participant makes is denoted as |p_a|, which is the sum of the sequence: |p_a| = p_a(1) + p_a(2) + ... + p_a(n) The sample mean participation of a participant is defined as: p_a_mean = |p_a| / n
Eventually, the participation score is calculated as: participation_score = p_a_mean - 1/k Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name discussion_id: The discussion ID Returns: dict: Participation score, participation sequence """ # Get the sequence of all authors contributors_lst = discussion_data['author'].tolist() # Total number of contributions in the discussion n_contributors = len(contributors_lst) # Number of unique participants in the discussion k = discussion_data['author'].nunique() # Create the participation sequence p_a for this participant # p_a(t) = 1 if participant contributes at time t, else 0 p_a = [1 if contributor == participant else 0 for contributor in contributors_lst] # Calculate |p_a| - total number of contributions by this participant p_a_total = sum(p_a) # Calculate sample mean participation p_a_mean = p_a_total / n_contributors # Calculate the participation score participation_score = p_a_mean - (1 / k) return {'participation_score': participation_score, 'participation_seq': p_a}def get_lsa_embedding_by_id(discussion_data, contribution_id) -> np.ndarray: text = discussion_data.loc[discussion_data['contribution_id'] == contribution_id, 'text'].iloc[0] tfidf_vec = vectorizer.transform([text]) lsa_vec = svd.transform(tfidf_vec) # shape (1,dim) return lsa_vec.ravel() # return shape (dim,)
def calc_cos_by_id(discussion_data: pd.DataFrame, id1: int, id2: int) -> float: v1 = get_lsa_embedding_by_id(discussion_data, id1) # returns (1,dim) v2 = get_lsa_embedding_by_id(discussion_data, id2) return float(cosine_similarity(v1, v2)[0,0])
def get_lsa_matrix_by_ids(discussion_data, ids: list[int]) -> np.ndarray: return np.vstack([get_lsa_embedding_by_id(discussion_data, i) for i in ids]) # shape (m,dim)
def calc_semantic_similarity(discussion_data: pd.DataFrame, contribution_id_1: int, contribution_id_2: int) -> float: """ calculate the semantic similarity between two contributions: if contribution c_t was made by a and c_u was made by b: s_a,b(t, u) = p_a(t)*p_b(u)*cos(d_t, d_u) else: s_a,b(t, u) = 0
:param discussion_data: :param contribution_1: :param contribution_2: :return: """ # TODO hold a class with these values instead of calculating multiple times # Find metadata row_1 = discussion_data[discussion_data.contribution_id == contribution_id_1].iloc[0] row_2 = discussion_data[discussion_data.contribution_id == contribution_id_2].iloc[0] # Get participation sequence values p_a_t = calculate_participation(discussion_data, row_1.author)['participation_seq'][contribution_id_1 - 1] p_b_u = calculate_participation(discussion_data, row_2.author)['participation_seq'][contribution_id_2 - 1] # Cosine similarity of texts cos_sim = calc_cos_by_id(discussion_data, contribution_id_1, contribution_id_2) return p_a_t * p_b_u * cos_simdef calc_cross_cohesion_matrix(discussion_data: pd.DataFrame, lag: int) -> tuple[list[list[float]], list[str]]: """ computes the cross-cohesion matrix for a given lag of the discussion. The element in row i and column j is the cross-cohesion value for authors i and j. For two authors a and b, the cross-cohesion value a lag Tao is defined as: c_cohesion_a,b(Tao) = (sim_a_b(1,Tao+1) + sim_a_b(2,Tao+2) + ... + sim_a_b(n-Tao, n))/norm(p_ab(Tao))
and norm(p_ab(Tao)) is defined as the sum: p_a(1)p_b(Tao+1) + p_a(2)p_b(Tao+2) + ... + p_a(n-Tao)p_b(n) :return: the cross-cohesion matrix and the list of contributors (authors) in the discussion """ # TODO test this function with real data authors = discussion_data['author'].unique().tolist() num_contributions = discussion_data.shape[0] num_authors = len(authors) res = [[0.0 for _ in range(num_authors)] for _ in range(num_authors)] # Iterate over all pairs of authors and fill the matrix for i, a in enumerate(authors): for j, b in enumerate(authors): numerator = 0.0 norm_p_ab = 0.0 for t in range(lag, num_contributions): # contributions indexed by position c_prev = discussion_data.iloc[t - lag] c_curr = discussion_data.iloc[t] # only consider contributions by a (at t-lag) and b (at t) if c_prev.author == a and c_curr.author == b: # p_a(t-lag) * p_b(t) is guaranteed to be 1 here sim = calc_cos_by_id(discussion_data, c_prev.contribution_id, c_curr.contribution_id) numerator += sim norm_p_ab += 1 # since p_a* p_b = 1 in this case # Assign cross-cohesion value if norm_p_ab > 0: res[i][j] = numerator / norm_p_ab else: res[i][j] = 0.0 return res, authorsdef calc_responsivity(discussion_data: pd.DataFrame, window: int) -> tuple[list[list[float]], list[str]]: """ Responsivity across a time window is defined: R(w) = (cross_cohesion_m(1) + cross_cohesion_m(2) + ... + cross_cohesion_m(w))/w :return: """ authors = discussion_data['author'].unique().tolist() num_authors = len(authors) res_matrix = [[0.0 for _ in range(num_authors)] for _ in range(num_authors)]
# Calculate cross-cohesion for each lag from 1 to window # Sum over lags for tau in range(1, window + 1): cross_cohesion_m, _ = calc_cross_cohesion_matrix(discussion_data, lag=tau) for i in range(num_authors): for j in range(num_authors): res_matrix[i][j] += cross_cohesion_m[i][j] # Average over w for i in range(num_authors): for j in range(num_authors): res_matrix[i][j] /= window return res_matrix, authorsdef calculate_overall_responsivity(discussion_data: pd.DataFrame, participant: str, window: int) -> float: """ Calculate overall responsivity score for a specific participant in a discussion. First, we need the responsivity matrix for the discussion. Then we take the mean of the row corresponding to the participant.
Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name discussion_id: The discussion ID Returns: float: Overall responsivity score :param discussion_data: :param participant: :param window: """ res_matrix, authors = calc_responsivity(discussion_data, window) total_res = 0 participant_idx = authors.index(participant) for i in range(len(authors)): if not i == participant_idx: # exclude self total_res += res_matrix[participant_idx][i] return total_res / (len(authors) - 1) if len(authors) > 1 else 0.0def calculate_internal_cohesion(discussion_data: pd.DataFrame, participant: str, window: int) -> float: """ Internal cohesion = diagonal value R_aa(w) from responsivity matrix R(w), averaged across all lags up to w.
Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name Returns: float: Internal cohesion score :param participant: :param discussion_data: :param window: """ # Get the cross-cohesion matrix and authors cross_cohesion_matrix, authors = calc_responsivity(discussion_data, window) author_idx = authors.index(participant) return cross_cohesion_matrix[author_idx][author_idx]def calculate_social_impact(discussion_data: pd.DataFrame, participant: str, window: int) -> float: """ Calculate social impact score for a specific participant in a discussion. By averaging over the colum vector of the responsivity matrix R(w) corresponding to the all other participants.
impact_a(w) = (R_1a(w) + R_2a(w) + ... + R_ka(w)) / (k-1) Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name discussion_id: The discussion ID Returns: float: Social impact score :param participant: :param discussion_data: :param window: """ # Get the cross-cohesion matrix and authors cross_cohesion_matrix, authors = calc_responsivity(discussion_data, window) # window=1 for social impact author_idx = authors.index(participant) # Calculate social impact as average of column corresponding to this participant total_impact = 0.0 num_participants = len(authors) for i in range(len(authors)): if i != author_idx: # exclude self total_impact += cross_cohesion_matrix[i][author_idx] return total_impact / (num_participants - 1) if num_participants > 1 else 0.0def span_given(discussion_data: pd.DataFrame, time: int) -> np.ndarray: """ Return an orthonormal basis Q for G_t (columns are orthonormal basis vectors), call get_embedding for each contribution up to time t.
Returns: Q: np.ndarray shape (dim, k) (dim = n_components if SVD used, else tfidf_dim) """ # Filter contributions up to the given time contributions = discussion_data[discussion_data['contribution_id'] < time]['text'].tolist() # compute the tf-idf embeddings for the contributions tfidf_embeddings = get_lsa_embedding_by_id(discussion_data, time) # transpose to (dim, m) then orthonormalize columns return orth(tfidf_embeddings.T) # returns (dim, k) with orthonormal columnsdef project_and_residual(given_space: np.ndarray, cur_vector: np.ndarray) -> tuple[np.ndarray, np.ndarray]: """ G: basis candidates, shape (dim, r) or (r, dim) V: contributions, shape (n_samples, dim) Returns: (proj, residual) each shape (n_samples, dim) which are equivalent to eq. 22 and 23 in the paper """ cur_vector = np.atleast_2d(cur_vector) # (n, dim) # If Q empty if given_space.size == 0: proj = np.zeros_like(cur_vector) return proj, cur_vector.copy() # project each row vector in v: proj = v @ Q @ Q.T proj = (cur_vector @ given_space) @ given_space.T residual = cur_vector - proj return proj, residual
def new_semantic_content(discussion_data: pd.DataFrame, time: int) -> float: """ Calculate the newness of the contribution at a given time point: n(c_t) = ||new_t|| / (||new_t|| + ||given_t||) :param discussion_data: :param time: :return: """ contribution = discussion_data[discussion_data['contribution_id'] == time]['text'].values[0] # Get the orthonormal basis for contributions up to time t given_space = span_given(discussion_data, time) # Project the current contribution onto the given space cur_vector = get_lsa_embedding_by_id(discussion_data, time)
# span of previous contributions proj, resid = project_and_residual(given_space, cur_vector) g_norm = np.linalg.norm(proj) n_norm = np.linalg.norm(resid) return n_norm / (n_norm + g_norm) if (n_norm + g_norm) > 0 else 0.0def calculate_newness(discussion_data: pd.DataFrame, participant: str) -> float: """ Calculate newness score for a specific participant in a discussion - First calculate the participant's contribution probabilities, then calculate the newness of each contribution, and finally average the newness scores weighted by the probabilities.
Average over all the given-new scores of this participant's contributions: newness(a) = (p_a(1)*new(c_1) + p_a(2)*new(c_2) + ... + p_a(n)*new(c_n)) / norm_p_a Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name Returns: float: Newness score """ # Get the participation sequence for this participant # Get participation probabilities participation_data = calculate_participation(discussion_data, participant) participation_seq = participation_data['participation_seq'] norm_p_a = participation_seq.sum() if norm_p_a == 0: return 0.0 # Calculate newness for each contribution # Vectorized version using apply or list comprehension newness_scores = participation_seq.index.to_series().apply( lambda t: new_semantic_content(discussion_data, t) ) # Weighted average newness_score = (participation_seq * newness_scores).sum() / norm_p_a return float(newness_score)def calc_single_density(discussion_data: pd.DataFrame, time: int) -> float: """ Calculate the communication density for a single contribution at a given time point. Defined as: d(c_t) = || lsa_vector(c_t) || / || c_t || :param discussion_data: :param time: :return: """ contribution = discussion_data[discussion_data['contribution_id'] == time]['text'].values[0] # Get the current contribution vector cur_vector = get_lsa_embedding_by_id(discussion_data, time)
# norm of the contribution vector norm_contribution = np.linalg.norm(cur_vector) return float(norm_contribution / len(contribution.split()))def calculate_communication_density(discussion_data: pd.DataFrame, participant: str) -> float: """ Calculate communication density score for a specific participant in a discussion. The average communication density over all contributions made by the participant: communication_density(a) = (d(c_1) + d(c_2) + ... + d(c_n)) / |p_a|
Args: discussion_data: DataFrame containing the discussion data participant: The participant ID/name Returns: float: Communication density score """ # Get the participation sequence for this participant participation_data = calculate_participation(discussion_data, participant) participation_seq = participation_data['participation_seq'] norm_p_a = participation_seq.sum() if norm_p_a == 0: return 0.0 # Calculate communication density for each contribution # Vectorized version using apply or list comprehension density_scores = participation_seq.index.to_series().apply( lambda t: calc_single_density(discussion_data, t) ) # Weighted average communication_density_score = (participation_seq * density_scores).sum() / norm_p_a return float(communication_density_score)def analyze_group_discussions(df: pd.DataFrame, discussion_id_col: str = 'discussion_id', participant_col: str = 'participant', output_file: str = 'discussion_analysis_results.csv') -> pd.DataFrame: """ Analyze group discussions and calculate metrics for each discussion and participant.
Args: df: Input DataFrame containing discussion data discussion_id_col: Column name for discussion ID participant_col: Column name for participant output_file: Path to save the results CSV file Returns: pd.DataFrame: Results DataFrame with calculated metrics """ # Get unique discussions and participants discussions = df[discussion_id_col].unique() # Initialize results list results = [] print(f"Analyzing {len(discussions)} discussions...") # Iterate over each discussion for i, discussion_id in enumerate(discussions): print(f"Processing discussion {i + 1}/{len(discussions)}: {discussion_id}") # Filter data for current discussion discussion_data = df[df[discussion_id_col] == discussion_id] # Get participants in this discussion participants = discussion_data[participant_col].unique() # Iterate over each participant in the discussion for participant in participants: print(f" - Analyzing participant: {participant}") # Calculate all metrics for this participant in this discussion participation_score = calculate_participation(discussion_data, participant, discussion_id) responsivity_score = calculate_overall_responsivity(discussion_data, participant, discussion_id) cohesion_score = calculate_internal_cohesion(discussion_data, participant, discussion_id) social_impact_score = calculate_social_impact(discussion_data, participant, discussion_id) newness_score = calculate_newness(discussion_data, participant, discussion_id) communication_density_score = calculate_communication_density(discussion_data, participant, discussion_id) # Store results results.append({ 'discussion_id': discussion_id, 'participant': participant, 'participation_score': participation_score, 'overall_responsivity_score': responsivity_score, 'internal_cohesion_score': cohesion_score, 'social_impact_score': social_impact_score, 'newness_score': newness_score, 'communication_density_score': communication_density_score }) # Create results DataFrame results_df = pd.DataFrame(results) # Save results to CSV results_df.to_csv(output_file, index=False) print(f"\nResults saved to: {output_file}") print(f"Total rows in results: {len(results_df)}") print(f"Columns: {list(results_df.columns)}") return results_dfdef get_summary_statistics(results_df: pd.DataFrame) -> pd.DataFrame: """ Generate summary statistics for the analysis results.
Args: results_df: DataFrame with analysis results Returns: pd.DataFrame: Summary statistics """ # Define metric columns metric_cols = [col for col in results_df.columns if col.endswith('_score')] # Calculate summary statistics summary_stats = results_df[metric_cols].describe() return summary_stats""" - Assistant
I see what you want: caching the heavy computations (TF-IDF, SVD embeddings, participation sequences, etc.) so you don't recompute them repeatedly. Here’s a class-based refactor for your script. It encapsulates preprocessing, caching, and metric calculations so expensive computations happen once per discussion, and all functions reuse them.
Class-based refactor pythonCopyEditimport numpy as np import pandas as pd from collections import defaultdict from scipy.linalg import orth from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.decomposition import TruncatedSVD from sklearn.metrics.pairwise import cosine_similarity
class DiscussionAnalyzer: def init(self, data: pd.DataFrame, n_components=100, seed=100): """ Initialize the analyzer with the discussion data. Precompute TF-IDF, SVD, and caches for later use. """ self.data = data self.seed = seed self.n_components = n_components
# Precompute TF-IDF & LSA self.vectorizer = TfidfVectorizer(ngram_range=(1, 2), min_df=1) self.X_tfidf = self.vectorizer.fit_transform(data['text'].tolist()) self.svd = TruncatedSVD(n_components=n_components, random_state=seed) self.lsa_vectors = self.svd.fit_transform(self.X_tfidf) # Map contribution_id → row index for caching self.id_to_idx = { cid: i for i, cid in enumerate(data['contribution_id']) } # Cache for participation sequences per participant self.participation_cache = {} def get_lsa_vector(self, contribution_id: int) -> np.ndarray: """Return LSA vector for a given contribution.""" idx = self.id_to_idx[contribution_id] return self.lsa_vectors[idx] def calc_cosine_similarity(self, id1: int, id2: int) -> float: """Cosine similarity between two contributions.""" v1 = self.get_lsa_vector(id1).reshape(1, -1) v2 = self.get_lsa_vector(id2).reshape(1, -1) return float(cosine_similarity(v1, v2)[0, 0]) def get_participation(self, participant: str) -> dict: """Cache and return participation sequence & score for a participant.""" if participant in self.participation_cache: return self.participation_cache[participant] authors = self.data['author'].tolist() n = len(authors) k = self.data['author'].nunique() p_seq = np.array([1 if a == participant else 0 for a in authors]) p_total = p_seq.sum() p_mean = p_total / n score = p_mean - 1 / k result = {'participation_score': score, 'participation_seq': p_seq} self.participation_cache[participant] = result return result def semantic_similarity(self, id1: int, id2: int) -> float: """Semantic similarity with participation weighting.""" row1 = self.data.loc[self.data.contribution_id == id1].iloc[0] row2 = self.data.loc[self.data.contribution_id == id2].iloc[0] p1 = self.get_participation(row1.author)['participation_seq'][self.id_to_idx[id1]] p2 = self.get_participation(row2.author)['participation_seq'][self.id_to_idx[id2]] cos_sim = self.calc_cosine_similarity(id1, id2) return p1 * p2 * cos_sim def span_given(self, time: int) -> np.ndarray: """Return orthonormal basis of contributions up to time t.""" idxs = [i for i, cid in enumerate(self.data['contribution_id']) if cid < time] if not idxs: return np.empty((self.n_components, 0)) mat = self.lsa_vectors[idxs].T # shape (dim, m) return orth(mat) def new_semantic_content(self, time: int) -> float: """Newness of contribution at time t.""" Q = self.span_given(time) v = self.get_lsa_vector(time).reshape(1, -1) if Q.size == 0: return 1.0 # all new if no prior contributions proj = v @ Q @ Q.T resid = v - proj n_norm = np.linalg.norm(resid) g_norm = np.linalg.norm(proj) return n_norm / (n_norm + g_norm) if n_norm + g_norm > 0 else 0.0 def communication_density(self, time: int) -> float: """Communication density for a single contribution.""" v = self.get_lsa_vector(time) text = self.data.loc[self.data.contribution_id == time, 'text'].iloc[0] return np.linalg.norm(v) / len(text.split())Key improvements:
LSA vectors computed once → get_lsa_vector() uses precomputed matrix.
Participation sequences cached → avoids recomputing in every metric.
Semantic similarity uses cached vectors & participation scores.
Newness & density reuse the same cached data.
Next steps
You can now add methods for cross-cohesion, responsivity, and social impact in this class — they’ll be much cleaner because the caching is already handled.
We can also vectorize some parts to make it even faster.
Do you want me to rewrite the cross-cohesion and responsivity metrics into this class too so everything is fully cached?
- User
For each discussion, the contribution_id gos from 1 to number of contributions. Therefore, this isn't needed:
# Map contribution_id → row index for caching self.id_to_idx = { cid: i for i, cid in enumerate(data['contribution_id']) } - Assistant
Ah, got it — if contribution_id is already guaranteed to be sequential from 1 to n, you don’t need the id_to_idx mapping at all. You can just use contribution_id - 1 as the row index directly because DataFrame indexing is zero-based while your IDs start at 1. Here’s the simplified version of those parts in the class: pythonCopyEditdef get_lsa_vector(self, contribution_id: int) -> np.ndarray: """Return LSA vector for a given contribution.""" return self.lsa_vectors[contribution_id - 1]
def get_participation(self, participant: str) -> dict: """Cache and return participation sequence & score for a participant.""" if participant in self.participation_cache: return self.participation_cache[participant]
authors = self.data['author'].tolist() n = len(authors) k = self.data['author'].nunique() p_seq = np.array([1 if a == participant else 0 for a in authors]) p_total = p_seq.sum() p_mean = p_total / n score = p_mean - 1 / k result = {'participation_score': score, 'participation_seq': p_seq} self.participation_cache[participant] = result return resultdef semantic_similarity(self, id1: int, id2: int) -> float: """Semantic similarity with participation weighting.""" row1 = self.data.iloc[id1 - 1] row2 = self.data.iloc[id2 - 1]
p1 = self.get_participation(row1.author)['participation_seq'][id1 - 1] p2 = self.get_participation(row2.author)['participation_seq'][id2 - 1] cos_sim = self.calc_cosine_similarity(id1, id2) return p1 * p2 * cos_simThis keeps everything indexed correctly while removing the unnecessary dictionary.
If you want, I can now also move cross-cohesion, responsivity, and the rest of the metrics into the class so they all share the same caching. Do you want me to do that? It will make the entire pipeline consistent and much faster.
- User
In this funtion:
def span_given(self, time: int) -> np.ndarray: """Return orthonormal basis of contributions up to time t.""" idxs = [i for i, cid in enumerate(self.data['contribution_id']) if cid < time] if not idxs: return np.empty((self.n_components, 0)) mat = self.lsa_vectors[idxs].T # shape (dim, m) return orth(mat)do we need both i and cid
- Assistant
No, you don’t need both i and cid if contribution_id is always sequential from 1 to n. Right now: pythonCopyEditidxs = [i for i, cid in enumerate(self.data['contribution_id']) if cid < time]
i is the DataFrame row index (0-based)
cid is the actual contribution_id (1-based)
But if contribution_id == i+1, you can just use the row index directly: pythonCopyEditdef span_given(self, time: int) -> np.ndarray: """Return orthonormal basis of contributions up to time t.""" # rows before contribution_id == time idxs = list(range(time - 1)) # because time-1 rows are before current if not idxs: return np.empty((self.n_components, 0)) mat = self.lsa_vectors[idxs].T # shape (dim, m) return orth(mat)
This is cleaner and faster since it avoids iterating over the DataFrame entirely.
I can also rewrite the newness and density methods to use this simplified indexing and caching properly. Do you want me to show you the full updated class with all metrics included?
- User
Is this correct?
""" def calc_cross_cohesion_matrix(self, lag: int) -> tuple[list[list[float]], list[str]]: """ computes the cross-cohesion matrix for a given lag of the discussion. The element in row i and column j is the cross-cohesion value for authors i and j. For two authors a and b, the cross-cohesion value a lag Tao is defined as: c_cohesion_a,b(Tao) = (sim_a_b(1,Tao+1) + sim_a_b(2,Tao+2) + ... + sim_a_b(n-Tao, n))/norm(p_ab(Tao))
and norm(p_ab(Tao)) is defined as the sum: p_a(1)p_b(Tao+1) + p_a(2)p_b(Tao+2) + ... + p_a(n-Tao)p_b(n) :return: the cross-cohesion matrix and the list of contributors (authors) in the discussion """ # TODO test this function with real data authors = self.data['author'].unique().tolist() num_contributions = len(self.data) num_authors = len(authors) res = [[0.0 for _ in range(num_authors)] for _ in range(num_authors)] # Iterate over all pairs of authors and fill the matrix for i, a in enumerate(authors): for j, b in enumerate(authors): numerator = 0.0 norm_p_ab = 0.0 for t in range(lag, num_contributions): # contributions indexed by position c_prev = self.data.iloc[t - lag] c_curr = self.data.iloc[t] # only consider contributions by a (at t-lag) and b (at t) if c_prev.author == a and c_curr.author == b: # p_a(t-lag) * p_b(t) is guaranteed to be 1 here sim = self.calc_cosine_similarity(c_prev.contribution_id, c_curr.contribution_id) numerator += sim norm_p_ab += 1 # since p_a* p_b = 1 in this case # Assign cross-cohesion value if norm_p_ab > 0: res[i][j] = numerator / norm_p_ab else: res[i][j] = 0.0 return res, authors"""
- Assistant
Your calc_cross_cohesion_matrix method is mostly correct, but there are a couple of points worth noting: ✅ Correct parts
Loop structure: Iterates over all pairs of authors correctly.
Lag logic: Uses t - lag and t for contributions at lag distance.
Norm calculation: Matches the definition pa(t−τ)pb(t)p_a(t-\tau) p_b(t)pa(t−τ)pb(t).
Cosine similarity: Uses your cached calc_cosine_similarity method, so no redundant recalculations.
⚠️ Things to check or improve
Index range:
Python is zero-based, but your contribution_id starts at 1.
When you do for t in range(lag, num_contributions), t refers to row indices, so iloc works correctly.
→ ✅ This is correct as long as num_contributions = len(self.data).
Performance:
Right now, for each (a,b) pair, you loop over all contributions again.
For large discussions, this is O(n² × m) complexity.
We can vectorize or precompute author sequences to speed this up.
Matrix type:
You return a list of lists.
Returning a NumPy array makes further calculations easier:
pythonCopyEditres = np.zeros((num_authors, num_authors))
Authors ordering:
self.data['author'].unique() keeps the first appearance order, which is fine as long as you stay consistent across methods.
Cleaner version pythonCopyEditdef calc_cross_cohesion_matrix(self, lag: int) -> tuple[np.ndarray, list[str]]: """ Compute cross-cohesion matrix for a given lag. """ authors = self.data['author'].unique().tolist() num_contributions = len(self.data) num_authors = len(authors) res = np.zeros((num_authors, num_authors))
for i, a in enumerate(authors): for j, b in enumerate(authors): numerator = 0.0 norm_p_ab = 0.0 for t in range(lag, num_contributions): c_prev = self.data.iloc[t - lag] c_curr = self.data.iloc[t] if c_prev.author == a and c_curr.author == b: sim = self.calc_cosine_similarity( c_prev.contribution_id, c_curr.contribution_id ) numerator += sim norm_p_ab += 1 res[i, j] = numerator / norm_p_ab if norm_p_ab > 0 else 0.0 return res, authorsThis works exactly the same, but with NumPy arrays for easier future computations.
If you want, I can show you a vectorized version of cross-cohesion that avoids all those nested loops and speeds things up dramatically. Do you want me to do that? It will make a big difference for large discussions.