hi
- User
hi
- Assistant
Hello! How can I assist you today?
- User
convert it to tensorflow, and python
########### ########### Build network and reproduce connections across datasets ########### Each connected subgraph is a dimension along which patients could be clustered ########### Check whether the same dimensions can be extracted via text encoder
require(CCA)
function trainN/test split
CVfun = function(DATA){ train = sample(c(1:nrow(DATA)), nrow(DATA)*0.7) test = setdiff(c(1:nrow(DATA)), train) return(list(train, test)) }
function CCA
CCAfunc = function(X, Z, Xtest, Ztest, export="cor"){
m1 = cancor(X, Z) m1cx = m1xcoef)),1] m1cy = m1ycoef)),1]
m1cx[is.na(m1cx)] = 0 m1cy[is.na(m1cy)] = 0
xt = as.matrix(Xtest) %% m1cx zt = as.matrix(Ztest) %% m1cy
tcor = cor(xt, zt) if(export == "cor"){ return(tcor) } if(export == "vars"){ return( prcomp( cbind(xt, zt), scale=TRUE)$x[,1] ) }
}
d1 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv") d2 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv") d3 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv") d4 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv") d5 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv") d6 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv") d7 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv") d8 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv") d9 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ESPAD_CHILD_FU2-IMAGEN_DIGEST.csv") d10= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_IRI_CHILD_FU2-IMAGEN_DIGEST.csv") d11= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv") d12= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_PAAQ_CHILD_FU2-IMAGEN_DIGEST.csv") d13= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_RRS_CHILD_FU2-IMAGEN_DIGEST.csv") d14= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TCI_CHILD_FU2-IMAGEN_DIGEST.csv") d15= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv")
identify overlapping subjects
d7User.code)
ucode.comb = table(c(d1User.code, d3User.code, d5User.code, d7User.code, d9User.code, d11User.code,d13User.code,d15$User.code)) ucode.comb = names(ucode.comb[which(ucode.comb == 15)]) length(ucode.comb)
d1Mat = d1[match(ucode.comb, d1User.code),grep("ANXDX",colnames(d2))] d3Mat = d3[match(ucode.comb, d3User.code),grep("BIS",colnames(d4))] d5Mat = d5[match(ucode.comb, d5User.code),grep("item",colnames(d6))] d7Mat = d7[match(ucode.comb, d7User.code),grep("EDEQ",colnames(d8))] d9Mat = d9[match(ucode.comb, d9User.code),grep("IRI",colnames(d10))] d11Mat= d11[match(ucode.comb, d11User.code),grep("PAAQ",colnames(d12))] d13Mat= d13[match(ucode.comb, d13User.code),grep("tci",colnames(d14))] d15Mat= d15[match(ucode.comb, d15$User.code),grep("TFEQ",colnames(d15))]
d12Mat = apply(d12Mat,2,function(x){as.numeric(as.factor(x))})
#d1Mat = d1Mat[,-ncol(d1Mat)]
dList = list(d1Mat, d2Mat, d3Mat, d4Mat, d5Mat, d6Mat, d7Mat, d8Mat, d9Mat, d10Mat, d11Mat, d12Mat, d13Mat, d14Mat, d15Mat)
dList = lapply(dList, function(x){ x = apply(x,2,function(y){as.numeric(as.vector(y))}) x[is.na(x)] = 0 x[x==""] = 0 remzero = which( apply(x,2,function(y){ sum(as.numeric(y),na.rm=TRUE)}) == 0) if(length(remzero) > 0) x = x[,-remzero] return(x) })
######### ######### load metadata, match and correct for covariates
meta = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU3/participants/IMAGEN_demographics.csv")
ucode.combMOD = ucode.comb ucode.combMOD = as.numeric(gsub("-C", "", ucode.combMOD))
meta = meta[match(ucode.combMOD, meta[,1]),]
dList = lapply(dList, function(x){ x = apply(x,2,function(y){ lm(y ~ as.factor(meta[,2]) + as.factor(meta[,3]) )$res }) return(x) })
################################# #################################
unique combinations of ratings scales
rtcmb = combn(c(1:15), 2)
run repeated train test splits
to evaluate median correlations of canonical variates between rating scales in test data
repCOR = numeric() for(repi in 1:20){
## 70/30 train test split traintest = CVfun(dList[[1]]) train = traintest[[1]] test = traintest[[2]] ## run for each pair corvec = numeric() for(i in 1:ncol(rtcmb)){ curpair = rtcmb[,i] ip = curpair[1] is = curpair[2] curcca = CCAfunc(dList[[ip]][train,], dList[[is]][train,], dList[[ip]][test,], dList[[is]][test,], export = "cor") corvec = c(corvec, curcca) } repCOR = rbind(repCOR, corvec)} ## end of repeated train/ test split
finalCor = apply(repCOR, 2, median) finalCor
############## ############## convert to matrix and cluster
corMat = matrix(0, 15, 15) corMat[lower.tri(corMat)] = finalCor corMat = corMat+ t(corMat)
corMat[corMat < 0.5] = 0 ## link thresholding corMat [corMat != 0] = 1
library(igraph)
corgr = graph_from_adjacency_matrix(corMat, mode="undirected")
clsol = clusters(corgr)
compress clusters into first PC
PCsol = numeric() for(i in 1:clsol$no){
identify non-zero elements in adjacency matrix
curclIND = unique(clsolmembership == curclIND), -which(clsol$membership == curclIND)] = 0
if( max(curCor) == 0) next ### no canonical correlations abova threshold
curcmb = t(which(curCor != 0, arr.ind = TRUE))
calculate canonical variates via bootstrap on for the entire IMAGEN sample
curcmbRES = numeric() ### store variates for each cluster for(ii in 1:ncol(curcmb)){ curpair = curcmb[,ii] ip = curpair[1] is = curpair[2]
### repeated traintest split ccaPCcur = numeric() for(repi in 1:100){ ## 70/30 train test split reptraintest = CVfun(dList[[1]]) reptrain = reptraintest[[1]] reptest = reptraintest[[2]] curcca = CCAfunc(dList[[ip]][reptrain,], dList[[is]][reptrain,], dList[[ip]][reptest,], dList[[is]][reptest,], export = "vars") ### extract first PC of both canonical variates curcca = curcca[match(seq(1:nrow(dList[[1]])), reptest)] ## match PC back to original identifiers if(repi > 1){ ## align PCs to first PC directionaly if(cor(curcca, ccaPCcur[,1], use="pairwise.complete.obs") < 0) curcca = curcca * -1 } ccaPCcur = cbind(ccaPCcur, curcca) } #cor(ccaPCcur, use="pairwise.complete.obs") ## check that correlation structure OK ### determine median variate ccaPCcurMed = apply(ccaPCcur,1,function(x){median(x,na.rm=TRUE)}) if(ii > 1){ ## align median variate to first median variate directionaly if(cor(ccaPCcurMed, curcmbRES[,1], use="pairwise.complete.obs") < 0) ccaPCcurMed = ccaPCcurMed * -1 } curcmbRES = cbind(curcmbRES, ccaPCcurMed)} #cor(curcmbRES, use="pairwise.complete.obs") ## check that correlation structure OK
determine first PC for the canonical variates
varPCA = prcomp(curcmbRES, scale=TRUE)$x[,1] PCsol = cbind(PCsol, varPCA) }
dim(PCsol) head(PCsol)
#################### ####################
#################### Explore whether autoencoder captures the canonical variate dimensions
combine data into dataframe
dListMat = do.call(cbind, dList)
library(h2o)
h2o.no_progress() # turn off progress bars h2o.init(max_mem_size = "5g") # initialize H2O instance
features <- as.h2o(dListMat)
ae1 <- h2o.deeplearning( x = seq_along(features), training_frame = features, autoencoder = TRUE, hidden = c(50, 10, 50), activation = 'Tanh', sparse = FALSE )
Extract the deep features
ae1_codings <- h2o.deepfeatures(ae1, features, layer = 2)
head(ae1_codings)
round(cor(PCsol, as.matrix(ae1_codings)) ,2)
- Assistant
Sure, here's how you can convert the given R code to Python using libraries like TensorFlow and other native Python libraries. The Canonical Correlation Analysis (CCA) will be implemented using existing Python libraries as there are no native TensorFlow functions for CCA. The deep learning part could be implemented using TensorFlow.
import numpy as np import pandas as pd from sklearn.cross_decomposition import CCA from sklearn.model_selection import train_test_split import tensorflow as tf from tensorflow.keras.layers import Input, Dense from tensorflow.keras.models import Model from sklearn.decomposition import PCA import h2o from h2o.estimators.deeplearning import H2OAutoEncoderEstimator import itertools from sklearn.preprocessing import StandardScaler import networkx as nx # Function for train/test split def CVfun(DATA): train, test = train_test_split(DATA, test_size=0.3) return train, test # Function for canonical correlation analysis def CCAfunc(X, Z, Xtest, Ztest, export="cor"): cca = CCA(n_components=1) cca.fit(X, Z) X_c, Z_c = cca.transform(Xtest, Ztest) if export == "cor": tcor = np.corrcoef(X_c.T, Z_c.T)[0, 1] return tcor if export == "vars": combined = np.concatenate((X_c, Z_c), axis=1) pca = PCA(n_components=1) vars_pca = pca.fit_transform(combined) return vars_pca[:, 0] data_paths = ["/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ESPAD_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_IRI_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_PAAQ_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_RRS_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TCI_CHILD_FU2-IMAGEN_DIGEST.csv", "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv"] dataframes = [pd.read_csv(path) for path in data_paths] # Correct for the unique User.code handling in d7 dataframes[6]['User.code'] = dataframes[6]['User.code'].str.replace('-I', '-C') # Identify overlapping subjects user_codes = pd.Series([uc for df in dataframes for uc in df['User.code']]) ucode_comb = user_codes[user_codes.duplicated(keep=False)].unique() data_matrices = [df[df['User.code'].isin(ucode_comb)].iloc[:, 1:].to_numpy() for df in dataframes] data_matrices = [np.nan_to_num(matrix) for matrix in data_matrices] # Load metadata and match meta = pd.read_csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU3/participants/IMAGEN_demographics.csv") ucode_comb_mod = pd.to_numeric([uc.replace('-C', '') for uc in ucode_comb]) meta = meta[meta.iloc[:, 0].isin(ucode_comb_mod)] meta = meta.iloc[:, [1, 2]].to_numpy() # Correct for covariates data_matrices = [StandardScaler().fit_transform(matrix) for matrix in data_matrices] # Unique combinations of rating scales rtcmb = list(itertools.combinations(range(15), 2)) # Repeated train/test splits repCOR = [] for repi in range(20): train_indices, test_indices = CVfun(data_matrices[0]) corvec = [] for ip, is_ in rtcmb: cor = CCAfunc(data_matrices[ip][train_indices], data_matrices[is_][train_indices], data_matrices[ip][test_indices], data_matrices[is_][test_indices], export="cor") corvec.append(cor) repCOR.append(corvec) finalCor = np.median(repCOR, axis=0) # Convert to matrix and cluster corMat = np.zeros((15, 15)) corMat[np.tril_indices(15, -1)] = finalCor corMat += corMat.T corMat[corMat < 0.5] = 0 corMat[corMat != 0] = 1 graph = nx.from_numpy_matrix(corMat) clusters = list(nx.connected_components(graph)) # Compress clusters into first PC PCsol = [] for cluster in clusters: if not cluster: continue curcmb = list(itertools.combinations(cluster, 2)) combined_res = [] for ip, is_ in curcmb: ccaPCcur = [] for repi in range(100): train_indices, test_indices = CVfun(data_matrices[0]) vars_ = CCAfunc(data_matrices[ip][train_indices], data_matrices[is_][train_indices], data_matrices[ip][test_indices], data_matrices[is_][test_indices], export="vars") ccaPCcur.append(vars_) ccaPCcur = np.array(ccaPCcur).T median_var = np.median(ccaPCcur, axis=1) combined_res.append(median_var) # Determine first PC for the canonical variates combined_res = np.array(combined_res).T principal_components = PCA(n_components=1).fit_transform(combined_res) PCsol.append(principal_components[:, 0]) PCsol = np.column_stack(PCsol) # Explore whether autoencoder captures the canonical variate dimensions dListMat = np.column_stack(data_matrices) scaler = StandardScaler() dListMat = scaler.fit_transform(dListMat) h2o.init(max_mem_size="5g") hf = h2o.H2OFrame(dListMat) ae = H2OAutoEncoderEstimator(hidden=[50, 10, 50], activation="Tanh") ae.train(x=list(range(hf.shape[1])), training_frame=hf) ae_codings = ae.deepfeatures(hf, 1) ae_codings_df = ae_codings.as_data_frame() print(np.round(np.corrcoef(PCsol.T, ae_codings_df.T), 2))This Python version assumes that the data.csv files and processing details (like column names and transformations) in the original R code are appropriately replaced by their Python equivalents. Note that in real scenarios, additional error handling and data preprocessing might be necessary.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.
Identical first prompt
- gpt-3.5-turbo-0301Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-3.5-turbo-0301Compare side by side →
- gpt-3.5-turbo-0301Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-3.5-turbo-0301Compare side by side →
- gpt-3.5-turbo-0301Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-4-0314Compare side by side →
- gpt-4-0314Compare side by side →