Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

hi

  1. User

    hi

  2. Assistant

    Hello! How can I assist you today?

  3. User

    convert it to tensorflow, and python

    ########### ########### Build network and reproduce connections across datasets ########### Each connected subgraph is a dimension along which patients could be clustered ########### Check whether the same dimensions can be extracted via text encoder

    require(CCA)

    function trainN/test split

    CVfun = function(DATA){ train = sample(c(1:nrow(DATA)), nrow(DATA)*0.7) test = setdiff(c(1:nrow(DATA)), train) return(list(train, test)) }

    function CCA

    CCAfunc = function(X, Z, Xtest, Ztest, export="cor"){

    m1 = cancor(X, Z) m1cx = m1xcoef[match(colnames(Xtest),rownames(m1xcoef[match(colnames(Xtest), rownames(m1xcoef)),1] m1cy = m1ycoef[match(colnames(Ztest),rownames(m1ycoef[match(colnames(Ztest), rownames(m1ycoef)),1]

    m1cx[is.na(m1cx)] = 0 m1cy[is.na(m1cy)] = 0

    xt = as.matrix(Xtest) %% m1cx zt = as.matrix(Ztest) %% m1cy

    tcor = cor(xt, zt) if(export == "cor"){ return(tcor) } if(export == "vars"){ return( prcomp( cbind(xt, zt), scale=TRUE)$x[,1] ) }

    }

    d1 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv") d2 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv") d3 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv") d4 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv") d5 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv") d6 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv") d7 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv") d8 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv") d9 = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ESPAD_CHILD_FU2-IMAGEN_DIGEST.csv") d10= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_IRI_CHILD_FU2-IMAGEN_DIGEST.csv") d11= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv") d12= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_PAAQ_CHILD_FU2-IMAGEN_DIGEST.csv") d13= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_RRS_CHILD_FU2-IMAGEN_DIGEST.csv") d14= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TCI_CHILD_FU2-IMAGEN_DIGEST.csv") d15= read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv")

    identify overlapping subjects

    d7User.code=gsub("−I","−C",d7User.code = gsub("-I", "-C", d7User.code)

    ucode.comb = table(c(d1User.code,d2User.code, d2User.code, d3User.code,d4User.code, d4User.code, d5User.code,d6User.code, d6User.code, d7User.code,d8User.code, d8User.code, d9User.code,d10User.code, d10User.code, d11User.code,d12User.code,d12User.code,d13User.code,d14User.code,d14User.code,d15$User.code)) ucode.comb = names(ucode.comb[which(ucode.comb == 15)]) length(ucode.comb)

    d1Mat = d1[match(ucode.comb, d1User.code),grep("adrs",colnames(d1))]d2Mat=d2[match(ucode.comb,d2User.code),grep("adrs",colnames(d1))] d2Mat = d2[match(ucode.comb, d2User.code),grep("ANXDX",colnames(d2))] d3Mat = d3[match(ucode.comb, d3User.code),grep("audit",colnames(d3))]d4Mat=d4[match(ucode.comb,d4User.code),grep("audit",colnames(d3))] d4Mat = d4[match(ucode.comb, d4User.code),grep("BIS",colnames(d4))] d5Mat = d5[match(ucode.comb, d5User.code),grep("CAPE42",colnames(d5))]d6Mat=d6[match(ucode.comb,d6User.code),grep("CAPE42",colnames(d5))] d6Mat = d6[match(ucode.comb, d6User.code),grep("item",colnames(d6))] d7Mat = d7[match(ucode.comb, d7User.code),grep("CTQ",colnames(d7))]d8Mat=d8[match(ucode.comb,d8User.code),grep("CTQ",colnames(d7))] d8Mat = d8[match(ucode.comb, d8User.code),grep("EDEQ",colnames(d8))] d9Mat = d9[match(ucode.comb, d9User.code),grep("SCID",colnames(d9))]d10Mat=d10[match(ucode.comb,d10User.code),grep("SCID",colnames(d9))] d10Mat= d10[match(ucode.comb, d10User.code),grep("IRI",colnames(d10))] d11Mat= d11[match(ucode.comb, d11User.code),grep("JVQ",colnames(d11))]d12Mat=d12[match(ucode.comb,d12User.code),grep("JVQ",colnames(d11))] d12Mat= d12[match(ucode.comb, d12User.code),grep("PAAQ",colnames(d12))] d13Mat= d13[match(ucode.comb, d13User.code),grep("RRS",colnames(d13))]d14Mat=d14[match(ucode.comb,d14User.code),grep("RRS",colnames(d13))] d14Mat= d14[match(ucode.comb, d14User.code),grep("tci",colnames(d14))] d15Mat= d15[match(ucode.comb, d15$User.code),grep("TFEQ",colnames(d15))]

    d12Mat = apply(d12Mat,2,function(x){as.numeric(as.factor(x))})

    #d1Mat = d1Mat[,-ncol(d1Mat)]

    dList = list(d1Mat, d2Mat, d3Mat, d4Mat, d5Mat, d6Mat, d7Mat, d8Mat, d9Mat, d10Mat, d11Mat, d12Mat, d13Mat, d14Mat, d15Mat)

    dList = lapply(dList, function(x){ x = apply(x,2,function(y){as.numeric(as.vector(y))}) x[is.na(x)] = 0 x[x==""] = 0 remzero = which( apply(x,2,function(y){ sum(as.numeric(y),na.rm=TRUE)}) == 0) if(length(remzero) > 0) x = x[,-remzero] return(x) })

    ######### ######### load metadata, match and correct for covariates

    meta = read.csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU3/participants/IMAGEN_demographics.csv")

    ucode.combMOD = ucode.comb ucode.combMOD = as.numeric(gsub("-C", "", ucode.combMOD))

    meta = meta[match(ucode.combMOD, meta[,1]),]

    dList = lapply(dList, function(x){ x = apply(x,2,function(y){ lm(y ~ as.factor(meta[,2]) + as.factor(meta[,3]) )$res }) return(x) })

    ################################# #################################

    unique combinations of ratings scales

    rtcmb = combn(c(1:15), 2)

    run repeated train test splits

    to evaluate median correlations of canonical variates between rating scales in test data

    repCOR = numeric() for(repi in 1:20){

    ## 70/30 train test split
    traintest = CVfun(dList[[1]])
    train = traintest[[1]]
    test  = traintest[[2]]
    
    ## run for each pair
    
    corvec = numeric()
    for(i in 1:ncol(rtcmb)){
      curpair = rtcmb[,i]
      ip = curpair[1]
      is = curpair[2]
      curcca = CCAfunc(dList[[ip]][train,], dList[[is]][train,], 
                       dList[[ip]][test,], dList[[is]][test,], export = "cor")  
      corvec = c(corvec, curcca)
    }
    
    repCOR = rbind(repCOR, corvec)
    

    } ## end of repeated train/ test split

    finalCor = apply(repCOR, 2, median) finalCor

    ############## ############## convert to matrix and cluster

    corMat = matrix(0, 15, 15) corMat[lower.tri(corMat)] = finalCor corMat = corMat+ t(corMat)

    corMat[corMat < 0.5] = 0 ## link thresholding corMat [corMat != 0] = 1

    library(igraph)

    corgr = graph_from_adjacency_matrix(corMat, mode="undirected")

    clsol = clusters(corgr)

    compress clusters into first PC

    PCsol = numeric() for(i in 1:clsol$no){

    identify non-zero elements in adjacency matrix

    curclIND = unique(clsolmembership)[i]curCor=corMatcurCor[−which(clsolmembership)[i] curCor = corMat curCor[-which(clsolmembership == curclIND), -which(clsol$membership == curclIND)] = 0

    if( max(curCor) == 0) next ### no canonical correlations abova threshold

    curcmb = t(which(curCor != 0, arr.ind = TRUE))

    calculate canonical variates via bootstrap on for the entire IMAGEN sample

    curcmbRES = numeric() ### store variates for each cluster for(ii in 1:ncol(curcmb)){ curpair = curcmb[,ii] ip = curpair[1] is = curpair[2]

    ### repeated traintest split
    
    ccaPCcur = numeric()
    for(repi in 1:100){
        ## 70/30 train test split
        reptraintest = CVfun(dList[[1]])
        reptrain = reptraintest[[1]]
        reptest  = reptraintest[[2]]
    
        curcca = CCAfunc(dList[[ip]][reptrain,], dList[[is]][reptrain,], 
                     dList[[ip]][reptest,], dList[[is]][reptest,], export = "vars")   ### extract first PC of both canonical variates
        curcca = curcca[match(seq(1:nrow(dList[[1]])), reptest)] ## match PC back to original identifiers
        
        if(repi > 1){   ## align PCs to first PC directionaly
          if(cor(curcca, ccaPCcur[,1], use="pairwise.complete.obs") < 0) curcca = curcca * -1
        }
        
        ccaPCcur = cbind(ccaPCcur, curcca)
    }    
    #cor(ccaPCcur, use="pairwise.complete.obs")       ## check that correlation structure OK
        
    ### determine median variate
    
    ccaPCcurMed = apply(ccaPCcur,1,function(x){median(x,na.rm=TRUE)})
    
    if(ii > 1){   ## align median variate to first median variate directionaly
      if(cor(ccaPCcurMed, curcmbRES[,1], use="pairwise.complete.obs") < 0) ccaPCcurMed = ccaPCcurMed * -1
    }
    
    curcmbRES = cbind(curcmbRES, ccaPCcurMed)
    

    } #cor(curcmbRES, use="pairwise.complete.obs") ## check that correlation structure OK

    determine first PC for the canonical variates

    varPCA = prcomp(curcmbRES, scale=TRUE)$x[,1] PCsol = cbind(PCsol, varPCA) }

    dim(PCsol) head(PCsol)

    #################### ####################

    #################### Explore whether autoencoder captures the canonical variate dimensions

    combine data into dataframe

    dListMat = do.call(cbind, dList)

    library(h2o)

    h2o.no_progress() # turn off progress bars h2o.init(max_mem_size = "5g") # initialize H2O instance

    features <- as.h2o(dListMat)

    ae1 <- h2o.deeplearning( x = seq_along(features), training_frame = features, autoencoder = TRUE, hidden = c(50, 10, 50), activation = 'Tanh', sparse = FALSE )

    Extract the deep features

    ae1_codings <- h2o.deepfeatures(ae1, features, layer = 2)

    head(ae1_codings)

    round(cor(PCsol, as.matrix(ae1_codings)) ,2)

  4. Assistant

    Sure, here's how you can convert the given R code to Python using libraries like TensorFlow and other native Python libraries. The Canonical Correlation Analysis (CCA) will be implemented using existing Python libraries as there are no native TensorFlow functions for CCA. The deep learning part could be implemented using TensorFlow.

    import numpy as np
    import pandas as pd
    from sklearn.cross_decomposition import CCA
    from sklearn.model_selection import train_test_split
    import tensorflow as tf
    from tensorflow.keras.layers import Input, Dense
    from tensorflow.keras.models import Model
    from sklearn.decomposition import PCA
    import h2o
    from h2o.estimators.deeplearning import H2OAutoEncoderEstimator
    import itertools
    from sklearn.preprocessing import StandardScaler
    import networkx as nx
    
    # Function for train/test split
    def CVfun(DATA):
        train, test = train_test_split(DATA, test_size=0.3)
        return train, test
    
    # Function for canonical correlation analysis
    def CCAfunc(X, Z, Xtest, Ztest, export="cor"):
        cca = CCA(n_components=1)
        cca.fit(X, Z)
        X_c, Z_c = cca.transform(Xtest, Ztest)
        
        if export == "cor":
            tcor = np.corrcoef(X_c.T, Z_c.T)[0, 1]
            return tcor
        if export == "vars":
            combined = np.concatenate((X_c, Z_c), axis=1)
            pca = PCA(n_components=1)
            vars_pca = pca.fit_transform(combined)
            return vars_pca[:, 0]
    
    data_paths = ["/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_ESPAD_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_IRI_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_PAAQ_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_RRS_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TCI_CHILD_FU2-IMAGEN_DIGEST.csv",
                  "/zi/flstorage/HITKIP/datasets/IMAGEN/FU2/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv"]
    
    dataframes = [pd.read_csv(path) for path in data_paths]
    # Correct for the unique User.code handling in d7
    dataframes[6]['User.code'] = dataframes[6]['User.code'].str.replace('-I', '-C')
    
    # Identify overlapping subjects
    user_codes = pd.Series([uc for df in dataframes for uc in df['User.code']])
    ucode_comb = user_codes[user_codes.duplicated(keep=False)].unique()
    
    data_matrices = [df[df['User.code'].isin(ucode_comb)].iloc[:, 1:].to_numpy() for df in dataframes]
    data_matrices = [np.nan_to_num(matrix) for matrix in data_matrices]
    
    # Load metadata and match
    meta = pd.read_csv("/zi/flstorage/HITKIP/datasets/IMAGEN/FU3/participants/IMAGEN_demographics.csv")
    ucode_comb_mod = pd.to_numeric([uc.replace('-C', '') for uc in ucode_comb])
    meta = meta[meta.iloc[:, 0].isin(ucode_comb_mod)]
    meta = meta.iloc[:, [1, 2]].to_numpy()
    
    # Correct for covariates
    data_matrices = [StandardScaler().fit_transform(matrix) for matrix in data_matrices]
    
    # Unique combinations of rating scales
    rtcmb = list(itertools.combinations(range(15), 2))
    
    # Repeated train/test splits
    repCOR = []
    for repi in range(20):
        train_indices, test_indices = CVfun(data_matrices[0])
        corvec = []
        for ip, is_ in rtcmb:
            cor = CCAfunc(data_matrices[ip][train_indices], data_matrices[is_][train_indices], 
                          data_matrices[ip][test_indices], data_matrices[is_][test_indices], export="cor")
            corvec.append(cor)
        repCOR.append(corvec)
    
    finalCor = np.median(repCOR, axis=0)
    
    # Convert to matrix and cluster
    corMat = np.zeros((15, 15))
    corMat[np.tril_indices(15, -1)] = finalCor
    corMat += corMat.T
    corMat[corMat < 0.5] = 0
    corMat[corMat != 0] = 1
    
    graph = nx.from_numpy_matrix(corMat)
    clusters = list(nx.connected_components(graph))
    
    # Compress clusters into first PC
    PCsol = []
    
    for cluster in clusters:
        if not cluster:
            continue
        
        curcmb = list(itertools.combinations(cluster, 2))
    
        combined_res = []
        for ip, is_ in curcmb:
            ccaPCcur = []
            for repi in range(100):
                train_indices, test_indices = CVfun(data_matrices[0])
                vars_ = CCAfunc(data_matrices[ip][train_indices], data_matrices[is_][train_indices], 
                                data_matrices[ip][test_indices], data_matrices[is_][test_indices], export="vars")
                ccaPCcur.append(vars_)
                
            ccaPCcur = np.array(ccaPCcur).T
            median_var = np.median(ccaPCcur, axis=1)
            combined_res.append(median_var)
        
        # Determine first PC for the canonical variates
        combined_res = np.array(combined_res).T
        principal_components = PCA(n_components=1).fit_transform(combined_res)
        PCsol.append(principal_components[:, 0])
    
    PCsol = np.column_stack(PCsol)
    
    # Explore whether autoencoder captures the canonical variate dimensions
    dListMat = np.column_stack(data_matrices)
    scaler = StandardScaler()
    dListMat = scaler.fit_transform(dListMat)
    
    h2o.init(max_mem_size="5g")
    hf = h2o.H2OFrame(dListMat)
    
    ae = H2OAutoEncoderEstimator(hidden=[50, 10, 50], activation="Tanh")
    ae.train(x=list(range(hf.shape[1])), training_frame=hf)
    
    ae_codings = ae.deepfeatures(hf, 1)
    
    ae_codings_df = ae_codings.as_data_frame()
    print(np.round(np.corrcoef(PCsol.T, ae_codings_df.T), 2))
    

    This Python version assumes that the data.csv files and processing details (like column names and transformations) in the original R code are appropriately replaced by their Python equivalents. Note that in real scenarios, additional error handling and data preprocessing might be necessary.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.