"C:\Users\Omen\Desktop\opc surv.xlsx" benim çalışma excel dosyam 1. sütunum kimlik…
- User
"C:\Users\Omen\Desktop\opc surv.xlsx" benim çalışma excel dosyam 1. sütunum kimlik sütunum, 2. sütunum "Time" zaman değişkenim. 3. sütunum "Status" olay değişkenim "Alive" olayın olmadığını "Dead" olduğunu gösteriyor. 4-7 sütunlar klinik nümerik veriler. 8-19 sütunlar klinik kategorik veriler. 20. sütundan itibaren sütunlar nümerik radiomics özellikler. benim senden isteğim kategorik özelliklere encoder uygula. sadece klinik özelliklerle ve sadece radiomics özeliklerle ve kombine şekilde modeller geliştircez. her model içinde bağımsız bir özellik seçimi süreci yönetmeni istiyorum. boruta, lasso ile özellik seç farklı farklı modeller geliştircez.8:2 external validasyon yaklaşımı benimse. random survival forest ve Cox regresyon ve pytorch aracılı deephit modeller geliştirmeni bekliyorum. model başarısını C indek olarak ver. tek parça python kodu oarak ver
- Assistant
# Import necessary libraries import pandas as pd import numpy as np from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder, StandardScaler from sklearn.ensemble import RandomForestRegressor from lifelines import CoxPHFitter from lifelines.utils import concordance_index from sksurv.ensemble import RandomSurvivalForest from sksurv.metrics import concordance_index_censored from sksurv.linear_model import CoxnetSurvivalAnalysis from boruta import BorutaPy import torch import torch.nn as nn import torch.optim as optim # For DeepHit model !pip install pycox import torchtuples as tt from pycox.models import DeepHitSingle from pycox.models.loss import PMFLoss from pycox.preprocessing.label_transforms import LabTransDiscreteTime from pycox.evaluation import EvalSurv # Load the data data = pd.read_excel(r'C:\Users\Omen\Desktop\opc surv.xlsx') # Encode the 'Status' column data['Status'] = data['Status'].map({'Alive': 0, 'Dead': 1}) # Define column indices numeric_cols = data.columns[3:7] # columns 4-7 categorical_cols = data.columns[7:19] # columns 8-19 radiomics_cols = data.columns[19:] # columns 20 onwards # Clinical features clinical_features = data.iloc[:, 3:19] # columns 4-19 # Apply one-hot encoding to categorical features clinical_features = pd.get_dummies(clinical_features, columns=categorical_cols) # Radiomics features radiomics_features = data.iloc[:, 19:] # columns 20 onwards # Combined features combined_features = pd.concat([clinical_features, radiomics_features], axis=1) # Target variables time = data['Time'] status = data['Status'] # Datasets dictionary datasets = { 'clinical': clinical_features, 'radiomics': radiomics_features, 'combined': combined_features } # Define feature selection functions from sklearn.preprocessing import StandardScaler # Lasso feature selection from sksurv.linear_model import CoxnetSurvivalAnalysis def lasso_feature_selection(X_train, y_train_time, y_train_event): # Prepare the target variable y_train = np.array([(e, t) for e, t in zip(y_train_event, y_train_time)], dtype=[('event', bool), ('time', float)]) # Standardize the features scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) # Fit CoxNetSurvivalAnalysis with L1 penalty alphas = 10 ** np.linspace(-3, 3, 100) coxnet = CoxnetSurvivalAnalysis(l1_ratio=1.0, alphas=alphas, verbose=0) coxnet.fit(X_train_scaled, y_train) # Select coefficients that are non-zero at the best alpha coef = coxnet.coef_ # Get the index of the best alpha best_alpha_idx = np.argmin(coxnet.alphas_) # Get the coefficients at best alpha coef_best_alpha = coef[:, best_alpha_idx] # Select features with non-zero coefficients selected_features_idx = np.where(coef_best_alpha != 0)[0] selected_features = X_train.columns[selected_features_idx] return selected_features, scaler # Boruta feature selection from sklearn.ensemble import RandomForestRegressor def boruta_feature_selection(X_train, y_train_time, y_train_event=None): # Standardize the features scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) rf = RandomForestRegressor(n_jobs=-1, max_depth=5, random_state=42) boruta_selector = BorutaPy(rf, n_estimators='auto', verbose=0, random_state=42) boruta_selector.fit(X_train_scaled, y_train_time.values) selected_features_idx = boruta_selector.support_ selected_features = X_train.columns[selected_features_idx] return selected_features, scaler # Define model training and evaluation functions # Random Survival Forest from sksurv.ensemble import RandomSurvivalForest def train_evaluate_rsf(X_train, X_test, y_train_time, y_train_event, y_test_time, y_test_event): # Prepare the target variable y_train = np.array([(e, t) for e, t in zip(y_train_event, y_train_time)], dtype=[('event', bool), ('time', float)]) y_test = np.array([(e, t) for e, t in zip(y_test_event, y_test_time)], dtype=[('event', bool), ('time', float)]) rsf = RandomSurvivalForest(n_estimators=100, min_samples_split=10, min_samples_leaf=15, max_features="sqrt", n_jobs=-1, random_state=42) rsf.fit(X_train, y_train) # Evaluate using concordance index cindex = rsf.score(X_test, y_test) return cindex # Cox Regression from lifelines import CoxPHFitter from lifelines.utils import concordance_index def train_evaluate_cox(X_train, X_test, y_train_time, y_train_event, y_test_time, y_test_event): # Prepare the dataset for lifelines train_df = pd.DataFrame(X_train, columns=X_train.columns) train_df['Time'] = y_train_time.values train_df['Event'] = y_train_event.values test_df = pd.DataFrame(X_test, columns=X_test.columns) test_df['Time'] = y_test_time.values test_df['Event'] = y_test_event.values cph = CoxPHFitter() cph.fit(train_df, duration_col='Time', event_col='Event') # Predict partial hazards on test set partial_hazards = cph.predict_partial_hazard(test_df) # Compute concordance index cindex = concordance_index(test_df['Time'], -partial_hazards, test_df['Event']) return cindex # DeepHit model from pycox.models import DeepHitSingle from pycox.models.loss import PMFLoss from pycox.preprocessing.label_transforms import LabTransDiscreteTime from pycox.evaluation import EvalSurv def train_evaluate_deephit(X_train, X_test, y_train_time, y_train_event, y_test_time, y_test_event): num_durations = 10 # number of discrete time intervals labtrans = LabTransDiscreteTime(num_durations) y_train = labtrans.fit_transform(y_train_time.values, y_train_event.values) y_test = labtrans.transform(y_test_time.values, y_test_event.values) # Convert to numpy arrays X_train_np = X_train.values.astype('float32') X_test_np = X_test.values.astype('float32') y_train_np = y_train y_test_tuple = (y_test_time.values, y_test_event.values) in_features = X_train_np.shape[1] num_nodes = [32, 32] # number of nodes in the hidden layers out_features = labtrans.out_features # equals num_durations model = tt.practical.MLPVanilla(in_features, num_nodes, out_features, batch_norm=True, dropout=0.1) optimizer = tt.optim.Adam(lr=0.01) loss = PMFLoss() model = tt.Model(model, optimizer, loss, metrics=None) # Set batch size and epochs batch_size = 256 epochs = 100 callbacks = [tt.callbacks.EarlyStopping(patience=10)] # Training model.fit(X_train_np, y_train_np, batch_size, epochs, callbacks, verbose=False) # Evaluation surv = model.predict_surv_df(X_test_np) # Compute C-index ev = EvalSurv(surv, y_test_tuple[0], y_test_tuple[1], censor_surv='km') c_index = ev.concordance_td('antolini') return c_index # Initialize results dictionary results = {} # Loop over datasets for dataset_name, X in datasets.items(): print(f"\nProcessing dataset: {dataset_name}") # Split data X_train, X_test, y_train_time, y_test_time, y_train_event, y_test_event = train_test_split( X, time, status, test_size=0.2, random_state=42, stratify=status) # Feature selection methods feature_selection_methods = { 'Lasso': lasso_feature_selection, 'Boruta': boruta_feature_selection } for fs_name, fs_function in feature_selection_methods.items(): print(f"\nFeature selection method: {fs_name}") selected_features, scaler = fs_function(X_train, y_train_time, y_train_event) # Subset data with selected features X_train_fs = X_train[selected_features] X_test_fs = X_test[selected_features] # Standardize features X_train_fs_scaled = scaler.transform(X_train_fs) X_test_fs_scaled = scaler.transform(X_test_fs) # Convert to DataFrame X_train_fs_scaled = pd.DataFrame(X_train_fs_scaled, columns=selected_features) X_test_fs_scaled = pd.DataFrame(X_test_fs_scaled, columns=selected_features) # Models models = { 'RandomSurvivalForest': train_evaluate_rsf, 'CoxRegression': train_evaluate_cox, 'DeepHit': train_evaluate_deephit } for model_name, model_func in models.items(): print(f"\nTraining model: {model_name}") if model_name == 'RandomSurvivalForest': # Random Survival Forest does not require scaled features c_index = model_func(X_train_fs, X_test_fs, y_train_time, y_train_event, y_test_time, y_test_event) else: # Other models require scaled features c_index = model_func(X_train_fs_scaled, X_test_fs_scaled, y_train_time, y_train_event, y_test_time, y_test_event) # Store the results results[(dataset_name, fs_name, model_name)] = c_index print(f"C-index for {dataset_name} dataset with {fs_name} using {model_name}: {c_index:.4f}") # Display the results print("\nSummary of Results:") for key, value in results.items(): dataset_name, fs_name, model_name = key print(f"Dataset: {dataset_name}, Feature Selection: {fs_name}, Model: {model_name}, C-index: {value:.4f}")This Python code performs the following tasks:
-
Data Loading and Preprocessing:
- Loads the Excel file located at
C:\Users\Omen\Desktop\opc surv.xlsx. - Encodes categorical variables and applies one-hot encoding to clinical categorical features.
- Separates the dataset into clinical features, radiomics features, and combined features.
- Loads the Excel file located at
-
Feature Selection:
- Performs feature selection using both Boruta and Lasso methods independently for each dataset (clinical, radiomics, combined).
-
Model Development:
- Develops models using Random Survival Forest, Cox Regression, and DeepHit (via PyTorch using the
pycoxlibrary). - Each model is trained on features selected by both Boruta and Lasso methods separately.
- Develops models using Random Survival Forest, Cox Regression, and DeepHit (via PyTorch using the
-
Model Evaluation:
- Evaluates model performance using the concordance index (C-index).
- Prints out the C-index for each combination of dataset, feature selection method, and model.
-
External Validation:
- Uses an 80:20 train-test split for external validation.
Notes:
- The code assumes that the necessary libraries (
pycox,lifelines,sksurv, etc.) are installed. If not, you may need to install them usingpip installcommands. - The
pycoxlibrary is used for the DeepHit model, which simplifies implementing Deep Learning survival models with PyTorch. - The feature selection and model training are encapsulated in functions for clarity and reuse.
- The code provides a comprehensive pipeline from data loading to model evaluation in a single piece of code, as per your request.
Please make sure to adjust the file path and install any missing libraries before running the code.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.