I have the following model estimation structure: #Transforming into dedicated structures…
- User
I have the following model estimation structure: #Transforming into dedicated structures xdm_train = xgboost.DMatrix(X, Y, enable_categorical=True, missing=True) xdm_test = xgboost.DMatrix(X_test, Y_test, enable_categorical=True, missing=True)
#Model training model = xgboost.train({'max_depth': 5, "seed": 123, 'objective': 'binary:logitraw','learning_rate': 0.2, 'min_split_loss': 3, 'eval_metric': 'auc'}, xdm_train, 20, [(xdm_test, 'eval'), (xdm_train, 'train')]) print(model)
I need to add standard scaler in there somehow. Furthermore I want to optimize hyperparameter space using the following code: def validate(model, X, y, X_test=None, y_test=None, show_results=False): is_classification = is_classifier(model) use_cross_validation = X_test is None or y_test is None
if use_cross_validation: if is_classification: scoring = "roc_auc" else: scoring = "r2" val_results = cross_validate(model, X, y, scoring=scoring, n_jobs=-1, return_train_score=True, cv=3) train_score = np.mean(val_results["train_score"]) test_score = np.mean(val_results["test_score"]) else: model.fit(X, y) pred_train = model.predict(X) pred_test = model.predict(X_test) if is_classification: train_score = metrics.accuracy_score(y, pred_train) test_score = metrics.accuracy_score(y_test, pred_test) if show_results: plot_confusion_matrix(y_test, pred_test) else: train_score = metrics.r2_score(y, pred_train) test_score = metrics.r2_score(y_test, pred_test) if show_results: raw_y_train = exp_y(y) raw_y_test = exp_y(y_test) raw_pred_train = exp_y(pred_train) raw_pred_test = exp_y(pred_test) plot_pred_data(raw_y_test, raw_pred_test) print(f"train MAE on y:", metrics.mean_absolute_error(raw_y_train, raw_pred_train)) print(f"test MAE on y:", metrics.mean_absolute_error(raw_y_test, raw_pred_test)) is_overfitted = train_score - test_score > 0.05 if show_results: if is_classification: print(f"train acc:", train_score) print(f"test acc:", test_score) else: print(f"train r2 on log y:", train_score) print(f"test r2 on log y:", test_score) if is_overfitted: print("Overfitted") return test_score, is_overfittedfrom sklearn.model_selection import cross_val_score from sklearn.model_selection import cross_validate from sklearn.base import is_classifier
def objective(trial): n_estimators = trial.suggest_int('n_estimators', 15,500) max_depth=trial.suggest_int('max_depth',1,15) learning_rate= trial.suggest_float('learning_rate',0,0.3) subsample= trial.suggest_float('subsample',0,1) gamma = trial.suggest_float("gamma", 1e-4, 1e2) reg_alpha=trial.suggest_float('reg_alpha',0,1) reg_lambda=trial.suggest_float('reg_lambda',0,1) model= xgboost.DMatrix(X, Y, enable_categorical=True, missing=True)
score, is_overfitted = validate(model, X,Y) if is_overfitted: return 0 else: return scorestudy = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=500)
If you think that roc_auc is not the right choice for the problem of credit scoring (binary). ADUJST THE WHOLE CODE so it would work.
- Assistant
First, let's modify your existing
objectivefunction to use XGBoost'strainfunction. You can remove the model declaration inside the function as it's not necessary. Instead, pass the hyperparameters to thexgboost.trainfunction inside thevalidatefunction. Additionally, add the standard scaler before splitting your dataset into the train and test sets.Furthermore, I have updated the function
validateto scale the input features using StandardScaler.Start by splitting your dataset into training and testing sets and scaling them using the StandardScaler:
from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 42) # Scale the data using StandardScaler scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) # Transform the datasets into DMatrix format xdm_train = xgboost.DMatrix(X_train, Y_train, enable_categorical=True, missing=True) xdm_test = xgboost.DMatrix(X_test, Y_test, enable_categorical=True, missing=True)Update the
validatefunction to work with your modifiedobjectivefunction:def validate(params, xdm_train, xdm_test, show_results=False): model = xgboost.train(params, xdm_train, 20, [(xdm_test, 'eval'), (xdm_train, 'train')]) pred_train = model.predict(xdm_train) pred_test = model.predict(xdm_test) is_classification = True train_score = metrics.roc_auc_score(Y_train, pred_train) test_score = metrics.roc_auc_score(Y_test, pred_test) if show_results: print(f"train roc_auc:", train_score) print(f"test roc_auc:", test_score) is_overfitted = train_score - test_score > 0.05 return test_score, is_overfittedFinally, update the
objectivefunction and run the optimization:def objective(trial): n_estimators = trial.suggest_int('n_estimators', 15, 500) max_depth = trial.suggest_int('max_depth', 1, 15) learning_rate = trial.suggest_float('learning_rate', 0, 0.3) subsample = trial.suggest_float('subsample', 0, 1) gamma = trial.suggest_float("gamma", 1e-4, 1e2) reg_alpha = trial.suggest_float('reg_alpha', 0, 1) reg_lambda = trial.suggest_float('reg_lambda', 0, 1) params = { 'n_estimators': n_estimators, 'max_depth': max_depth, 'learning_rate': learning_rate, 'subsample': subsample, 'gamma': gamma, 'reg_alpha': reg_alpha, 'reg_lambda': reg_lambda, 'seed': 123, 'objective': 'binary:logitraw', 'eval_metric': 'auc' } score, is_overfitted = validate(params, xdm_train, xdm_test) if is_overfitted: return 0 else: return score study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=500)ROC AUC is a good evaluation metric for binary classification problems like credit scoring. The updated code will allow you to use the StandardScaler and run hyperparameter optimization concurrently.