Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· OpenAIAi2 WildChat-4.8M (AI2)

I have the following model estimation structure: #Transforming into dedicated structures…

  1. User

    I have the following model estimation structure: #Transforming into dedicated structures xdm_train = xgboost.DMatrix(X, Y, enable_categorical=True, missing=True) xdm_test = xgboost.DMatrix(X_test, Y_test, enable_categorical=True, missing=True)

    #Model training model = xgboost.train({'max_depth': 5, "seed": 123, 'objective': 'binary:logitraw','learning_rate': 0.2, 'min_split_loss': 3, 'eval_metric': 'auc'}, xdm_train, 20, [(xdm_test, 'eval'), (xdm_train, 'train')]) print(model)

    I need to add standard scaler in there somehow. Furthermore I want to optimize hyperparameter space using the following code: def validate(model, X, y, X_test=None, y_test=None, show_results=False): is_classification = is_classifier(model) use_cross_validation = X_test is None or y_test is None

    if use_cross_validation:
        if is_classification:
            scoring = "roc_auc"
        else:
            scoring = "r2"
            
        val_results = cross_validate(model, X, y, scoring=scoring, n_jobs=-1, return_train_score=True, cv=3)
        train_score = np.mean(val_results["train_score"])
        test_score = np.mean(val_results["test_score"])
    else:           
        model.fit(X, y)
        pred_train = model.predict(X)
        pred_test = model.predict(X_test)
        
        if is_classification:
            train_score = metrics.accuracy_score(y, pred_train)
            test_score = metrics.accuracy_score(y_test, pred_test)
            
            if show_results:
                plot_confusion_matrix(y_test, pred_test)
        else:
            train_score = metrics.r2_score(y, pred_train)
            test_score = metrics.r2_score(y_test, pred_test)
            
            if show_results:
                raw_y_train = exp_y(y)
                raw_y_test = exp_y(y_test)
                raw_pred_train = exp_y(pred_train)
                raw_pred_test = exp_y(pred_test)
                
                plot_pred_data(raw_y_test, raw_pred_test)
                print(f"train MAE on y:", metrics.mean_absolute_error(raw_y_train, raw_pred_train))
                print(f"test MAE on y:", metrics.mean_absolute_error(raw_y_test, raw_pred_test))
            
    is_overfitted = train_score - test_score > 0.05
        
    if show_results:
        if is_classification:
            print(f"train acc:", train_score)
            print(f"test acc:", test_score)
        else:
            print(f"train r2 on log y:", train_score)
            print(f"test r2 on log y:", test_score)
            
        if is_overfitted:
            print("Overfitted")
        
    return test_score, is_overfitted    
    

    from sklearn.model_selection import cross_val_score from sklearn.model_selection import cross_validate from sklearn.base import is_classifier

    def objective(trial): n_estimators = trial.suggest_int('n_estimators', 15,500) max_depth=trial.suggest_int('max_depth',1,15) learning_rate= trial.suggest_float('learning_rate',0,0.3) subsample= trial.suggest_float('subsample',0,1) gamma = trial.suggest_float("gamma", 1e-4, 1e2) reg_alpha=trial.suggest_float('reg_alpha',0,1) reg_lambda=trial.suggest_float('reg_lambda',0,1) model= xgboost.DMatrix(X, Y, enable_categorical=True, missing=True)

    score, is_overfitted = validate(model, X,Y)
    
    if is_overfitted:
        return 0
    else:
        return score
    

    study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=500)

    If you think that roc_auc is not the right choice for the problem of credit scoring (binary). ADUJST THE WHOLE CODE so it would work.

  2. Assistant

    First, let's modify your existing objective function to use XGBoost's train function. You can remove the model declaration inside the function as it's not necessary. Instead, pass the hyperparameters to the xgboost.train function inside the validate function. Additionally, add the standard scaler before splitting your dataset into the train and test sets.

    Furthermore, I have updated the function validate to scale the input features using StandardScaler.

    Start by splitting your dataset into training and testing sets and scaling them using the StandardScaler:

    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import StandardScaler
    
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 42)
    
    # Scale the data using StandardScaler
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    
    # Transform the datasets into DMatrix format
    xdm_train = xgboost.DMatrix(X_train, Y_train, enable_categorical=True, missing=True)
    xdm_test  = xgboost.DMatrix(X_test, Y_test, enable_categorical=True, missing=True)
    

    Update the validate function to work with your modified objective function:

    def validate(params, xdm_train, xdm_test, show_results=False):
        model = xgboost.train(params, xdm_train, 20, [(xdm_test, 'eval'), (xdm_train, 'train')])
        pred_train = model.predict(xdm_train)
        pred_test = model.predict(xdm_test)
        
        is_classification = True
        train_score = metrics.roc_auc_score(Y_train, pred_train)
        test_score = metrics.roc_auc_score(Y_test, pred_test)
        
        if show_results:
            print(f"train roc_auc:", train_score)
            print(f"test roc_auc:", test_score)
                
        is_overfitted = train_score - test_score > 0.05
        return test_score, is_overfitted    
    

    Finally, update the objective function and run the optimization:

    def objective(trial):
        n_estimators =  trial.suggest_int('n_estimators', 15, 500)
        max_depth = trial.suggest_int('max_depth', 1, 15) 
        learning_rate = trial.suggest_float('learning_rate', 0, 0.3)
        subsample = trial.suggest_float('subsample', 0, 1)
        gamma = trial.suggest_float("gamma", 1e-4, 1e2)
        reg_alpha = trial.suggest_float('reg_alpha', 0, 1)
        reg_lambda = trial.suggest_float('reg_lambda', 0, 1)
        
        params = {
            'n_estimators': n_estimators, 
            'max_depth': max_depth, 
            'learning_rate': learning_rate,
            'subsample': subsample,
            'gamma': gamma,
            'reg_alpha': reg_alpha,
            'reg_lambda': reg_lambda,
            'seed': 123, 
            'objective': 'binary:logitraw',
            'eval_metric': 'auc'
        }
        
        score, is_overfitted = validate(params, xdm_train, xdm_test)
    
        if is_overfitted:
            return 0
        else:
            return score
    
    study = optuna.create_study(direction='maximize')
    study.optimize(objective, n_trials=500)
    

    ROC AUC is a good evaluation metric for binary classification problems like credit scoring. The updated code will allow you to use the StandardScaler and run hyperparameter optimization concurrently.