Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· OpenAIAi2 WildChat-4.8M (AI2)

i have following code to train a lstm model update the code so it implements validation…

  1. User

    i have following code to train a lstm model update the code so it implements validation set too:

    %%

    from sklearn.preprocessing import MinMaxScaler import pandas as pd import numpy as np from tensorflow import keras import joblib from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM,Dense,Dropout,Conv1D, Dense, Dropout, MaxPooling1D, GlobalAveragePooling1D

    import os

    %%

    csv_directory = r"C:\Users\arisa\Desktop\day_spot_summary_3" csv_files = [file for file in os.listdir(csv_directory) if file.endswith('.csv')] batch_size = 64

    %%

    def calculate_min_max_params(features_to_drop): scaler = MinMaxScaler() min_vals, max_vals = None, None for csv_file in csv_files: # Read the CSV file file_path = os.path.join(csv_directory, csv_file) chunk = pd.read_csv(file_path) filtered_chunk = chunk.drop(features_to_drop, axis=1) if min_vals is None: min_vals = filtered_chunk.min().values max_vals = filtered_chunk.max().values else: min_vals = np.minimum(min_vals, filtered_chunk.min().values) max_vals = np.maximum(max_vals, filtered_chunk.max().values)

    # Set the min and max values in scaler manually
    scaler.data_min_ = min_vals
    scaler.data_max_ = max_vals
    scaler.feature_range = (0, 1)
    scaler.fit(np.array([[0]*len(min_vals), [1]*len(min_vals)]))  # Dummy fit to set attributes correctly
    
    return scaler.data_min_, scaler.data_max_
    

    def calculate_targets_min_max_params(): target_cols = [ 'y_High_1d', 'y_Low_1d' , 'y_Priority_1d', # 'y_High_2d', 'y_Low_2d', 'y_Priority_2d', # 'y_High_3d', 'y_Low_3d', 'y_Priority_3d', # 'y_High_5d', 'y_Low_5d', 'y_Priority_5d' ] scaler = MinMaxScaler() min_vals, max_vals = None, None for csv_file in csv_files: # Read the CSV file file_path = os.path.join(csv_directory, csv_file) chunk = pd.read_csv(file_path) filtered_chunk = chunk[target_cols] if min_vals is None: min_vals = filtered_chunk.min().values max_vals = filtered_chunk.max().values else: min_vals = np.minimum(min_vals, filtered_chunk.min().values) max_vals = np.maximum(max_vals, filtered_chunk.max().values)

    # Set the min and max values in scaler manually
    scaler.data_min_ = min_vals
    scaler.data_max_ = max_vals
    scaler.feature_range = (0, 1)
    scaler.fit(np.array([[0]*len(min_vals), [1]*len(min_vals)]))  # Dummy fit to set attributes correctly
    
    return scaler.data_min_, scaler.data_max_
    

    %%

    from tensorflow.keras.layers import BatchNormalization from tensorflow.keras.optimizers import SGD

    def build_lstm_model(input_shape): model = Sequential([ LSTM(512, activation='tanh', input_shape=input_shape, return_sequences=True), # Adjusted for LSTM Dropout(0.20), LSTM(512, activation='tanh'), # BatchNormalization(), # LSTM(2716, activation='tanh', return_sequences=False), # Additional LSTM layer # Dropout(0.10), # Dense(2716, activation='tanh'), # Dropout(0.15), # # BatchNormalization(), # Dense(1024, activation='tanh'), # Dropout(0.15), # BatchNormalization(), Dense(512, activation='relu'), Dropout(0.15), # BatchNormalization(), Dense(256, activation='relu'), Dropout(0.10), # BatchNormalization(), Dense(128, activation='relu'), # BatchNormalization(), Dense(64, activation='relu'), # BatchNormalization(), Dense(32, activation='relu'), # BatchNormalization(), Dense(3), ]) optimizer = keras.optimizers.Adam(learning_rate=0.000001) model.compile(optimizer='adam', loss='mse', # Use Mean Squared Error for regression metrics=['mae']) # Mean Absolute Error as an additional metric return model

    %%

    x_scaler_loaded = joblib.load('nn_x_52_minmaxscaler.sav') y_scaler_loaded = joblib.load('nn_y_hlp1_minmaxscaler.sav')

    %%

    def data_generator_lstm( n_steps,x_scaler,y_scaler):

    while True:
        for csv_file in csv_files:
            # Read the CSV file
            file_path = os.path.join(csv_directory, csv_file)
            chunk = pd.read_csv(file_path)
        
    
            feature_data = chunk.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = chunk[[
                'y_High_1d','y_Low_1d','y_Priority_1d'
    
                   ]]
            
            feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
            # Assuming target_data also needs to be scaled, apply scaler separately
            target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
            
    
            # ensuring end_ix does not go out of feature_data_scaled’s bounds
            num_samples = (len(feature_data_scaled) - n_steps) // batch_size
            for i in range(num_samples):
                start_ix = i * batch_size
                end_ix = start_ix + n_steps
    
                X = feature_data_scaled[start_ix:end_ix]
                # using .iloc to avoid KeyError, and selecting the corresponding outputs
                y = target_data_scaled.iloc[start_ix:end_ix].iloc[-1]
    
                yield X.values.reshape((1, n_steps, -1)), y.values.reshape((1, -1))
            
            
    

    %%

    model = build_lstm_model((30, 52))

    model.summary()

    %%

    import warnings warnings.filterwarnings(action='ignore', message='X has feature names, but MinMaxScaler was fitted without feature names') train_generator = data_generator_lstm(30,x_scaler_loaded,y_scaler_loaded)

    from tensorflow.keras.callbacks import EarlyStopping early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)

    Update total_samples, train_samples, and val_samples according to your dataset after transformations

    model.fit( train_generator, steps_per_epoch=2000, epochs=1000, # callbacks=[early_stopping] # Add validation_data if you have a validation generator )

    %%

    model.save('lstm_1hot_extra_2716.h5')

  2. Assistant

    To implement a validation set in your training process, you'll need to create a separate validation data generator, similar to your training data generator. Additionally, you'll update the model.fit() call to include validation data and possibly re-enable the commented-out early stopping callback to monitor the validation loss. Here's how to make these updates:

    1. Create a Validation Set: If your dataset allows it, you should separate some of the CSV files as validation dataset which would not be touched by the training data generator. This separation is crucial in assessing how well your model generalizes to unseen data.

    2. Validation Data Generator: Implement a similar data generator function for the validation data. You might want to rename your current data_generator_lstm to train_data_generator_lstm and create a new one named val_data_generator_lstm using the validation data. For simplicity in this illustration, we'll assume you manually split some CSVs out of csv_files into a val_csv_files list.

    3. Early Stopping Callback: Uncomment and use the EarlyStopping callback during training to prevent overfitting.

    Here's how to modify the provided code:

    Split Your CSV Files into Train and Validation Sets

    Before the data generator section, manully define a validation set:

    # Manually define train_csv_files and val_csv_files. Here's a simple way to split, for example, 80-20:
    total_files = len(csv_files)
    train_files_count = int(total_files * 0.8)
    train_csv_files = csv_files[:train_files_count]
    val_csv_files = csv_files[train_files_count:]
    
    

    Update your data generator to take csv_files as a parameter to make it reusable for both training and validation data.

    Create a Validation Generator

    No changes in the generator function's code are needed, but when you call it, use the val_csv_files to generate validation data:

    val_generator = data_generator_lstm(30, x_scaler_loaded, y_scaler_loaded, val_csv_files)
    

    Make sure to adjust the generator function signature if you implement it to accept csv_files as an argument.

    Update Model Fit

    Enable the validation set in the model.fit call and bring back the early stopping callback:

    model.fit(
        train_generator,
        steps_per_epoch=2000,
        epochs=1000,
        validation_data=val_generator,
        validation_steps=500,  # You'll need to adjust this based on the size of your validation set
        callbacks=[early_stopping]
    )
    

    Note

    For your generators, ensure you correctly divide the dataset for training and validation, as discussed. Depending on your dataset size, validation_steps might also need adjustment.

    The validation steps represent how many batches of data from the validation generator will be processed. If your validation CSV files or the chosen batch size doesn't neatly divide, you might process some validation data multiple times or skip some.

    Final Touches

    Ensure your generators (train_generator and val_generator) correctly cycle through their respective sets of CSV files (train_csv_files and val_csv_files). Also, ensure any generator function adjustments account for CSV file selection, and consider using a random split or more robust cross-validation if you're aiming for more reliable model evaluations.