Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· OpenAIAi2 WildChat-4.8M (AI2)

i have following code to train a model on my dataset i want to implement validation set…

  1. User

    i have following code to train a model on my dataset i want to implement validation set in my code update the code and implement it i have no separate file for validation ,and my data set is just a large csv file:

    from sklearn.preprocessing import StandardScaler import pandas as pd import numpy as np from tensorflow import keras import joblib

    def calculate_targets_scaling_params(file_path): scaler = StandardScaler() for chunk in pd.read_csv(file_path, chunksize=10000): # Adjust chunksize based on your memory capacity filtered_chunk = chunk[['y_High_1d','y_Low_1d','y_Priority_1d']]

        scaler.partial_fit(filtered_chunk)  # Accumulate means and variances
    
    return scaler.mean_, scaler.var_
    

    %%

    file_path = r"C:\Users\arisa\Desktop\combined_day.csv" batch_size = 1024

    x_scaler_loaded = joblib.load('nn_x_scaler.sav') y_scaler_loaded = joblib.load('nn_y_hlp1_scaler.sav')

    def data_generator(file_path, batch_size, x_scaler, y_scaler):

    chunksize = batch_size
    
    while True:  # Loop forever, so the generator never terminates
        for chunk in pd.read_csv(file_path, chunksize=chunksize):
    
          
            filtered_c = chunk.drop(['Date', 'Symbol'], axis=1)
            
            feature_data = filtered_c.drop([
                'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
                 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
                  'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
                   'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
            target_data = filtered_c[['y_High_1d', 'y_Low_1d','y_Priority_1d"
                   ]]
            
            feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
            # Assuming target_data also needs to be scaled, apply scaler separately
            target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
    
            # Now, feature_data_scaled and target_data_scaled are both DataFrames, scaled and ready to use
            yield feature_data_scaled.values, target_data_scaled.values
            # row_counter += len(chunk)
    

    %%

    from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Dropout, Input import tensorflow as tf from tensorflow.keras.layers import BatchNormalization

    def build_model(): input_shape = (6427,)

    model = Sequential([
    Dense(6427, activation='relu', input_shape = input_shape),
    Dropout(0.17),
    BatchNormalization(),
    Dense(4096, activation='relu'),
    Dropout(0.15),
    BatchNormalization(),
    Dense(2048, activation='relu'),
    Dropout(0.12),
    BatchNormalization(),
    Dense(1024, activation='relu'),
    Dropout(0.10),
    BatchNormalization(),
    Dense(512, activation='relu'),
    Dropout(0.05),
    BatchNormalization(),
    Dense(256, activation='relu'),
    BatchNormalization(),
    Dense(128, activation='relu'),
    BatchNormalization(),
    Dense(64, activation='relu'),
    BatchNormalization(),
    Dense(32, activation='relu'),
    BatchNormalization(),
    Dense(3),
    ])
    
    
    model.compile(optimizer='adam',
                  loss='mse',  # Use Mean Squared Error for regression
                  metrics=['mae'])  # Mean Absolute Error as an additional metric
    return model
    

    Instantiate the model

    model = build_model()

    model.summary()

    import warnings warnings.filterwarnings(action='ignore', message='X has feature names, but StandardScaler was fitted without feature names') train_generator = data_generator(file_path, batch_size,x_scaler=x_scaler_loaded,y_scaler=y_scaler_loaded)

    total_samples = 301617 # Assuming same example size train_samples = int(total_samples)

    steps_per_epoch = train_samples // batch_size

    Modify the model fitting call to include validation data

    model.fit( train_generator, steps_per_epoch=steps_per_epoch, epochs=45,

    )

    %%

    model.save('snn_hlp1_relu_batchn_l20_mae27_6427_79m_e45_bz2048.h5')

  2. Assistant

    To include a validation set in your training process without having a separate file for the validation data, you can split your dataset internally. This method involves modifying your data_generator function to accept a parameter that defines whether to generate training data or validation data and splitting your dataset accordingly.

    Here's how you can update your code to incorporate a validation set:

    1. Modify data_generator to accept and handle splitting:

      Update the data_generator function to include a new parameter, mode, which determines whether the generator yields training or validation data. Then, use a portion of each chunk for validation based on a specified split ratio.

    2. Split data in generator:

      Within the modified data_generator, use a simple method to split the chunk into training and validation data.

    3. Adjust model fitting to include validation data:

      Update the model.fit() call to include validation data by creating a separate generator instance for validation.

    Now, let me show you how to implement these changes in your code:

    def data_generator(file_path, batch_size, x_scaler, y_scaler, mode='train', split=0.1):
        chunksize = batch_size
        while True:  # Loop forever, so the generator never terminates
            for chunk in pd.read_csv(file_path, chunksize=chunksize):
                # Split chunk for training and validation
                split_index = int((1 - split) * len(chunk))
                
                if mode == 'train':
                    chunk_data = chunk[:split_index]
                elif mode == 'val':
                    chunk_data = chunk[split_index:]
                else:
                    raise ValueError("Mode should be 'train' or 'val'")
                
                filtered_c = chunk_data.drop(['Date', 'Symbol'], axis=1)
                
                feature_data = filtered_c.drop([...])  # Same as before, excluding target columns
                target_data = filtered_c[['y_High_1d', 'y_Low_1d', 'y_Priority_1d']]
                
                feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
                target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
                
                yield feature_data_scaled.values, target_data_scaled.values
    
    # Split the dataset for training and validation
    total_samples = 301617  # Example size
    validation_split = 0.1  # Assuming you want 10% of data for validation
    train_samples = int(total_samples * (1 - validation_split))
    validation_samples = total_samples - train_samples
    
    # Adjust steps per epoch
    steps_per_epoch = train_samples // batch_size
    validation_steps = validation_samples // batch_size
    
    # Create separate generators for training and validation
    train_generator = data_generator(file_path, batch_size, x_scaler_loaded, y_scaler_loaded, mode='train', split=validation_split)
    validation_generator = data_generator(file_path, batch_size, x_scaler_loaded, y_scaler_loaded, mode='val', split=validation_split)
    
    # Update model fitting to include validation data
    model.fit(
        train_generator,
        steps_per_epoch=steps_per_epoch,
        validation_data=validation_generator,
        validation_steps=validation_steps,
        epochs=45,
    )
    

    This script now includes a validation step during training. It assumes that the validation data is the last portion of each chunk. Remember, this simple split might not be perfect for time-series data, where order or chronological splits might be more appropriate depending on your use case and data characteristics.