i have following code to train a model on my dataset i want to implement validation set…
- User
i have following code to train a model on my dataset i want to implement validation set in my code update the code and implement it i have no separate file for validation ,and my data set is just a large csv file:
from sklearn.preprocessing import StandardScaler import pandas as pd import numpy as np from tensorflow import keras import joblib
def calculate_targets_scaling_params(file_path): scaler = StandardScaler() for chunk in pd.read_csv(file_path, chunksize=10000): # Adjust chunksize based on your memory capacity filtered_chunk = chunk[['y_High_1d','y_Low_1d','y_Priority_1d']]
scaler.partial_fit(filtered_chunk) # Accumulate means and variances return scaler.mean_, scaler.var_%%
file_path = r"C:\Users\arisa\Desktop\combined_day.csv" batch_size = 1024
x_scaler_loaded = joblib.load('nn_x_scaler.sav') y_scaler_loaded = joblib.load('nn_y_hlp1_scaler.sav')
def data_generator(file_path, batch_size, x_scaler, y_scaler):
chunksize = batch_size while True: # Loop forever, so the generator never terminates for chunk in pd.read_csv(file_path, chunksize=chunksize): filtered_c = chunk.drop(['Date', 'Symbol'], axis=1) feature_data = filtered_c.drop([ 'y_High_1d', 'y_Low_1d', 'y_Priority_1d', 'y_High_2d', 'y_Low_2d', 'y_Priority_2d', 'y_High_3d', 'y_Low_3d', 'y_Priority_3d', 'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1) target_data = filtered_c[['y_High_1d', 'y_Low_1d','y_Priority_1d" ]] feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns) # Assuming target_data also needs to be scaled, apply scaler separately target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns) # Now, feature_data_scaled and target_data_scaled are both DataFrames, scaled and ready to use yield feature_data_scaled.values, target_data_scaled.values # row_counter += len(chunk)%%
from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Dropout, Input import tensorflow as tf from tensorflow.keras.layers import BatchNormalization
def build_model(): input_shape = (6427,)
model = Sequential([ Dense(6427, activation='relu', input_shape = input_shape), Dropout(0.17), BatchNormalization(), Dense(4096, activation='relu'), Dropout(0.15), BatchNormalization(), Dense(2048, activation='relu'), Dropout(0.12), BatchNormalization(), Dense(1024, activation='relu'), Dropout(0.10), BatchNormalization(), Dense(512, activation='relu'), Dropout(0.05), BatchNormalization(), Dense(256, activation='relu'), BatchNormalization(), Dense(128, activation='relu'), BatchNormalization(), Dense(64, activation='relu'), BatchNormalization(), Dense(32, activation='relu'), BatchNormalization(), Dense(3), ]) model.compile(optimizer='adam', loss='mse', # Use Mean Squared Error for regression metrics=['mae']) # Mean Absolute Error as an additional metric return modelInstantiate the model
model = build_model()
model.summary()
import warnings warnings.filterwarnings(action='ignore', message='X has feature names, but StandardScaler was fitted without feature names') train_generator = data_generator(file_path, batch_size,x_scaler=x_scaler_loaded,y_scaler=y_scaler_loaded)
total_samples = 301617 # Assuming same example size train_samples = int(total_samples)
steps_per_epoch = train_samples // batch_size
Modify the model fitting call to include validation data
model.fit( train_generator, steps_per_epoch=steps_per_epoch, epochs=45,
)
%%
model.save('snn_hlp1_relu_batchn_l20_mae27_6427_79m_e45_bz2048.h5')
- Assistant
To include a validation set in your training process without having a separate file for the validation data, you can split your dataset internally. This method involves modifying your
data_generatorfunction to accept a parameter that defines whether to generate training data or validation data and splitting your dataset accordingly.Here's how you can update your code to incorporate a validation set:
-
Modify
data_generatorto accept and handle splitting:Update the
data_generatorfunction to include a new parameter,mode, which determines whether the generator yields training or validation data. Then, use a portion of each chunk for validation based on a specified split ratio. -
Split data in generator:
Within the modified
data_generator, use a simple method to split the chunk into training and validation data. -
Adjust model fitting to include validation data:
Update the
model.fit()call to include validation data by creating a separate generator instance for validation.
Now, let me show you how to implement these changes in your code:
def data_generator(file_path, batch_size, x_scaler, y_scaler, mode='train', split=0.1): chunksize = batch_size while True: # Loop forever, so the generator never terminates for chunk in pd.read_csv(file_path, chunksize=chunksize): # Split chunk for training and validation split_index = int((1 - split) * len(chunk)) if mode == 'train': chunk_data = chunk[:split_index] elif mode == 'val': chunk_data = chunk[split_index:] else: raise ValueError("Mode should be 'train' or 'val'") filtered_c = chunk_data.drop(['Date', 'Symbol'], axis=1) feature_data = filtered_c.drop([...]) # Same as before, excluding target columns target_data = filtered_c[['y_High_1d', 'y_Low_1d', 'y_Priority_1d']] feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns) target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns) yield feature_data_scaled.values, target_data_scaled.values # Split the dataset for training and validation total_samples = 301617 # Example size validation_split = 0.1 # Assuming you want 10% of data for validation train_samples = int(total_samples * (1 - validation_split)) validation_samples = total_samples - train_samples # Adjust steps per epoch steps_per_epoch = train_samples // batch_size validation_steps = validation_samples // batch_size # Create separate generators for training and validation train_generator = data_generator(file_path, batch_size, x_scaler_loaded, y_scaler_loaded, mode='train', split=validation_split) validation_generator = data_generator(file_path, batch_size, x_scaler_loaded, y_scaler_loaded, mode='val', split=validation_split) # Update model fitting to include validation data model.fit( train_generator, steps_per_epoch=steps_per_epoch, validation_data=validation_generator, validation_steps=validation_steps, epochs=45, )This script now includes a validation step during training. It assumes that the validation data is the last portion of each chunk. Remember, this simple split might not be perfect for time-series data, where order or chronological splits might be more appropriate depending on your use case and data characteristics.
-