my code: %% import os import numpy as np import pandas as pd from…
- User
my code:
%%
import os import numpy as np import pandas as pd from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from tensorflow.keras.layers import Input, Dropout from tensorflow.keras import Model
%%
def data_generator(file_paths, batch_size, train=True, scaler=None): while True: # Loop forever so the generator never terminates # Shuffle file paths to ensure random distribution of data across files np.random.shuffle(file_paths)
for file_path in file_paths: data = pd.read_csv(file_path) # Split features and labels X = data.drop([ 'Date', 'Symbol', 'y_High_1d', 'y_Low_1d', 'y_Priority_1d', 'y_High_2d', 'y_Low_2d', 'y_Priority_2d', 'y_High_3d', 'y_Low_3d', 'y_Priority_3d', 'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1).values y_high_1d = data['y_High_1d'].values y_low_1d = data['y_Low_1d'].values y_priority_1d = data['y_Priority_1d'].values y_high_2d = data['y_High_2d'].values y_low_2d = data['y_Low_2d'].values y_priority_2d = data['y_Priority_2d'].values y_high_3d = data['y_High_3d'].values y_low_3d = data['y_Low_3d'].values y_priority_3d = data['y_Priority_3d'].values y_high_5d = data['y_High_5d'].values y_low_5d = data['y_Low_5d'].values y_priority_5d = data['y_Priority_5d'].values # Optionally, apply preprocessing here (e.g., Scaler transform) if scaler is not None and train: X = scaler.fit_transform(X) elif scaler is not None: X = scaler.transform(X) # Splitting data into batches for i in range(0, len(data), batch_size): end = i + batch_size X_batch = X[i:end] y_high_batch_1d = y_high_1d[i:end] y_low_batch_1d = y_low_1d[i:end] y_priority_batch_1d = y_priority_1d[i:end] y_high_batch_2d = y_high_2d[i:end] y_low_batch_2d = y_low_2d[i:end] y_priority_batch_2d = y_priority_2d[i:end] y_high_batch_3d = y_high_3d[i:end] y_low_batch_3d = y_low_3d[i:end] y_priority_batch_3d = y_priority_3d[i:end] y_high_batch_5d = y_high_5d[i:end] y_low_batch_5d = y_low_5d[i:end] y_priority_batch_5d = y_priority_5d[i:end] yield X_batch, [y_high_batch_1d, y_low_batch_1d,y_priority_batch_1d, y_high_batch_2d, y_low_batch_2d,y_priority_batch_2d, y_high_batch_3d, y_low_batch_3d, y_priority_batch_3d, y_high_batch_5d, y_low_batch_5d, y_priority_batch_5d]%%
dataset_dir = r'C:\Users\arisa\Desktop\day_spot' file_paths = [os.path.join(dataset_dir, file_name) for file_name in os.listdir(dataset_dir)] train_files, val_files = train_test_split(file_paths, test_size=0.06, random_state=42)
batch_size = 72
%%
Optional: Initialize a scaler for data normalization
scaler = StandardScaler()
%%
Create data generators
train_gen = data_generator(train_files, batch_size, train=True, scaler=scaler) val_gen = data_generator(val_files, batch_size, train=False, scaler=scaler)
%%
input_shape = (72, 6427)
inputs = Input(shape=input_shape) # Corrected the input shape specification
x = Dense(6427, activation='relu')(inputs) x = Dropout(0.25) (x) x = Dense(3200, activation='relu') (x) x = Dropout(0.20) (x) x = Dense(1800, activation='relu') (x) x = Dropout(0.15) (x) x = Dense(1024, activation='relu') (x) x = Dropout(0.10) (x) x = Dense(512, activation='relu') (x) x = Dropout(0.05) (x) x = Dense(256, activation='relu') (x) x = Dense(128, activation='relu') (x) x = Dense(64, activation='relu') (x) x = Dense(32, activation='relu') (x)
Defining three separate outputs
out_high_1d = Dense(1, name='high_output_1d')(x) # No activation, linear output out_low_1d = Dense(1, name='low_output_1d')(x) # No activation, linear output out_priority_1d = Dense(1, activation='sigmoid', name='priority_output_1d')(x)
out_high_2d = Dense(1, name='high_output_2d')(x) # No activation, linear output out_low_2d = Dense(1, name='low_output_2d')(x) # No activation, linear output out_priority_2d = Dense(1, activation='sigmoid', name='priority_output_2d')(x)
out_high_3d = Dense(1, name='high_output_3d')(x) # No activation, linear output out_low_3d = Dense(1, name='low_output_3d')(x) # No activation, linear output out_priority_3d = Dense(1, activation='sigmoid', name='priority_output_3d')(x)
out_high_5d = Dense(1, name='high_output_5d')(x) # No activation, linear output out_low_5d = Dense(1, name='low_output_5d')(x) # No activation, linear output out_priority_5d = Dense(1, activation='sigmoid', name='priority_output_5d')(x)
Constructing the model
model = Model(inputs=inputs, outputs=[ out_high_1d, out_low_1d, out_priority_1d,out_high_2d, out_low_2d, out_priority_2d, out_high_3d, out_low_3d, out_priority_3d,out_high_5d, out_low_5d, out_priority_5d])
model.compile(optimizer='adam', loss={ 'high_output_1d': 'mse', 'low_output_1d': 'mse', 'priority_output_1d': 'binary_crossentropy', 'high_output_2d': 'mse', 'low_output_2d': 'mse', 'priority_output_2d': 'binary_crossentropy', 'high_output_3d': 'mse', 'low_output_3d': 'mse', 'priority_output_3d': 'binary_crossentropy', 'high_output_5d': 'mse', 'low_output_5d': 'mse', 'priority_output_5d': 'binary_crossentropy' }, metrics={ 'high_output_1d': ['mae'], 'low_output_1d': ['mae'], 'priority_output_1d': ['accuracy'], 'high_output_2d': ['mae'], 'low_output_2d': ['mae'], 'priority_output_2d': ['accuracy'], 'high_output_3d': ['mae'], 'low_output_3d': ['mae'], 'priority_output_3d': ['accuracy'], 'high_output_5d': ['mae'], 'low_output_5d': ['mae'], 'priority_output_5d': ['accuracy'] }, loss_weights={ 'high_output_1d': 1.0, 'low_output_1d': 1.0, 'priority_output_1d': 1.0, 'high_output_2d': 1.0, 'low_output_2d': 1.0, 'priority_output_2d': 1.0, 'high_output_3d': 1.0, 'low_output_3d': 1.0, 'priority_output_3d': 1.0, 'high_output_5d': 1.0, 'low_output_5d': 1.0, 'priority_output_5d': 1.0 } )
%%
import os import pandas as pd
def count_samples(file_paths): total_samples = 0 for file_path in file_paths: df = pd.read_csv(file_path) samples = len(df) total_samples += samples return total_samples
number_of_training_samples = count_samples(train_files) number_of_validation_samples = count_samples(val_files)
print(f'Number of training samples: {number_of_training_samples}') print(f'Number of validation samples: {number_of_validation_samples}')
%%
import tensorflow as tf output_signature = ( tf.TensorSpec(shape=(None, X_batch.shape[1]), dtype=tf.float32), # for X_batch ( tf.TensorSpec(shape=(None, 1), dtype=tf.float32), # For each y_batch tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), tf.TensorSpec(shape=(None, 1), dtype=tf.float32), ) )
train_dataset = tf.data.Dataset.from_generator( lambda: train_gen, output_signature=output_signature )
val_dataset = tf.data.Dataset.from_generator( lambda: val_gen, output_signature=output_signature )
%%
You'll need to determine steps_per_epoch and validation_steps based on your data size and batch size
steps_per_epoch = np.ceil(number_of_training_samples / batch_size).astype(int) # Replace number_of_training_samples validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int) # Replace number_of_validation_samples
model.fit(train_gen,
steps_per_epoch=steps_per_epoch,
validation_data=val_gen,
validation_steps=validation_steps,
epochs=1000)
Then use these datasets in your model.fit call
model.fit(train_dataset, steps_per_epoch=steps_per_epoch, validation_data=val_dataset, validation_steps=validation_steps, epochs=1000)
error: { "name": "InvalidArgumentError", "message": "Graph execution error:
Detected at node PyFunc defined at (most recent call last): TypeError:
generatoryielded an element that did not match the expected structure. The expected structure was (tf.float32, (tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32)), but the yielded element was (array([[-0.39491499, -0.40790701, -0.37860123, ..., 0. , 0. , 0. ], [-0.387636 , -0.40270658, -0.36623326, ..., 0. , 0. , 0. ], [-0.37878316, -0.3817255 , -0.36090224, ..., 0. , 0. , 0. ], ..., [-0.52475656, -0.52626182, -0.50974435, ..., 0. , 0. , 0. ], [-0.51747756, -0.52410992, -0.52168584, ..., 0. , 0. , 0. ], [-0.5349865 , -0.54096651, -0.52360501, ..., 0. , 0. , 0. ]]), [array([ 1.88098495, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 0.48143054, 3.14661134, 0.27275827, 1.09270356, 1.1967617 , 1.36746143, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 0.57430007, 4.045605 , 0. , 1.76322418, -0.03586801, 4.12411626, 0.07616146, 1.39458573, 1.54723127, 0. , 1.61366872, 2.4262607 , 3.27255727, 2.55102041, 2.03117619, 2.6119403 , 3.37806571, 0.36798528, 2.41134752, 3.83536015, 4.19296664, 3.51339482, 5.09009009, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 3.21543408, 5.70910739, 2.98313878, 1.13350126, 3.37781485, 3.61201299, 1.90555095, 3.76229158, 1.95295162, 0.65387969, 1.0521701 , 3.44827586, 0.92470277, 1.36503743, 6.97368421, 4.68364832, 2.41014799, 0.3776752 , 4.12684622, 1.22466216, 1.92560175, 2.47131509, 4.88021295, 1.13636364, 2.78276481, 1.63355408, 1.51724138, 6.37413395]), array([ -1.19699042, -1.85247558, -4.78061559, -0.94466937, -2.002002 , -3.58579088, -1.89133425, -0.82987552, -4.90964882, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -0.99315068, -5.15641114, -3.51758794, -1.25045973, -2.2411953 , -1.11550918, -11.94404591, -0.90337785, -7.99695354, -1.64068909, -1.79153094, -22.43485342, -1.99335548, -3.04471931, -0.46750818, -4.63821892, -2.50354275, -2.51865672, -0.78667284, -3.31186753, -0.80378251, -1.87090739, -3.69702435, -3.20597277, -1.66666667, -1.69565217, -0.99050764, -5.91420242, -0.56374675, -9.66494845, -0.68902159, -3.94200272, -0.51880674, -3.61041142, -0.54211843, -2.5974026 , -4.97100249, -3.8477982 , -1.42032845, -2.83347864, -2.41122315, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -1.18393235, -4.1963911 , -1.39009557, -5.40540541, -1.48796499, -2.33892321, -0.88731145, -3.71503497, -0.53859964, -4.63576159, -1.10344828, -1.61662818]), array([0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0], dtype=int64), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.33944307, 1.75994368, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 2.18319886, 5.0903901 , 3.36605891, 2.55102041, 3.92064242, 4.19776119, 3.37806571, 0.36798528, 4.96453901, 8.09167446, 6.26690712, 3.51339482, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 7.16582453, 7.92931581, 4.15045396, 4.07220823, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.65387969, 1.27137221, 3.44827586, 1.36503743, 7.39762219, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 1.92560175, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.63355408, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -0.94466937, -4.004004 , -4.39008043, -1.89133425, -3.5615491 , -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -5.51369863, -7.59711241, -3.6252692 , -1.25045973, -2.2411953 , -11.6588701 , -11.94404591, -5.1060487 , -8.6824067 , -1.64068909, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.73021749, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.69565217, -6.76846884, -5.91420242, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -0.54211843, -6.89935065, -6.83512842, -5.04489098, -1.42032845, -2.96425458, -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -3.46723044, -4.74192195, -2.69331017, -5.40540541, -3.15098468, -2.33892321, -2.26264419, -3.71503497, -3.05206463, -5.03311258, -2.06896552, -1.61662818]), array([0., 1., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.9034191 , 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.84100617, 5.18553758, 3.36605891, 2.55102041, 5.52668871, 4.19776119, 3.37806571, 2.11591536, 9.26713948, 10.24321796, 6.26690712, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.21477663, 9.41662839, 9.15269597, 7.17682663, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.69747167, 1.27137221, 3.44827586, 7.39762219, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -2.93522267, -4.8048048 , -4.39008043, -4.09215956, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -6.63282572, -7.94520548, -7.70024063, -3.6252692 , -1.25045973, -12.66453219, -11.6588701 , -13.34289813, -5.81304006, -8.6824067 , -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.7826087 , -6.76846884, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -4.33694746, -8.72564935, -7.995029 , -5.04489098, -1.42032845, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.20377979, -4.01691332, -6.00083928, -2.69331017, -6.54560811, -3.15098468, -2.78022948, -2.26264419, -5.59440559, -3.45601436, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 2.21616093, 5.67500881, 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.93592786, 5.18553758, 4.44132772, 3.61781076, 5.52668871, 4.19776119, 6.94123091, 8.41766329, 11.44208038, 23.52666043, 19.07123535, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.29575022, 3.47938144, 13.87230133, 15.67739012, 10.37613489, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.52996005, 0.69747167, 6.9267865 , 14.10658307, 12.19726993, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.36624776, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -3.09868643, -6.58153242, -3.74493927, -6.94027361, -7.77479893, -5.36451169, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -3.26086957, -6.56934307, -9.13705584, -8.04794521, -7.70024063, -11.88083274, -9.709452 , -14.05193881, -13.70996761, -13.98852224, -25.17674784, -27.45620716, -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -8.56521739, -13.20676847, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.82144401, -5.58354324, -7.38115096, -9.86201299, -7.995029 , -7.6528431 , -4.12782956, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -7.97041906, -5.28541226, -7.13386488, -3.8662033 , -6.96790541, -3.58862144, -4.67784643, -4.56965395, -6.90559441, -4.39856373, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.])]). Traceback (most recent call last):File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 204, in generator_py_func flattened_values = nest.flatten_up_to(output_types, values)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\util
est.py", line 237, in flatten_up_to return nest_util.flatten_up_to(File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1541, in flatten_up_to return _tf_data_flatten_up_to(shallow_tree, input_tree)File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1570, in _tf_data_flatten_up_to _tf_data_assert_shallow_structure(shallow_tree, input_tree)File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1444, in _tf_data_assert_shallow_structure _tf_data_assert_shallow_structure(File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1414, in _tf_data_assert_shallow_structure raise TypeError(TypeError: If shallow structure is a sequence, input must also be a sequence. Input has type: 'list'.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\ops\script_ops.py", line 270, in call ret = func(*args)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper return func(*args, **kwargs)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 206, in generator_py_func raise TypeError(
TypeError:
generatoryielded an element that did not match the expected structure. The expected structure was (tf.float32, (tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32)), but the yielded element was (array([[-0.39491499, -0.40790701, -0.37860123, ..., 0. , 0. , 0. ], [-0.387636 , -0.40270658, -0.36623326, ..., 0. , 0. , 0. ], [-0.37878316, -0.3817255 , -0.36090224, ..., 0. , 0. , 0. ], ..., [-0.52475656, -0.52626182, -0.50974435, ..., 0. , 0. , 0. ], [-0.51747756, -0.52410992, -0.52168584, ..., 0. , 0. , 0. ], [-0.5349865 , -0.54096651, -0.52360501, ..., 0. , 0. , 0. ]]), [array([ 1.88098495, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 0.48143054, 3.14661134, 0.27275827, 1.09270356, 1.1967617 , 1.36746143, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 0.57430007, 4.045605 , 0. , 1.76322418, -0.03586801, 4.12411626, 0.07616146, 1.39458573, 1.54723127, 0. , 1.61366872, 2.4262607 , 3.27255727, 2.55102041, 2.03117619, 2.6119403 , 3.37806571, 0.36798528, 2.41134752, 3.83536015, 4.19296664, 3.51339482, 5.09009009, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 3.21543408, 5.70910739, 2.98313878, 1.13350126, 3.37781485, 3.61201299, 1.90555095, 3.76229158, 1.95295162, 0.65387969, 1.0521701 , 3.44827586, 0.92470277, 1.36503743, 6.97368421, 4.68364832, 2.41014799, 0.3776752 , 4.12684622, 1.22466216, 1.92560175, 2.47131509, 4.88021295, 1.13636364, 2.78276481, 1.63355408, 1.51724138, 6.37413395]), array([ -1.19699042, -1.85247558, -4.78061559, -0.94466937, -2.002002 , -3.58579088, -1.89133425, -0.82987552, -4.90964882, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -0.99315068, -5.15641114, -3.51758794, -1.25045973, -2.2411953 , -1.11550918, -11.94404591, -0.90337785, -7.99695354, -1.64068909, -1.79153094, -22.43485342, -1.99335548, -3.04471931, -0.46750818, -4.63821892, -2.50354275, -2.51865672, -0.78667284, -3.31186753, -0.80378251, -1.87090739, -3.69702435, -3.20597277, -1.66666667, -1.69565217, -0.99050764, -5.91420242, -0.56374675, -9.66494845, -0.68902159, -3.94200272, -0.51880674, -3.61041142, -0.54211843, -2.5974026 , -4.97100249, -3.8477982 , -1.42032845, -2.83347864, -2.41122315, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -1.18393235, -4.1963911 , -1.39009557, -5.40540541, -1.48796499, -2.33892321, -0.88731145, -3.71503497, -0.53859964, -4.63576159, -1.10344828, -1.61662818]), array([0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0], dtype=int64), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.33944307, 1.75994368, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 2.18319886, 5.0903901 , 3.36605891, 2.55102041, 3.92064242, 4.19776119, 3.37806571, 0.36798528, 4.96453901, 8.09167446, 6.26690712, 3.51339482, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 7.16582453, 7.92931581, 4.15045396, 4.07220823, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.65387969, 1.27137221, 3.44827586, 1.36503743, 7.39762219, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 1.92560175, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.63355408, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -0.94466937, -4.004004 , -4.39008043, -1.89133425, -3.5615491 , -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -5.51369863, -7.59711241, -3.6252692 , -1.25045973, -2.2411953 , -11.6588701 , -11.94404591, -5.1060487 , -8.6824067 , -1.64068909, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.73021749, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.69565217, -6.76846884, -5.91420242, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -0.54211843, -6.89935065, -6.83512842, -5.04489098, -1.42032845, -2.96425458, -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -3.46723044, -4.74192195, -2.69331017, -5.40540541, -3.15098468, -2.33892321, -2.26264419, -3.71503497, -3.05206463, -5.03311258, -2.06896552, -1.61662818]), array([0., 1., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.9034191 , 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.84100617, 5.18553758, 3.36605891, 2.55102041, 5.52668871, 4.19776119, 3.37806571, 2.11591536, 9.26713948, 10.24321796, 6.26690712, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.21477663, 9.41662839, 9.15269597, 7.17682663, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.69747167, 1.27137221, 3.44827586, 7.39762219, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -2.93522267, -4.8048048 , -4.39008043, -4.09215956, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -6.63282572, -7.94520548, -7.70024063, -3.6252692 , -1.25045973, -12.66453219, -11.6588701 , -13.34289813, -5.81304006, -8.6824067 , -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.7826087 , -6.76846884, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -4.33694746, -8.72564935, -7.995029 , -5.04489098, -1.42032845, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.20377979, -4.01691332, -6.00083928, -2.69331017, -6.54560811, -3.15098468, -2.78022948, -2.26264419, -5.59440559, -3.45601436, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 2.21616093, 5.67500881, 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.93592786, 5.18553758, 4.44132772, 3.61781076, 5.52668871, 4.19776119, 6.94123091, 8.41766329, 11.44208038, 23.52666043, 19.07123535, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.29575022, 3.47938144, 13.87230133, 15.67739012, 10.37613489, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.52996005, 0.69747167, 6.9267865 , 14.10658307, 12.19726993, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.36624776, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -3.09868643, -6.58153242, -3.74493927, -6.94027361, -7.77479893, -5.36451169, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -3.26086957, -6.56934307, -9.13705584, -8.04794521, -7.70024063, -11.88083274, -9.709452 , -14.05193881, -13.70996761, -13.98852224, -25.17674784, -27.45620716, -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -8.56521739, -13.20676847, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.82144401, -5.58354324, -7.38115096, -9.86201299, -7.995029 , -7.6528431 , -4.12782956, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -7.97041906, -5.28541226, -7.13386488, -3.8662033 , -6.96790541, -3.58862144, -4.67784643, -4.56965395, -6.90559441, -4.39856373, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.])]).\t [[{{node PyFunc}}]] \t [[IteratorGetNext]] [Op:__inference_one_step_on_iterator_7596]", "stack": "--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) Cell In[31], line 12 3 validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int) # Replace number_of_validation_samples 5 # model.fit(train_gen, 6 # steps_per_epoch=steps_per_epoch, 7 # validation_data=val_gen, (...) 10 11 # Then use these datasets in your model.fit call ---> 12 model.fit(train_dataset, 13 steps_per_epoch=steps_per_epoch, 14 validation_data=val_dataset, 15 validation_steps=validation_steps, 16 epochs=1000)
File c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py:122, in filter_traceback..error_handler(*args, **kwargs) 119 filtered_tb = _process_traceback_frames(e.traceback) 120 # To get the full stack trace, call: 121 #
keras.config.disable_traceback_filtering()--> 122 raise e.with_traceback(filtered_tb) from None 123 finally: 124 del filtered_tbFile c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\eager\execute.py:53, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 51 try: 52 ctx.ensure_initialized() ---> 53 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, 54 inputs, attrs, num_outputs) 55 except core._NotOkStatusException as e: 56 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node PyFunc defined at (most recent call last): TypeError:
generatoryielded an element that did not match the expected structure. The expected structure was (tf.float32, (tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32)), but the yielded element was (array([[-0.39491499, -0.40790701, -0.37860123, ..., 0. , 0. , 0. ], [-0.387636 , -0.40270658, -0.36623326, ..., 0. , 0. , 0. ], [-0.37878316, -0.3817255 , -0.36090224, ..., 0. , 0. , 0. ], ..., [-0.52475656, -0.52626182, -0.50974435, ..., 0. , 0. , 0. ], [-0.51747756, -0.52410992, -0.52168584, ..., 0. , 0. , 0. ], [-0.5349865 , -0.54096651, -0.52360501, ..., 0. , 0. , 0. ]]), [array([ 1.88098495, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 0.48143054, 3.14661134, 0.27275827, 1.09270356, 1.1967617 , 1.36746143, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 0.57430007, 4.045605 , 0. , 1.76322418, -0.03586801, 4.12411626, 0.07616146, 1.39458573, 1.54723127, 0. , 1.61366872, 2.4262607 , 3.27255727, 2.55102041, 2.03117619, 2.6119403 , 3.37806571, 0.36798528, 2.41134752, 3.83536015, 4.19296664, 3.51339482, 5.09009009, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 3.21543408, 5.70910739, 2.98313878, 1.13350126, 3.37781485, 3.61201299, 1.90555095, 3.76229158, 1.95295162, 0.65387969, 1.0521701 , 3.44827586, 0.92470277, 1.36503743, 6.97368421, 4.68364832, 2.41014799, 0.3776752 , 4.12684622, 1.22466216, 1.92560175, 2.47131509, 4.88021295, 1.13636364, 2.78276481, 1.63355408, 1.51724138, 6.37413395]), array([ -1.19699042, -1.85247558, -4.78061559, -0.94466937, -2.002002 , -3.58579088, -1.89133425, -0.82987552, -4.90964882, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -0.99315068, -5.15641114, -3.51758794, -1.25045973, -2.2411953 , -1.11550918, -11.94404591, -0.90337785, -7.99695354, -1.64068909, -1.79153094, -22.43485342, -1.99335548, -3.04471931, -0.46750818, -4.63821892, -2.50354275, -2.51865672, -0.78667284, -3.31186753, -0.80378251, -1.87090739, -3.69702435, -3.20597277, -1.66666667, -1.69565217, -0.99050764, -5.91420242, -0.56374675, -9.66494845, -0.68902159, -3.94200272, -0.51880674, -3.61041142, -0.54211843, -2.5974026 , -4.97100249, -3.8477982 , -1.42032845, -2.83347864, -2.41122315, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -1.18393235, -4.1963911 , -1.39009557, -5.40540541, -1.48796499, -2.33892321, -0.88731145, -3.71503497, -0.53859964, -4.63576159, -1.10344828, -1.61662818]), array([0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0], dtype=int64), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.33944307, 1.75994368, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 2.18319886, 5.0903901 , 3.36605891, 2.55102041, 3.92064242, 4.19776119, 3.37806571, 0.36798528, 4.96453901, 8.09167446, 6.26690712, 3.51339482, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 7.16582453, 7.92931581, 4.15045396, 4.07220823, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.65387969, 1.27137221, 3.44827586, 1.36503743, 7.39762219, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 1.92560175, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.63355408, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -0.94466937, -4.004004 , -4.39008043, -1.89133425, -3.5615491 , -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -5.51369863, -7.59711241, -3.6252692 , -1.25045973, -2.2411953 , -11.6588701 , -11.94404591, -5.1060487 , -8.6824067 , -1.64068909, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.73021749, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.69565217, -6.76846884, -5.91420242, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -0.54211843, -6.89935065, -6.83512842, -5.04489098, -1.42032845, -2.96425458, -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -3.46723044, -4.74192195, -2.69331017, -5.40540541, -3.15098468, -2.33892321, -2.26264419, -3.71503497, -3.05206463, -5.03311258, -2.06896552, -1.61662818]), array([0., 1., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.9034191 , 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.84100617, 5.18553758, 3.36605891, 2.55102041, 5.52668871, 4.19776119, 3.37806571, 2.11591536, 9.26713948, 10.24321796, 6.26690712, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.21477663, 9.41662839, 9.15269597, 7.17682663, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.69747167, 1.27137221, 3.44827586, 7.39762219, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -2.93522267, -4.8048048 , -4.39008043, -4.09215956, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -6.63282572, -7.94520548, -7.70024063, -3.6252692 , -1.25045973, -12.66453219, -11.6588701 , -13.34289813, -5.81304006, -8.6824067 , -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.7826087 , -6.76846884, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -4.33694746, -8.72564935, -7.995029 , -5.04489098, -1.42032845, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.20377979, -4.01691332, -6.00083928, -2.69331017, -6.54560811, -3.15098468, -2.78022948, -2.26264419, -5.59440559, -3.45601436, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 2.21616093, 5.67500881, 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.93592786, 5.18553758, 4.44132772, 3.61781076, 5.52668871, 4.19776119, 6.94123091, 8.41766329, 11.44208038, 23.52666043, 19.07123535, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.29575022, 3.47938144, 13.87230133, 15.67739012, 10.37613489, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.52996005, 0.69747167, 6.9267865 , 14.10658307, 12.19726993, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.36624776, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -3.09868643, -6.58153242, -3.74493927, -6.94027361, -7.77479893, -5.36451169, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -3.26086957, -6.56934307, -9.13705584, -8.04794521, -7.70024063, -11.88083274, -9.709452 , -14.05193881, -13.70996761, -13.98852224, -25.17674784, -27.45620716, -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -8.56521739, -13.20676847, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.82144401, -5.58354324, -7.38115096, -9.86201299, -7.995029 , -7.6528431 , -4.12782956, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -7.97041906, -5.28541226, -7.13386488, -3.8662033 , -6.96790541, -3.58862144, -4.67784643, -4.56965395, -6.90559441, -4.39856373, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.])]). Traceback (most recent call last):File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 204, in generator_py_func flattened_values = nest.flatten_up_to(output_types, values)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\util
est.py", line 237, in flatten_up_to return nest_util.flatten_up_to(File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1541, in flatten_up_to return _tf_data_flatten_up_to(shallow_tree, input_tree)File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1570, in _tf_data_flatten_up_to _tf_data_assert_shallow_structure(shallow_tree, input_tree)File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1444, in _tf_data_assert_shallow_structure _tf_data_assert_shallow_structure(File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\util
est_util.py", line 1414, in _tf_data_assert_shallow_structure raise TypeError(TypeError: If shallow structure is a sequence, input must also be a sequence. Input has type: 'list'.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\ops\script_ops.py", line 270, in call ret = func(*args)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper return func(*args, **kwargs)
File "c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 206, in generator_py_func raise TypeError(
TypeError:
generatoryielded an element that did not match the expected structure. The expected structure was (tf.float32, (tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.float32)), but the yielded element was (array([[-0.39491499, -0.40790701, -0.37860123, ..., 0. , 0. , 0. ], [-0.387636 , -0.40270658, -0.36623326, ..., 0. , 0. , 0. ], [-0.37878316, -0.3817255 , -0.36090224, ..., 0. , 0. , 0. ], ..., [-0.52475656, -0.52626182, -0.50974435, ..., 0. , 0. , 0. ], [-0.51747756, -0.52410992, -0.52168584, ..., 0. , 0. , 0. ], [-0.5349865 , -0.54096651, -0.52360501, ..., 0. , 0. , 0. ]]), [array([ 1.88098495, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 0.48143054, 3.14661134, 0.27275827, 1.09270356, 1.1967617 , 1.36746143, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 0.57430007, 4.045605 , 0. , 1.76322418, -0.03586801, 4.12411626, 0.07616146, 1.39458573, 1.54723127, 0. , 1.61366872, 2.4262607 , 3.27255727, 2.55102041, 2.03117619, 2.6119403 , 3.37806571, 0.36798528, 2.41134752, 3.83536015, 4.19296664, 3.51339482, 5.09009009, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 3.21543408, 5.70910739, 2.98313878, 1.13350126, 3.37781485, 3.61201299, 1.90555095, 3.76229158, 1.95295162, 0.65387969, 1.0521701 , 3.44827586, 0.92470277, 1.36503743, 6.97368421, 4.68364832, 2.41014799, 0.3776752 , 4.12684622, 1.22466216, 1.92560175, 2.47131509, 4.88021295, 1.13636364, 2.78276481, 1.63355408, 1.51724138, 6.37413395]), array([ -1.19699042, -1.85247558, -4.78061559, -0.94466937, -2.002002 , -3.58579088, -1.89133425, -0.82987552, -4.90964882, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -0.99315068, -5.15641114, -3.51758794, -1.25045973, -2.2411953 , -1.11550918, -11.94404591, -0.90337785, -7.99695354, -1.64068909, -1.79153094, -22.43485342, -1.99335548, -3.04471931, -0.46750818, -4.63821892, -2.50354275, -2.51865672, -0.78667284, -3.31186753, -0.80378251, -1.87090739, -3.69702435, -3.20597277, -1.66666667, -1.69565217, -0.99050764, -5.91420242, -0.56374675, -9.66494845, -0.68902159, -3.94200272, -0.51880674, -3.61041142, -0.54211843, -2.5974026 , -4.97100249, -3.8477982 , -1.42032845, -2.83347864, -2.41122315, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -1.18393235, -4.1963911 , -1.39009557, -5.40540541, -1.48796499, -2.33892321, -0.88731145, -3.71503497, -0.53859964, -4.63576159, -1.10344828, -1.61662818]), array([0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0], dtype=int64), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.33944307, 1.75994368, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 2.18319886, 5.0903901 , 3.36605891, 2.55102041, 3.92064242, 4.19776119, 3.37806571, 0.36798528, 4.96453901, 8.09167446, 6.26690712, 3.51339482, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.04295533, 7.16582453, 7.92931581, 4.15045396, 4.07220823, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.65387969, 1.27137221, 3.44827586, 1.36503743, 7.39762219, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 1.92560175, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.63355408, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -0.94466937, -4.004004 , -4.39008043, -1.89133425, -3.5615491 , -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -2.43654822, -5.51369863, -7.59711241, -3.6252692 , -1.25045973, -2.2411953 , -11.6588701 , -11.94404591, -5.1060487 , -8.6824067 , -1.64068909, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.73021749, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.69565217, -6.76846884, -5.91420242, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -0.54211843, -6.89935065, -6.83512842, -5.04489098, -1.42032845, -2.96425458, -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.03944125, -3.46723044, -4.74192195, -2.69331017, -5.40540541, -3.15098468, -2.33892321, -2.26264419, -3.71503497, -3.05206463, -5.03311258, -2.06896552, -1.61662818]), array([0., 1., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 0.27275827, 1.9034191 , 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.84100617, 5.18553758, 3.36605891, 2.55102041, 5.52668871, 4.19776119, 3.37806571, 2.11591536, 9.26713948, 10.24321796, 6.26690712, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.20901995, 0.21477663, 9.41662839, 9.15269597, 7.17682663, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.48557479, 0.69747167, 1.27137221, 3.44827586, 7.39762219, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.32136445, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -2.05456383, -4.78061559, -2.93522267, -4.8048048 , -4.39008043, -4.09215956, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -0.56100982, -0.66041015, -6.63282572, -7.94520548, -7.70024063, -3.6252692 , -1.25045973, -12.66453219, -11.6588701 , -13.34289813, -5.81304006, -8.6824067 , -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -1.7826087 , -6.76846884, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.73497622, -3.61041142, -4.33694746, -8.72564935, -7.995029 , -5.04489098, -1.42032845, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -6.20377979, -4.01691332, -6.00083928, -2.69331017, -6.54560811, -3.15098468, -2.78022948, -2.26264419, -5.59440559, -3.45601436, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0., 0.]), array([ 5.88235294, 4.27753452, 0.06548788, 2.0242915 , -0.1001001 , 0.30160858, 2.57909216, 3.14661134, 2.21616093, 5.67500881, 5.52622316, 5.11921459, 4.2057699 , 0.94754653, 1.02739726, 0.44688897, 1.54343144, 4.045605 , 0.60476699, 1.76322418, -0.03586801, 4.12411626, 0.07616146, 2.29696473, 1.54723127, 0. , 4.93592786, 5.18553758, 4.44132772, 3.61781076, 5.52668871, 4.19776119, 6.94123091, 8.41766329, 11.44208038, 23.52666043, 19.07123535, 15.98594642, 18.96396396, 14.82608696, 3.75567478, 0.29154519, 3.29575022, 3.47938144, 13.87230133, 15.67739012, 10.37613489, 7.17884131, 6.46371977, 3.61201299, 1.90555095, 3.76229158, 2.52996005, 0.69747167, 6.9267865 , 14.10658307, 12.19726993, 12.19726993, 11.75438596, 4.68364832, 2.41014799, 0.58749475, 4.12684622, 1.22466216, 3.45733042, 4.32480141, 4.88021295, 1.13636364, 3.36624776, 1.67770419, 5.88505747, 6.37413395]), array([ -1.19699042, -3.09868643, -6.58153242, -3.74493927, -6.94027361, -7.77479893, -5.36451169, -4.84094053, -6.17115581, -2.99612266, -0.80957409, -3.26086957, -6.56934307, -9.13705584, -8.04794521, -7.70024063, -11.88083274, -9.709452 , -14.05193881, -13.70996761, -13.98852224, -25.17674784, -27.45620716, -21.86218212, -22.43485342, -22.43485342, -3.27479829, -3.04471931, -3.88031791, -4.63821892, -2.50354275, -2.51865672, -2.91531698, -3.49586017, -0.80378251, -1.87090739, -3.69702435, -4.12823891, -1.66666667, -8.56521739, -13.20676847, -12.41149521, -8.80312229, -9.66494845, -2.61828204, -3.94200272, -0.82144401, -5.58354324, -7.38115096, -9.86201299, -7.995029 , -7.6528431 , -4.12782956, -5.8413252 , -5.30469093, -3.26914465, -1.58520476, -1.58520476, -1.27192982, -7.97041906, -5.28541226, -7.13386488, -3.8662033 , -6.96790541, -3.58862144, -4.67784643, -4.56965395, -6.90559441, -4.39856373, -5.9602649 , -2.06896552, -1.61662818]), array([0., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.])]).\t [[{{node PyFunc}}]] \t [[IteratorGetNext]] [Op:__inference_one_step_on_iterator_7596]" }
- Assistant
The error you're encountering is related to the structure of the data being yielded by your generator and the expected structure defined by your
output_signature. Specifically, TensorFlow expects the generator to yield tuples in a specific structure that matches the definition inoutput_signature, but there's a discrepancy in what's being yielded.It appears the core issue is how the outputs (targets) are being yielded from the generator and how they're defined in the
output_signature. The error message indicates that TensorFlow is expecting a structure of (tf.float32, ([tf.float32, ...])), which means a tuple containing an input and a tuple of outputs, but instead, it's receiving a different structure.Before diving into specific solutions, it's critical to ensure that each yield statement in your generator matches exactly with the structure defined in
output_signature. Here's a simplified example of what TensorFlow is expecting based on youroutput_signature:yield (input_features_array, (target1, target2, ... , target12))Given the complexity of your generator and the detailed structure expected, please double-check the following:
-
Matching Structure: Ensure that when the generator yields batches, it yields them in a structure exactly matching the
output_signature. From the error message, it seems there's a mismatch – likely around how the labels/targets are being structured. -
Nested Tuples for Outputs: Your model has multiple outputs, which means you should be yielding a single input array and then a tuple of arrays for each output in the exact order your model expects. From the error log, it's clear TensorFlow is not recognizing the yielded outputs as a proper sequence (tuple) of arrays.
Considering these points, the immediate solution would involve adjusting the yield part of your generator to match the expected structure precisely. Here's a sketch of how you might adjust the yielding part of your generator:
yield X_batch, ( y_high_batch_1d, y_low_batch_1d, y_priority_batch_1d, ... y_high_batch_5d, y_low_batch_5d, y_priority_batch_5d )Make sure the order and structure of these yielded items exactly match what you specified in
output_signature, including ensuring each target (y) is correctly structured as individual elements of a tuple and not as a list of arrays. The structure of the yield statement must be a tuple where the first element is your input batch and the second element is a tuple containing all the output batches in order.Lastly, review your
output_signatureto confirm it accurately reflects this structure. Eachtf.TensorSpecwithin should correspond to one of the arrays your generator yields, with the shape and dtype correctly specified.If the problem persists after these adjustments, systematically debug by simplifying the generator to yield batches for a single output case, then gradually add the complexity back while ensuring the structure always aligns with the
output_signatureexpectation. -
- User
i updated the code to:
%%
import os import numpy as np import pandas as pd from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from tensorflow.keras.layers import Input, Dropout from tensorflow.keras import Model
%%
def data_generator(file_paths, batch_size, train=True, scaler=None): while True: # Loop forever so the generator never terminates # Shuffle file paths to ensure random distribution of data across files np.random.shuffle(file_paths)
for file_path in file_paths: data = pd.read_csv(file_path) # Split features and labels X = data.drop([ 'Date', 'Symbol', 'y_High_1d', 'y_Low_1d', 'y_Priority_1d', 'y_High_2d', 'y_Low_2d', 'y_Priority_2d', 'y_High_3d', 'y_Low_3d', 'y_Priority_3d', 'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1).values y_high_1d = data['y_High_1d'].values y_low_1d = data['y_Low_1d'].values y_priority_1d = data['y_Priority_1d'].values y_high_2d = data['y_High_2d'].values y_low_2d = data['y_Low_2d'].values y_priority_2d = data['y_Priority_2d'].values y_high_3d = data['y_High_3d'].values y_low_3d = data['y_Low_3d'].values y_priority_3d = data['y_Priority_3d'].values y_high_5d = data['y_High_5d'].values y_low_5d = data['y_Low_5d'].values y_priority_5d = data['y_Priority_5d'].values # Optionally, apply preprocessing here (e.g., Scaler transform) if scaler is not None and train: X = scaler.fit_transform(X) elif scaler is not None: X = scaler.transform(X) # Splitting data into batches for i in range(0, len(data), batch_size): end = i + batch_size X_batch = X[i:end] y_high_batch_1d = y_high_1d[i:end] y_low_batch_1d = y_low_1d[i:end] y_priority_batch_1d = y_priority_1d[i:end] y_high_batch_2d = y_high_2d[i:end] y_low_batch_2d = y_low_2d[i:end] y_priority_batch_2d = y_priority_2d[i:end] y_high_batch_3d = y_high_3d[i:end] y_low_batch_3d = y_low_3d[i:end] y_priority_batch_3d = y_priority_3d[i:end] y_high_batch_5d = y_high_5d[i:end] y_low_batch_5d = y_low_5d[i:end] y_priority_batch_5d = y_priority_5d[i:end] yield X_batch, (y_high_batch_1d, y_low_batch_1d,y_priority_batch_1d, y_high_batch_2d, y_low_batch_2d,y_priority_batch_2d, y_high_batch_3d, y_low_batch_3d, y_priority_batch_3d, y_high_batch_5d, y_low_batch_5d, y_priority_batch_5d)%%
dataset_dir = r'C:\Users\arisa\Desktop\day_spot' file_paths = [os.path.join(dataset_dir, file_name) for file_name in os.listdir(dataset_dir)] train_files, val_files = train_test_split(file_paths, test_size=0.06, random_state=42)
batch_size = 72
%%
Optional: Initialize a scaler for data normalization
scaler = StandardScaler()
%%
Create data generators
train_gen = data_generator(train_files, batch_size, train=True, scaler=scaler) val_gen = data_generator(val_files, batch_size, train=False, scaler=scaler)
%%
input_shape = (72, 6427)
inputs = Input(shape=input_shape) # Corrected the input shape specification
x = Dense(6427, activation='relu')(inputs) x = Dropout(0.25) (x) x = Dense(3200, activation='relu') (x) x = Dropout(0.20) (x) x = Dense(1800, activation='relu') (x) x = Dropout(0.15) (x) x = Dense(1024, activation='relu') (x) x = Dropout(0.10) (x) x = Dense(512, activation='relu') (x) x = Dropout(0.05) (x) x = Dense(256, activation='relu') (x) x = Dense(128, activation='relu') (x) x = Dense(64, activation='relu') (x) x = Dense(32, activation='relu') (x)
Defining three separate outputs
out_high_1d = Dense(1, name='high_output_1d')(x) # No activation, linear output out_low_1d = Dense(1, name='low_output_1d')(x) # No activation, linear output out_priority_1d = Dense(1, activation='sigmoid', name='priority_output_1d')(x)
out_high_2d = Dense(1, name='high_output_2d')(x) # No activation, linear output out_low_2d = Dense(1, name='low_output_2d')(x) # No activation, linear output out_priority_2d = Dense(1, activation='sigmoid', name='priority_output_2d')(x)
out_high_3d = Dense(1, name='high_output_3d')(x) # No activation, linear output out_low_3d = Dense(1, name='low_output_3d')(x) # No activation, linear output out_priority_3d = Dense(1, activation='sigmoid', name='priority_output_3d')(x)
out_high_5d = Dense(1, name='high_output_5d')(x) # No activation, linear output out_low_5d = Dense(1, name='low_output_5d')(x) # No activation, linear output out_priority_5d = Dense(1, activation='sigmoid', name='priority_output_5d')(x)
Constructing the model
model = Model(inputs=inputs, outputs=[ out_high_1d, out_low_1d, out_priority_1d,out_high_2d, out_low_2d, out_priority_2d, out_high_3d, out_low_3d, out_priority_3d,out_high_5d, out_low_5d, out_priority_5d])
model.compile(optimizer='adam', loss={ 'high_output_1d': 'mse', 'low_output_1d': 'mse', 'priority_output_1d': 'binary_crossentropy', 'high_output_2d': 'mse', 'low_output_2d': 'mse', 'priority_output_2d': 'binary_crossentropy', 'high_output_3d': 'mse', 'low_output_3d': 'mse', 'priority_output_3d': 'binary_crossentropy', 'high_output_5d': 'mse', 'low_output_5d': 'mse', 'priority_output_5d': 'binary_crossentropy' }, metrics={ 'high_output_1d': ['mae'], 'low_output_1d': ['mae'], 'priority_output_1d': ['accuracy'], 'high_output_2d': ['mae'], 'low_output_2d': ['mae'], 'priority_output_2d': ['accuracy'], 'high_output_3d': ['mae'], 'low_output_3d': ['mae'], 'priority_output_3d': ['accuracy'], 'high_output_5d': ['mae'], 'low_output_5d': ['mae'], 'priority_output_5d': ['accuracy'] }, loss_weights={ 'high_output_1d': 1.0, 'low_output_1d': 1.0, 'priority_output_1d': 1.0, 'high_output_2d': 1.0, 'low_output_2d': 1.0, 'priority_output_2d': 1.0, 'high_output_3d': 1.0, 'low_output_3d': 1.0, 'priority_output_3d': 1.0, 'high_output_5d': 1.0, 'low_output_5d': 1.0, 'priority_output_5d': 1.0 } )
%%
import os import pandas as pd
def count_samples(file_paths): total_samples = 0 for file_path in file_paths: df = pd.read_csv(file_path) samples = len(df) total_samples += samples return total_samples
number_of_training_samples = count_samples(train_files) number_of_validation_samples = count_samples(val_files)
print(f'Number of training samples: {number_of_training_samples}') print(f'Number of validation samples: {number_of_validation_samples}')
%%
import tensorflow as tf output_signature = ( tf.TensorSpec(shape=(None,72, X_batch.shape[1]), dtype=tf.float32), # for X_batch ( tf.TensorSpec(shape=(None, 72), dtype=tf.float32), # For each y_batch tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), tf.TensorSpec(shape=(None, 72), dtype=tf.float32), ) )
train_dataset = tf.data.Dataset.from_generator( lambda: train_gen, output_signature=output_signature )
val_dataset = tf.data.Dataset.from_generator( lambda: val_gen, output_signature=output_signature )
%%
You'll need to determine steps_per_epoch and validation_steps based on your data size and batch size
steps_per_epoch = np.ceil(number_of_training_samples / batch_size).astype(int) # Replace number_of_training_samples validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int) # Replace number_of_validation_samples
model.fit(train_gen,
steps_per_epoch=steps_per_epoch,
validation_data=val_gen,
validation_steps=validation_steps,
epochs=1000)
Then use these datasets in your model.fit call
model.fit(train_dataset, steps_per_epoch=steps_per_epoch, validation_data=val_dataset, validation_steps=validation_steps, epochs=1000)
and the error changed to: { "name": "ValueError", "message": "Creating variables on a non-first call to a function decorated with tf.function.", "stack": "--------------------------------------------------------------------------- ValueError Traceback (most recent call last) Cell In[40], line 12 3 validation_steps = np.ceil(number_of_validation_samples / batch_size).astype(int) # Replace number_of_validation_samples 5 # model.fit(train_gen, 6 # steps_per_epoch=steps_per_epoch, 7 # validation_data=val_gen, (...) 10 11 # Then use these datasets in your model.fit call ---> 12 model.fit(train_dataset, 13 steps_per_epoch=steps_per_epoch, 14 validation_data=val_dataset, 15 validation_steps=validation_steps, 16 epochs=1000)
File c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py:122, in filter_traceback..error_handler(*args, **kwargs) 119 filtered_tb = _process_traceback_frames(e.traceback) 120 # To get the full stack trace, call: 121 #
keras.config.disable_traceback_filtering()--> 122 raise e.with_traceback(filtered_tb) from None 123 finally: 124 del filtered_tbFile c:\Users\arisa\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\backend\tensorflow\trainer.py:122, in TensorFlowTrainer.make_train_function..one_step_on_iterator(iterator) 120 """Runs a single training step given a Dataset iterator.""" 121 data = next(iterator) --> 122 outputs = self.distribute_strategy.run( 123 one_step_on_data, args=(data,) 124 ) 125 outputs = reduce_per_replica( 126 outputs, 127 self.distribute_strategy, 128 reduction=self.distribute_reduction_method, 129 ) 130 return outputs
ValueError: Creating variables on a non-first call to a function decorated with tf.function." }
- Assistant
The new error you're encountering, "Creating variables on a non-first call to a function decorated with tf.function," typically arises in TensorFlow when an operation that creates variables (like model instantiation or the first forward pass through a neural network) occurs inside a function that's being traced for graph execution (such as inside a
tf.functionor the first call to a dataset mapping operation where the graph mode is being inferred).However, this is an unusual error to encounter directly in a call to
model.fitwith a dataset generated bytf.data.Dataset.from_generator. Here are a few things to consider that might help resolve this issue:-
Ensure Consistent Output Shapes: Make sure the shapes defined in
output_signatureaccurately reflect the shapes of the data being yielded by the generator. For instance, youroutput_signatureforX_batchseems to expect a 3D tensor with shape(None, 72, X_batch.shape[1]), but your generator appears to produce batches withX_batchin a different shape. Ensure the first dimensionNoneis truly variable (batch size), and72matches your data's expected dimension. -
Review Generator Shape Consistency: Directly before yielding, you might want to print or assert the shapes of
X_batchand the variousy_*_batch_*arrays to confirm they're as expected. Sometimes, discrepancies in these shapes or mismatches with theoutput_signaturecan cause TensorFlow to attempt variable creation at inappropriate times. -
Variable Creation in tf.function: If there's any logic inside
model.fitor related processes (like custom callback functions) that inadvertently attempts to create new TensorFlow variables after the graph execution has started, it could trigger this error. Though it's less common for standard usage of kerasmodel.fit, if you're doing anything unconventional with your model or data processing, it's worth double-checking. -
Simplification and Debugging: As a diagnostic step, try simplifying your setup. For instance, test your generator with a much simpler model (even a dummy model) to confirm if the issue is with the dataset or the model logic. Sometimes working backward from a simpler case helps isolate the cause.
-
Review TensorFlow and Keras Versions: Ensure that your TensorFlow and any other relevant packages are up to date. It's possible that there's a bug or incompatibility in your current version, and updating to the latest version could resolve this issue.
Given the depth and the specific nature of the error, it appears to be more involved with TensorFlow's execution model than a simple misalignment in the data. If these steps do not resolve the issue, consider isolating further by creating a minimal reproducible example that triggers the error and testing in a new, clean environment to rule out environment-specific issues.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.