After running this python code: import tensorflow as tf from keras.layers import…
- User
After running this python code: import tensorflow as tf from keras.layers import TextVectorization, Embedding, LayerNormalization, Dropout, Dense, MultiHeadAttention, GlobalAveragePooling1D, Input from keras.models import Sequential, Model from keras import Sequential
Standard imports for saving tokenizer configuration
import json
Define some constants
MAX_TOKENS = 250 # Size of the vocabulary EMBED_DIM = 64 # Embedding size NUM_HEADS = 4 # Number of attention heads FF_DIM = 64 # Hidden layer size in feed forward network MAX_LEN = 40 # Max length of sequence BATCH_SIZE = 64 # Training batch size
Prepare your dataset text file
text_file_path = "dataset.txt"
Define the text vectorization layer with padding
vectorize_layer = TextVectorization( max_tokens=MAX_TOKENS, output_mode="int", output_sequence_length=MAX_LEN, pad_to_max_tokens=True )
Load the text data and adapt the vectorization layer
with open(text_file_path, "r", encoding="utf-8") as f: lines = f.readlines() vectorize_layer.adapt(lines)
Token and position embedding as a custom layer
class TokenAndPositionEmbedding(tf.keras.layers.Layer): def init(self, maxlen, vocab_size, embed_dim): super().init() self.token_emb = Embedding(input_dim=vocab_size, output_dim=embed_dim) self.pos_emb = Embedding(input_dim=maxlen, output_dim=embed_dim)
def call(self, x): maxlen = tf.shape(x)[-1] positions = tf.range(start=0, limit=maxlen, delta=1) positions = self.pos_emb(positions) x = self.token_emb(x) return x + positionsDefine the Transformer block as a function
def transformer_encoder(inputs, embed_dim, num_heads, ff_dim, rate=0.1): # Reshape the input tensor inputs = tf.expand_dims(inputs, axis=2)
# Multi-headed self-attention attention_output = MultiHeadAttention( num_heads=num_heads, key_dim=embed_dim )(inputs, inputs) attention_output = Dropout(rate)(attention_output) attention_output = LayerNormalization(epsilon=1e-6)(inputs + attention_output) # Feed-forward layer ff_output = Sequential( [ Dense(ff_dim, activation="relu"), Dense(embed_dim), ] )(attention_output) ff_output = Dropout(rate)(ff_output) ff_output = LayerNormalization(epsilon=1e-6)(attention_output + ff_output) # Reshape the output tensor ff_output = tf.squeeze(ff_output, axis=2) return ff_outputCreate the Transformer model
def create_transformer_model(maxlen, vocab_size, embed_dim, num_heads, ff_dim): inputs = Input(shape=(maxlen,)) embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)(inputs) transformer_block_output = transformer_encoder(embedding_layer, embed_dim, num_heads, ff_dim) outputs = GlobalAveragePooling1D()(transformer_block_output) outputs = Dropout(0.1)(outputs) outputs = Dense(vocab_size, activation="softmax")(outputs) # Assuming next-token prediction model = Model(inputs=inputs, outputs=outputs) return model
transformer_model = create_transformer_model(MAX_LEN, MAX_TOKENS, EMBED_DIM, NUM_HEADS, FF_DIM) transformer_model.compile(optimizer="adam", loss="sparse_categorical_crossentropy")
Prepare the dataset for training with next-token prediction
def prepare_dataset(text_data, vectorize_layer, batch_size): text_dataset = text_data.map(vectorize_layer).batch(batch_size) text_labels = text_dataset.map(lambda x: x[:, 1:]) # Shifted by one position for labels text_inputs = text_dataset.map(lambda x: x[:, :-1]) # Original sequence without the last token as input dataset = tf.data.Dataset.zip((text_inputs, text_labels)) dataset = dataset.shuffle(buffer_size=1024).batch(batch_size).prefetch(tf.data.AUTOTUNE) return dataset
text_data = tf.data.TextLineDataset(text_file_path) dataset = prepare_dataset(text_data, vectorize_layer, BATCH_SIZE)
Train the model
transformer_model.fit(dataset, epochs=10)
Save the model
transformer_model.save("STM-Experiment")
Save the tokenizer to a json file
tokenizer_json = vectorize_layer.get_vocabulary() with open("tokenizer.json", "w", encoding="utf-8") as f: json.dump(tokenizer_json, f, ensure_ascii=False)
print("Model and tokenizer have been saved.") i get this error: Traceback (most recent call last): File "c:\Users\Dell-PC\Desktop\Projets\Simple-But-Functional\main.py", line 101, in transformer_model.fit(dataset, epochs=10) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py", line 70, in error_handler
raise e.with_traceback(filtered_tb) from None File "C:\Users\Dell-PC\AppData\Local\Temp_autograph_generated_fileyw_jno0t.py", line 15, in tf__train_function retval = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope) ValueError: in user code:File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1338, in train_function * return step_function(self, iterator) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1322, in step_function ** outputs = model.distribute_strategy.run(run_step, args=(data,)) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1303, in run_step ** outputs = model.train_step(data) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1080, in train_step y_pred = self(x, training=True) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py", line 70, in error_handler raise e.with_traceback(filtered_tb) from None ValueError: Exception encountered when calling layer 'query' (type EinsumDense). Shape must be rank 4 but is rank 5 for 0th input and equation: abcd,def->abcef for '{{node model/multi_head_attention/query/einsum/Einsum}} = Einsum[N=2, T=DT_FLOAT, equation="abcd,def->abcef"](model/tf.expand_dims/ExpandDims, model/multi_head_attention/query/einsum/Einsum/ReadVariableOp)' with input shapes: [?,?,1,?,64], [64,4,64]. Call arguments received by layer 'query' (type EinsumDense): • inputs=tf.Tensor(shape=(None, None, 1, None, 64), dtype=float32) - Assistant
The error you are encountering is related to tensor shape compatibility within the MultiHeadAttention layer. Specifically, the message indicates that an input with shape rank 5 is being passed to a layer that expects an input of rank 4. The error message reports this as the following shape conflict: "Shape must be rank 4 but is rank 5".
There are a few potential issues in your code that could be causing this problem. Let's examine and correct them step by step:
-
Avoid Reshaping Before MultiHeadAttention: The transformer block is unnecessarily expanding the dimensions of the inputs tensor before passing it to the MultiHeadAttention layer. MultiHeadAttention expects tensors of rank 3, not rank 5 as created by
tf.expand_dims(inputs, axis=2). You should remove this expand_dims operation. -
Model Inputs and Outputs: The prepare_dataset function incorrectly handles the shifting of inputs and labels for next-token prediction. It attempts to map the shifts directly on the batched dataset, instead of applying them to the individual sequences.
Let's fix these problems:
First, remove the expand_dims part from the
transformer_encoderfunction:def transformer_encoder(inputs, embed_dim, num_heads, ff_dim, rate=0.1): # Multi-headed self-attention attention_output = MultiHeadAttention( num_heads=num_heads, key_dim=embed_dim )(inputs, inputs) attention_output = Dropout(rate)(attention_output) attention_output = LayerNormalization(epsilon=1e-6)(inputs + attention_output) # Feed-forward layer ff_output = Sequential( [ Dense(ff_dim, activation="relu"), Dense(embed_dim), ] )(attention_output) ff_output = Dropout(rate)(ff_output) ff_output = LayerNormalization(epsilon=1e-6)(attention_output + ff_output) return ff_outputNext, let's correct the
prepare_datasetfunction to properly handle input and label sequences:# Prepare the dataset for training with next-token prediction def prepare_dataset(text_data, vectorize_layer, batch_size): text_dataset = text_data.batch(batch_size).map(vectorize_layer) text_dataset = text_dataset.unbatch() # Flatten the batch to get individual sequences def split_input_target(sequence): input_text = sequence[:-1] target_text = sequence[1:] return input_text, target_text dataset = text_dataset.map(split_input_target) dataset = dataset.shuffle(buffer_size=1024).batch(batch_size).prefetch(tf.data.AUTOTUNE) return datasetWith these corrected changes, your Transformer model should be able to process the dataset you have prepared, and the error should be resolved. Ensure that your
dataset.txthas data compatible with the model's expectations, and that you're accounting for any preprocessing that the TextVectorization layer might require.Make sure you also check the versions of TensorFlow and Keras you are using, as functionality can differ between versions. The code is most compatible with TensorFlow 2.x.
Finally, ensure that
dataset.txthas the proper formatting—each line should be a separate text example to be used by the model. -
- User
After doing all the things you wrote, i got this shape mismatch error: ValueError: Input 0 of layer "model" is incompatible with the layer: expected shape=(None, 40), found shape=(None, 39)
- Assistant
The error message you're encountering indicates that there is a shape mismatch when feeding inputs into the model. Specifically, the model expects inputs of shape
(None, 40), which corresponds to sequences of length 40, but it is receiving inputs of shape(None, 39), which corresponds to sequences of length 39.This problem is caused by the way you're splitting the inputs and labels in the
prepare_datasetfunction. You are correctly creating input sequences by excluding the last token and labels by excluding the first token, which effectively reduces the sequence length by 1.To resolve this mismatch, you should modify the model's Input layer to expect sequences of length 39 instead of 40, as we are splitting the last token to use as the label.
Change the
create_transformer_modelfunction to set the input shape to(maxlen - 1,)which accounts for the split we do in theprepare_datasetfunction:def create_transformer_model(maxlen, vocab_size, embed_dim, num_heads, ff_dim): inputs = Input(shape=(maxlen - 1,)) # Adjusted input shape embedding_layer = TokenAndPositionEmbedding(maxlen - 1, vocab_size, embed_dim)(inputs) transformer_block_output = transformer_encoder(embedding_layer, embed_dim, num_heads, ff_dim) outputs = GlobalAveragePooling1D()(transformer_block_output) outputs = Dropout(0.1)(outputs) outputs = Dense(vocab_size, activation="softmax")(outputs) # Assuming next-token prediction model = Model(inputs=inputs, outputs=outputs) return model transformer_model = create_transformer_model(MAX_LEN, MAX_TOKENS, EMBED_DIM, NUM_HEADS, FF_DIM)By setting the input shape to
(MAX_LEN - 1,), the model will now correctly expect sequences of length 39, which matches the preprocessed dataset you are providing to thefitmethod. Remember to recompile the model after making changes to its structure.With this adjustment, the model should be compatible with the input data, and the shape mismatch error should be resolved. Make sure to recreate the model with the updated function and recompile it before attempting to train again.
- User
I got this error after the change: Traceback (most recent call last): File "c:\Users\Dell-PC\Desktop\Projets\Simple-But-Functional\main.py", line 100, in transformer_model.fit(dataset, epochs=10) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py", line 70, in error_handler raise e.with_traceback(filtered_tb) from None File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\tensorflow\python\eager\execute.py", line 53, in quick_execute tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:
Detected at node 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' defined at (most recent call last):
File "c:\Users\Dell-PC\Desktop\Projets\Simple-But-Functional\main.py", line 100, in transformer_model.fit(dataset, epochs=10) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\utils\traceback_utils.py", line 65, in error_handler return fn(*args, **kwargs) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1742, in fit tmp_logs = self.train_function(iterator) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1338, in train_function return step_function(self, iterator) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1322, in step_function outputs = model.distribute_strategy.run(run_step, args=(data,)) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1303, in run_step outputs = model.train_step(data) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1081, in train_step loss = self.compute_loss(x, y, y_pred, sample_weight) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\training.py", line 1139, in compute_loss return self.compiled_loss( File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\engine\compile_utils.py", line 265, in call loss_value = loss_obj(y_t, y_p, sample_weight=sw) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\losses.py", line 142, in call losses = call_fn(y_true, y_pred) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\losses.py", line 268, in call return ag_fn(y_true, y_pred, **self._fn_kwargs) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\losses.py", line 2354, in sparse_categorical_crossentropy return backend.sparse_categorical_crossentropy( File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\site-packages\keras\src\backend.py", line 5762, in sparse_categorical_crossentropy res = tf.nn.sparse_softmax_cross_entropy_with_logits( Node: 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' logits and labels must have the same first dimension, got logits shape [4,250] and labels shape [156] [[{{node sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_train_function_3603] - Assistant
The error message indicates that there is a mismatch between the logits and labels dimensions provided to the sparse_categorical_crossentropy loss function. The logits (model predictions) are expected to have the same first dimension as the labels, but they don't.
Here's the relevant part of the error message:
logits and labels must have the same first dimension, got logits shape [4,250] and labels shape [156]This suggests that the batch of logits has a shape with the first dimension being 4 and the second dimension being 250, which is the vocabulary size (
MAX_TOKENS). However, the labels have a shape with a completely different first dimension (156 in this case).The issue seems to be arising from how the dataset is being batched. The labels need to be a flat vector with the same leading dimension as the batch size of the logits. It appears that the labels may not be batched correctly.
Let's debug this step by step:
- Make sure your
prepare_datasetfunction is correct. It should batch the input/target pairs properly:
def prepare_dataset(text_data, vectorize_layer, batch_size): text_dataset = text_data.batch(batch_size).map(vectorize_layer) text_dataset = text_dataset.unbatch() # Flatten the batch to get individual sequences def split_input_target(sequence): input_text = sequence[:-1] target_text = sequence[1:] return input_text, target_text dataset = text_dataset.map(split_input_target) dataset = dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE) return dataset-
Ensure that the
split_input_targetfunction is being applied correctly and that the labels are indeed the next-token labels for your input sequences. -
Check if your dataset contains the correct number of examples. If the dataset is too small and doesn't contain enough data for even a single batch, this could also result in mismatched shapes.
After checking these points, run the training process again. If you continue to receive errors, it may be helpful to print the shapes of the inputs and labels from the dataset before training to confirm that they match:
for inputs, targets in dataset.take(1): print(f'Input shape: {inputs.shape}') print(f'Target shape: {targets.shape}')Ensure that the first dimensions of both inputs and targets are the same and correspond to the batch size. Also, confirm that the second dimension of targets does not exist (i.e., the targets should be a vector, not a matrix), and the second dimension of inputs should be
MAX_LEN - 1.By ensuring that the data is correctly preprocessed and batched, your model should be able to train without encountering dimension mismatches.
- Make sure your
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.