%%time !pip install -q -U tensorflow-text !pip install datasets !pip install -q…
- User
%%time !pip install -q -U tensorflow-text !pip install datasets !pip install -q tensorflow_datasets !pip install pydot !cd /content !clear
import tensorflow as tf import tensorflow_text as tf_text import tensorflow_datasets as tfds import numpy as np import matplotlib.pyplot as plt import requests import functools import collections import os import pathlib import re import string import tempfile import time import matplotlib.pyplot as plt import os import re import shutil import string import tensorflow as tf
from tensorflow.keras import layers from tensorflow.keras import losses import pydot
from datasets import load_dataset
dataset = load_dataset(“Helsinki-NLP/opus_books”, “en-fr”) data = dataset[“train”]
french_sentences = [example[“fr”] for example in data[“translation”][:127085]] english_sentences = [example[“en”] for example in data[“translation”][:127085]] dataset = tf.data.Dataset.from_tensor_slices((french_sentences, english_sentences))
french_sentences_decoded = [] english_sentences_decoded = []
for french_sentence, english_sentence in dataset.take(127085): french_sentences_decoded.append(“b '”+french_sentence.numpy().decode(‘utf-8’)) english_sentences_decoded.append(“b '”+english_sentence.numpy().decode(‘utf-8’))
print(“Nombre de phrases en français :”, len(french_sentences_decoded)) print(“Nombre de phrases en anglais :”, len(english_sentences_decoded))
train_fr = french_sentences train_en = english_sentences
from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab
bert_tokenizer_params = dict(lower_case=True) reserved_tokens = [“[PAD]”, “[UNK]”, “[START]”, “[END]”]
bert_vocab_args = { ‘vocab_size’: 8000, ‘reserved_tokens’: reserved_tokens, ‘bert_tokenizer_params’: bert_tokenizer_params, ‘learn_params’: {}, }
%%time en_vocab = bert_vocab.bert_vocab_from_dataset( tf.data.Dataset.from_tensor_slices(english_sentences).batch(1000).prefetch(2), **bert_vocab_args ) %%time fr_vocab = bert_vocab.bert_vocab_from_dataset( tf.data.Dataset.from_tensor_slices(french_sentences).batch(1000).prefetch(2), **bert_vocab_args )
def write_vocab_file(filepath, vocab): with open(filepath, ‘w’) as f: for token in vocab: print(token, file=f) write_vocab_file(‘en_vocab.txt’, en_vocab) write_vocab_file(‘fr_vocab.txt’, fr_vocab)
def write_vocab_file(filepath, vocab): with open(filepath, ‘w’) as f: for token in vocab: print(token, file=f) write_vocab_file(‘en_vocab.txt’, en_vocab) write_vocab_file(‘fr_vocab.txt’, fr_vocab)
Tokenize the examples -> (batch, word, word-piece)
en_tokenizere = en_tokenizer.tokenize(“hello how are you Vadim”)
Merge the word and word-piece axes -> (batch, tokens)
en_tokenizere= en_tokenizere.merge_dims(-2,-1)
for ex in en_tokenizere.to_list(): print(ex)
max_length = 200
fr_sequences = [fr_tokenizer.tokenize(french_sentence.numpy().decode(‘utf-8’)).merge_dims(-2,-1) for french_sentence, _ in dataset.take(1000)] fr_ragged = tf.ragged.stack(fr_sequences) fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])
en_sequences = [en_tokenizer.tokenize(english_sentence.numpy().decode(‘utf-8’)).merge_dims(-2,-1) for _, english_sentence in dataset.take(1000)] en_ragged = tf.ragged.stack(en_sequences) en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])
x_train = fr_padded y_train = en_padded
import numpy as np import tensorflow as tf from tensorflow.keras import layers, models
def get_positional_encoding(seq_len, d_model): positions = np.arange(seq_len)[:, np.newaxis] div_term = np.exp(np.arange(0, d_model, 2) * -(np.log(10000.0) / d_model)) pe = np.zeros((seq_len, d_model)) pe[:, 0::2] = np.sin(positions * div_term) pe[:, 1::2] = np.cos(positions * div_term) return pe
class TransformerBlock(layers.Layer): def init(self, embed_dim, num_heads, ff_dim, rate=0.1): super(TransformerBlock, self).init() self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim) self.ffn = tf.keras.Sequential( [layers.Dense(ff_dim, activation=“relu”), layers.Dense(embed_dim)] ) self.layernorm1 = layers.LayerNormalization(epsilon=1e-6) self.layernorm2 = layers.LayerNormalization(epsilon=1e-6) self.dropout1 = layers.Dropout(rate) self.dropout2 = layers.Dropout(rate)
def call(self, inputs, training=False): attn_output = self.att(inputs, inputs) attn_output = self.dropout1(attn_output, training=training) out1 = self.layernorm1(inputs + attn_output) ffn_output = self.ffn(out1) ffn_output = self.dropout2(ffn_output, training=training) return self.layernorm2(out1 + ffn_output)
class TokenAndPositionEmbedding(layers.Layer): def init(self, maxlen, vocab_size, embed_dim): super(TokenAndPositionEmbedding, self).init() self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim) self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
def call(self, x): maxlen = tf.shape(x)[-1] positions = tf.range(start=0, limit=maxlen, delta=1) positions = self.pos_emb(positions) x = self.token_emb(x) return x + positions
Model Construction
vocab_size = 20000 # Example vocabulary size embed_dim = 200 # Embedding size for each token num_heads = 8 # Number of attention heads ff_dim = 2048 # Hidden layer size in feed forward network inside transformer
inputs = layers.Input(shape=(200,)) embedding_layer = TokenAndPositionEmbedding(200, vocab_size, embed_dim) x = embedding_layer(inputs)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim) x = transformer_block(x, training=True) # Just as an example. In practice, training can be dynamically set.
x = layers.GlobalAveragePooling1D()(x) outputs = layers.Dense(vocab_size, activation=“softmax”)(x)
model = models.Model(inputs=inputs, outputs=outputs) model.compile(optimizer=“adam”, loss=“sparse_categorical_crossentropy”, metrics=[“accuracy”]) model.summary()
history = model.fit(x_train, y_train, epochs=10, batch_size = 2)
is my code in google colab cells by cell and i will ask to you why in the cells “history = model.fit(x_train, y_train, epochs=10, batch_size = 2)” they reply to me an error message your goal is to fix the probleme by proposing a cells change : Epoch 1/10
ValueError Traceback (most recent call last) in <cell line: 1>() ----> 1 history = model.fit(x_train, y_train, epochs=10, batch_size = 2)
1 frames /usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py in error_handler(*args, **kwargs) 120 # To get the full stack trace, call: 121 # keras.config.disable_traceback_filtering() –> 122 raise e.with_traceback(filtered_tb) from None 123 finally: 124 del filtered_tb
/usr/local/lib/python3.10/dist-packages/keras/src/layers/input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name) 243 if spec_dim is not None and dim is not None: 244 if spec_dim != dim: –> 245 raise ValueError( 246 f’Input {input_index} of layer “{layer_name}” is ' 247 "incompatible with the layer: "
ValueError: Input 0 of layer “functional_140” is incompatible with the layer: expected shape=(None, 200), found shape=(2, 1, 200)
- Assistant
The error message you're encountering indicates a shape mismatch between your model's expected input shape and the shape of the data you're trying to feed into it. The model expects input of shape
(None, 200), but it is receiving data of shape(2, 1, 200).To fix this issue, you'll need to adjust the shape of your data (
x_trainand possiblyy_train) to match what the model expects.It looks like an extra dimension has been introduced to your
x_trainandy_trainarrays which the model is not expecting. The error message(2, 1, 200)suggests that for some reason your batch size (2here) is being treated as a separate dimension, and there's an unnecessary1in the shape that likely needs to be removed.Here are recommended adjustments:
-
Ensure proper preparation of your data. The model expects data of 2 dimensions per sample but is getting 3 dimensions due to the way the tokenized sequences being padded or batched. Ensure that
x_trainandy_traindon't have an unnecessary middle dimension. -
Reshape or remove the unnecessary dimension. This can be done by reshaping your data to remove the unnecessary dimension (the extra '1'):
Before feeding
x_trainandy_traininto your model, try reshaping or squeezing out the unnecessary dimension:# Assuming x_train and y_train are NumPy arrays or TensorFlow tensors x_train = tf.squeeze(x_train, axis=1) # This should squeeze out the '1' dimension if it's unnecessary y_train = tf.squeeze(y_train, axis=1) # Adjust according to y_train's shape as wellMake sure the operation does not alter the intended data structure (e.g., by removing a necessary sequence length dimension).
-
Check the tokenization and batching logic. Review the code sections involved in tokenizing and preparing batches of your dataset to ensure they produce outputs in the expected shapes. The
fr_paddedanden_paddedshould be carefully checked to confirm they match the input shape anticipated by the model. -
Adjust the model's input layer if necessary. If the data preparation steps are correct and your task inherently requires different input dimensions (though less likely in this context), you may need to adjust the input shape expected by your model:
inputs = layers.Input(shape=(x_train.shape[-2], x_train.shape[-1]))
However, based on the error message and common practices, it seems the first approach (squeezing the unnecessary dimension) is likely to resolve your issue. Ensure that any reshaping aligns with the structure of your neural network and the nature of your task.
-