Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· OpenAIAi2 WildChat-4.8M (AI2)

%%time !pip install -q -U tensorflow-text !pip install datasets !pip install -q…

  1. User

    %%time !pip install -q -U tensorflow-text !pip install datasets !pip install -q tensorflow_datasets !pip install pydot !cd /content !clear

    import tensorflow as tf import tensorflow_text as tf_text import tensorflow_datasets as tfds import numpy as np import matplotlib.pyplot as plt import requests import functools import collections import os import pathlib import re import string import tempfile import time import matplotlib.pyplot as plt import os import re import shutil import string import tensorflow as tf

    from tensorflow.keras import layers from tensorflow.keras import losses import pydot

    from datasets import load_dataset

    dataset = load_dataset(“Helsinki-NLP/opus_books”, “en-fr”) data = dataset[“train”]

    french_sentences = [example[“fr”] for example in data[“translation”][:127085]] english_sentences = [example[“en”] for example in data[“translation”][:127085]] dataset = tf.data.Dataset.from_tensor_slices((french_sentences, english_sentences))

    french_sentences_decoded = [] english_sentences_decoded = []

    for french_sentence, english_sentence in dataset.take(127085): french_sentences_decoded.append(“b '”+french_sentence.numpy().decode(‘utf-8’)) english_sentences_decoded.append(“b '”+english_sentence.numpy().decode(‘utf-8’))

    print(“Nombre de phrases en français :”, len(french_sentences_decoded)) print(“Nombre de phrases en anglais :”, len(english_sentences_decoded))

    train_fr = french_sentences train_en = english_sentences

    from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab

    bert_tokenizer_params = dict(lower_case=True) reserved_tokens = [“[PAD]”, “[UNK]”, “[START]”, “[END]”]

    bert_vocab_args = { ‘vocab_size’: 8000, ‘reserved_tokens’: reserved_tokens, ‘bert_tokenizer_params’: bert_tokenizer_params, ‘learn_params’: {}, }

    %%time en_vocab = bert_vocab.bert_vocab_from_dataset( tf.data.Dataset.from_tensor_slices(english_sentences).batch(1000).prefetch(2), **bert_vocab_args ) %%time fr_vocab = bert_vocab.bert_vocab_from_dataset( tf.data.Dataset.from_tensor_slices(french_sentences).batch(1000).prefetch(2), **bert_vocab_args )

    def write_vocab_file(filepath, vocab): with open(filepath, ‘w’) as f: for token in vocab: print(token, file=f) write_vocab_file(‘en_vocab.txt’, en_vocab) write_vocab_file(‘fr_vocab.txt’, fr_vocab)

    def write_vocab_file(filepath, vocab): with open(filepath, ‘w’) as f: for token in vocab: print(token, file=f) write_vocab_file(‘en_vocab.txt’, en_vocab) write_vocab_file(‘fr_vocab.txt’, fr_vocab)

    Tokenize the examples -> (batch, word, word-piece)

    en_tokenizere = en_tokenizer.tokenize(“hello how are you Vadim”)

    Merge the word and word-piece axes -> (batch, tokens)

    en_tokenizere= en_tokenizere.merge_dims(-2,-1)

    for ex in en_tokenizere.to_list(): print(ex)

    max_length = 200

    fr_sequences = [fr_tokenizer.tokenize(french_sentence.numpy().decode(‘utf-8’)).merge_dims(-2,-1) for french_sentence, _ in dataset.take(1000)] fr_ragged = tf.ragged.stack(fr_sequences) fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])

    en_sequences = [en_tokenizer.tokenize(english_sentence.numpy().decode(‘utf-8’)).merge_dims(-2,-1) for _, english_sentence in dataset.take(1000)] en_ragged = tf.ragged.stack(en_sequences) en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])

    x_train = fr_padded y_train = en_padded

    import numpy as np import tensorflow as tf from tensorflow.keras import layers, models

    def get_positional_encoding(seq_len, d_model): positions = np.arange(seq_len)[:, np.newaxis] div_term = np.exp(np.arange(0, d_model, 2) * -(np.log(10000.0) / d_model)) pe = np.zeros((seq_len, d_model)) pe[:, 0::2] = np.sin(positions * div_term) pe[:, 1::2] = np.cos(positions * div_term) return pe

    class TransformerBlock(layers.Layer): def init(self, embed_dim, num_heads, ff_dim, rate=0.1): super(TransformerBlock, self).init() self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim) self.ffn = tf.keras.Sequential( [layers.Dense(ff_dim, activation=“relu”), layers.Dense(embed_dim)] ) self.layernorm1 = layers.LayerNormalization(epsilon=1e-6) self.layernorm2 = layers.LayerNormalization(epsilon=1e-6) self.dropout1 = layers.Dropout(rate) self.dropout2 = layers.Dropout(rate)

    def call(self, inputs, training=False): attn_output = self.att(inputs, inputs) attn_output = self.dropout1(attn_output, training=training) out1 = self.layernorm1(inputs + attn_output) ffn_output = self.ffn(out1) ffn_output = self.dropout2(ffn_output, training=training) return self.layernorm2(out1 + ffn_output)

    class TokenAndPositionEmbedding(layers.Layer): def init(self, maxlen, vocab_size, embed_dim): super(TokenAndPositionEmbedding, self).init() self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim) self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)

    def call(self, x): maxlen = tf.shape(x)[-1] positions = tf.range(start=0, limit=maxlen, delta=1) positions = self.pos_emb(positions) x = self.token_emb(x) return x + positions

    Model Construction

    vocab_size = 20000 # Example vocabulary size embed_dim = 200 # Embedding size for each token num_heads = 8 # Number of attention heads ff_dim = 2048 # Hidden layer size in feed forward network inside transformer

    inputs = layers.Input(shape=(200,)) embedding_layer = TokenAndPositionEmbedding(200, vocab_size, embed_dim) x = embedding_layer(inputs)

    transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim) x = transformer_block(x, training=True) # Just as an example. In practice, training can be dynamically set.

    x = layers.GlobalAveragePooling1D()(x) outputs = layers.Dense(vocab_size, activation=“softmax”)(x)

    model = models.Model(inputs=inputs, outputs=outputs) model.compile(optimizer=“adam”, loss=“sparse_categorical_crossentropy”, metrics=[“accuracy”]) model.summary()

    history = model.fit(x_train, y_train, epochs=10, batch_size = 2)

    is my code in google colab cells by cell and i will ask to you why in the cells “history = model.fit(x_train, y_train, epochs=10, batch_size = 2)” they reply to me an error message your goal is to fix the probleme by proposing a cells change : Epoch 1/10

    ValueError Traceback (most recent call last) in <cell line: 1>() ----> 1 history = model.fit(x_train, y_train, epochs=10, batch_size = 2)

    1 frames /usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py in error_handler(*args, **kwargs) 120 # To get the full stack trace, call: 121 # keras.config.disable_traceback_filtering() –> 122 raise e.with_traceback(filtered_tb) from None 123 finally: 124 del filtered_tb

    /usr/local/lib/python3.10/dist-packages/keras/src/layers/input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name) 243 if spec_dim is not None and dim is not None: 244 if spec_dim != dim: –> 245 raise ValueError( 246 f’Input {input_index} of layer “{layer_name}” is ' 247 "incompatible with the layer: "

    ValueError: Input 0 of layer “functional_140” is incompatible with the layer: expected shape=(None, 200), found shape=(2, 1, 200)

  2. Assistant

    The error message you're encountering indicates a shape mismatch between your model's expected input shape and the shape of the data you're trying to feed into it. The model expects input of shape (None, 200), but it is receiving data of shape (2, 1, 200).

    To fix this issue, you'll need to adjust the shape of your data (x_train and possibly y_train) to match what the model expects.

    It looks like an extra dimension has been introduced to your x_train and y_train arrays which the model is not expecting. The error message (2, 1, 200) suggests that for some reason your batch size (2 here) is being treated as a separate dimension, and there's an unnecessary 1 in the shape that likely needs to be removed.

    Here are recommended adjustments:

    1. Ensure proper preparation of your data. The model expects data of 2 dimensions per sample but is getting 3 dimensions due to the way the tokenized sequences being padded or batched. Ensure that x_train and y_train don't have an unnecessary middle dimension.

    2. Reshape or remove the unnecessary dimension. This can be done by reshaping your data to remove the unnecessary dimension (the extra '1'):

      Before feeding x_train and y_train into your model, try reshaping or squeezing out the unnecessary dimension:

      # Assuming x_train and y_train are NumPy arrays or TensorFlow tensors
      x_train = tf.squeeze(x_train, axis=1) # This should squeeze out the '1' dimension if it's unnecessary
      y_train = tf.squeeze(y_train, axis=1) # Adjust according to y_train's shape as well
      

      Make sure the operation does not alter the intended data structure (e.g., by removing a necessary sequence length dimension).

    3. Check the tokenization and batching logic. Review the code sections involved in tokenizing and preparing batches of your dataset to ensure they produce outputs in the expected shapes. The fr_padded and en_padded should be carefully checked to confirm they match the input shape anticipated by the model.

    4. Adjust the model's input layer if necessary. If the data preparation steps are correct and your task inherently requires different input dimensions (though less likely in this context), you may need to adjust the input shape expected by your model:

      inputs = layers.Input(shape=(x_train.shape[-2], x_train.shape[-1]))
      

    However, based on the error message and common practices, it seems the first approach (squeezing the unnecessary dimension) is likely to resolve your issue. Ensure that any reshaping aligns with the structure of your neural network and the nature of your task.