Untitled conversation
- User
{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "3UeycYCyxDfE" }, "source": [ "# TRANSLATOR" ] }, { "cell_type": "markdown", "metadata": { "id": "8BfUjVxBcz5N" }, "source": [ "## instalation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WXqM38xBRHu2" }, "outputs": [], "source": [ "%%time\n", "!pip install tensorflow-text\n", "!pip install datasets\n", "!pip install -q tensorflow_datasets\n", "!pip install pydot\n", "!cd /content\n", "!clear" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ukvs1XfMG7aG" }, "outputs": [], "source": [ "import tensorflow as tf\n", "import tensorflow_text as tf_text\n", "import tensorflow_datasets as tfds\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import requests\n", "import functools\n", "import collections\n", "import os\n", "import pathlib\n", "import re\n", "import string\n", "import tempfile\n", "import time\n", "import matplotlib.pyplot as plt\n", "import os\n", "import re\n", "import shutil\n", "import string\n", "import tensorflow as tf\n", "\n", "from tensorflow.keras import layers\n", "from tensorflow.keras import losses\n", "import pydot" ] }, { "cell_type": "markdown", "metadata": { "id": "7V7igFwpc6Hs" }, "source": [ "## dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZaMtoUtAREzs" }, "outputs": [], "source": [ "from datasets import load_dataset\n", "\n", "dataset = load_dataset("Helsinki-NLP/opus_books", "en-fr")\n", "data = dataset["train"]\n", "\n", "french_sentences = [example["fr"] for example in data["translation"][:127085]]\n", "english_sentences = [example["en"] for example in data["translation"][:127085]]\n", "dataset = tf.data.Dataset.from_tensor_slices((french_sentences, english_sentences))\n", "\n", "french_sentences_decoded = []\n", "english_sentences_decoded = []\n", "\n", "for french_sentence, english_sentence in dataset.take(127085):\n", " french_sentences_decoded.append("b '"+french_sentence.numpy().decode('utf-8'))\n", " english_sentences_decoded.append("b '"+english_sentence.numpy().decode('utf-8'))\n", "\n", "print("Nombre de phrases en français :", len(french_sentences_decoded))\n", "print("Nombre de phrases en anglais :", len(english_sentences_decoded))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2tb7H5uFQoBA" }, "outputs": [], "source": [ "train_fr = french_sentences\n", "train_en = english_sentences" ] }, { "cell_type": "markdown", "metadata": { "id": "NzS8h0budHWv" }, "source": [ "## vocab" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "id": "72uMXsQhFIx8" }, "outputs": [], "source": [ "from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab\n", "\n", "bert_tokenizer_params = dict(lower_case=True)\n", "reserved_tokens = ["[PAD]", "[UNK]", "[START]", "[END]"]\n", "\n", "bert_vocab_args = {\n", " 'vocab_size': 8000,\n", " 'reserved_tokens': reserved_tokens,\n", " 'bert_tokenizer_params': bert_tokenizer_params,\n", " 'learn_params': {},\n", "}\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "q6f3mrA-DK0n" }, "outputs": [], "source": [ "%%time\n", "en_vocab = bert_vocab.bert_vocab_from_dataset(\n", " tf.data.Dataset.from_tensor_slices(english_sentences).batch(1000).prefetch(2),\n", " **bert_vocab_args\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EJZzz1x5YY0x" }, "outputs": [], "source": [ "%%time\n", "fr_vocab = bert_vocab.bert_vocab_from_dataset(\n", " tf.data.Dataset.from_tensor_slices(french_sentences).batch(1000).prefetch(2),\n", " **bert_vocab_args\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1hmXuHHNcBHg" }, "outputs": [], "source": [ "def write_vocab_file(filepath, vocab):\n", " with open(filepath, 'w') as f:\n", " for token in vocab:\n", " print(token, file=f)\n", "write_vocab_file('en_vocab.txt', en_vocab)\n", "write_vocab_file('fr_vocab.txt', fr_vocab)" ] }, { "cell_type": "markdown", "metadata": { "id": "kLTf_mEvfNR9" }, "source": [ "#
TOKENIZER\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jGshnQ2idy8I" }, "outputs": [], "source": [ "fr_tokenizer = tf_text.BertTokenizer('fr_vocab.txt', **bert_tokenizer_params)\n", "en_tokenizer = tf_text.BertTokenizer('en_vocab.txt', **bert_tokenizer_params)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "bbQyYKhHkkDe" }, "outputs": [], "source": [ "# Tokenize the examples -> (batch, word, word-piece)\n", "en_tokenizere = en_tokenizer.tokenize("hello how are you Vadim")\n", "# Merge the word and word-piece axes -> (batch, tokens)\n", "en_tokenizere= en_tokenizere.merge_dims(-2,-1)\n", "\n", "for ex in en_tokenizere.to_list():\n", " print(ex)\n" ] }, { "cell_type": "code", "source": [ "words = en_tokenizer.detokenize(token_batch)\n", "tf.strings.reduce_join(words, separator=' ', axis=-1)" ], "metadata": { "id": "k0m1461Gwy3e" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "## model" ], "metadata": { "id": "BjoPdwoxBWw2" } }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7OqoDOybbwi6" }, "outputs": [], "source": [ "max_length = 200\n", "\n", "fr_sequences = [fr_tokenizer.tokenize(french_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n", " for french_sentence, _ in dataset.take(127085)]\n", "fr_ragged = tf.ragged.stack(fr_sequences)\n", "fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n", "\n", "fr_sequencesdeocde = [fr_tokenizer.tokenize("[START]"+french_sentence.numpy().decode('utf-8')+"[END]").merge_dims(-2,-1)\n", " for french_sentence, _ in dataset.take(127085)]\n", "fr_raggeddecode = tf.ragged.stack(fr_sequences)\n", "fr_paddeddecode = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n", "\n", "en_sequences = [en_tokenizer.tokenize(english_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n", " for _, english_sentence in dataset.take(127085)]\n", "en_ragged = tf.ragged.stack(en_sequences)\n", "en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n", "\n", "x_train = fr_padded\n", "x2_train = fr_paddeddecode\n", "y_train = en_padded\n", "\n" ] }, { "cell_type": "code", "source": [ "inputs = tf.keras.Input(shape=(1,200))\n", "embedding_dim = 256\n", "lstm_units = 512\n", "vocab_size_en = len(en_vocab) + len(reserved_tokens)\n", "vocab_size_fr = len(fr_vocab) + len(reserved_tokens)\n", "\n", "encoder_inputs = tf.keras.layers.Input(shape=(200,))\n", "encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_en, output_dim=embedding_dim, mask_zero=True)(encoder_inputs)\n", "encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(lstm_units, return_state=True)(encoder_embedding)\n", "encoder_states = [state_h, state_c]\n", "\n", "decoder_inputs = tf.keras.layers.Input(shape=(200,))\n", "decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(decoder_inputs)\n", "decoder_lstm = tf.keras.layers.LSTM(lstm_units, return_sequences=True, return_state=True)\n", "decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)\n", "decoder_dense = tf.keras.layers.Dense(vocab_size_fr, activation='softmax')\n", "decoder_outputs = decoder_dense(decoder_outputs)\n", "model = tf.keras.models.Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_outputs)\n", "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n", "\n", "\n", "model.summary()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 493 }, "id": "FIp6S-MR7Pls", "outputId": "0a2d9241-1984-4aa7-c1d1-c5a8d982e542" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "\u001b[1mModel: "functional_9"\u001b[0m\n" ], "text/html": [ "<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="font-weight: bold">Model: "functional_9"\n", "\n" ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to \u001b[0m\u001 - Assistant
I'm sorry, but it seems like you are providing code snippets for creating a translation model, including sections for installations, dataset preparation, vocabulary creation, tokenization, and model definition. Unfortunately, without specific questions or directions on what you need assistance with, I can't provide a focused answer. If you have any questions about the code, how to improve it, or need explanations about how certain parts work, please clarify so I can offer you relevant assistance.
- User
can you optimized this code for the training ,
- Assistant
To optimize the provided code for training, several adjustments can be made. These adjustments include improving data preprocessing, optimizing the model architecture, employing efficient data loading, and utilizing callbacks for better training control. Given the context, I'll outline a series of improvements based on standard practices.
1. Efficient Data Loading and Preprocessing
When working with large datasets, it's crucial to ensure that data loading and preprocessing steps do not become bottlenecks. TensorFlow and
tf.dataAPI provide useful functionalities for efficient data loading.- Data Batching and Prefetching: Use
dataset.batch()anddataset.prefetch()to ensure that the GPU doesn't stay idle while waiting for data to be loaded and preprocessed. Prefetching allows the data loading to happen on the CPU while the GPU works on training.
BUFFER_SIZE = 20000 BATCH_SIZE = 64 dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE) dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)2. Tokenization and Vocabulary Handling
The provided code generates vocabularies and tokenizes inputs efficiently, but ensure that these tokenized inputs are correctly padded and used in the model. Consider verifying that the sequences are appropriately padded to the same length for both inputs and outputs, and ensure that the tokenization process matches the vocabulary's specifics.
Consider caching the preprocessed/tokenized dataset for faster subsequent loads if you run multiple experiments:
# After preprocessing/tokenization dataset.cache("preprocessed_data.cache")3. Model Architecture Optimization
In the model architecture, the use of embeddings, LSTM layers, and a dense layer is a solid baseline. Consider the following for optimization:
- Use of
EmbeddingLayer: Ensure the embedding dimension and the LSTM units are not too large, as this could lead to overfitting and increased training time without significantly improving performance. - Regularization: Add dropout layers or use recurrent dropout in the LSTM layers to regularize and prevent overfitting.
encoder_outputs, state_h, state_c = tf.keras.layers.LSTM( lstm_units, return_state=True, dropout=0.5)(encoder_embedding)4. Training Procedure Optimization
- Learning Rate Schedule: Consider using a learning rate scheduler to adjust the learning rate during training; this often leads to better training outcomes.
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( initial_learning_rate=0.01, decay_steps=10000, decay_rate=0.9) optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)- Callbacks: Use
ModelCheckpointandEarlyStoppingto monitor the model's performance and avoid overfitting. This also helps in saving only the best model weights.
callbacks = [ tf.keras.callbacks.ModelCheckpoint("best_model.h5", save_best_only=True), tf.keras.callbacks.EarlyStopping(patience=5) ]Include these callbacks in your training routine:
model.fit([x_train, y_train], epochs=10, callbacks=callbacks)5. Reducing Memory Consumption
- If your dataset and model are too large to fit in memory, consider using a generator with
tf.keras.utils.Sequenceto load batches of data dynamically during training.
Final Note
Lastly, always start with a smaller subset of your data to ensure everything works as expected before scaling up to the entire dataset. This approach saves time and resources in the early stages of model development and debugging.
These suggestions aim to enhance efficiency and performance. However, it's crucial to test changes incrementally to monitor their impact.
- Data Batching and Prefetching: Use
- User
what is the problem : Spaces Hugging Face's logo
xelosz / Translator private
Runtime error 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "", line 1030, in _gcd_import File "", line 1007, in _find_and_load File "", line 986, in _find_and_load_unlocked File "", line 680, in _load_unlocked File "", line 850, in exec_module File "", line 228, in _call_with_frames_removed File "/code/app.py", line 28, in dataset = load_dataset("Helsinki-NLP/opus_books", "en-fr") File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 2556, in load_dataset builder_instance = load_dataset_builder( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 2228, in load_dataset_builder dataset_module = dataset_module_factory( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1879, in dataset_module_factory raise e1 from None File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1854, in dataset_module_factory return HubDatasetModuleFactoryWithoutScript( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1206, in get_module dataset_readme_path = cached_path( File "/usr/local/lib/python3.9/site-packages/datasets/utils/file_utils.py", line 190, in cached_path output_path = get_from_cache( File "/usr/local/lib/python3.9/site-packages/datasets/utils/file_utils.py", line 493, in get_from_cache os.makedirs(cache_dir, exist_ok=True) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 225, in makedirs mkdir(name, mode) PermissionError: [Errno 13] Permission denied: '/.cache' Container logs:
===== Application Startup at 2024-04-17 10:56:14 =====
2024-04-17 10:57:04.209178: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable
TF_ENABLE_ONEDNN_OPTS=0. 2024-04-17 10:57:04.210358: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used. 2024-04-17 10:57:04.215257: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used. 2024-04-17 10:57:04.259693: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-04-17 10:57:05.150089: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Matplotlib created a temporary config/cache directory at /tmp/matplotlib-rvd05oes because the default path (/.config/matplotlib) is not a writable directory; it is highly recommended to set the MPLCONFIGDIR environment variable to a writable directory, in particular to speed up the import of Matplotlib and to better support multiprocessing. Fontconfig error: No writable cache directories Fontconfig error: No writable cache directories Fontconfig error: No writable cache directories Fontconfig error: No writable cache directories Traceback (most recent call last): File "/usr/local/bin/shiny", line 8, in sys.exit(main()) File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1157, in call return self.main(*args, **kwargs) File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1078, in main rv = self.invoke(ctx) File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1688, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1434, in invoke return ctx.invoke(self.callback, **ctx.params) File "/usr/local/lib/python3.9/site-packages/click/core.py", line 783, in invoke return __callback(*args, **kwargs) File "/usr/local/lib/python3.9/site-packages/shiny/_main.py", line 174, in run return run_app( File "/usr/local/lib/python3.9/site-packages/shiny/_main.py", line 356, in run_app uvicorn.run( # pyright: ignore[reportUnknownMemberType] File "/usr/local/lib/python3.9/site-packages/uvicorn/main.py", line 575, in run server.run() File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 65, in run return asyncio.run(self.serve(sockets=sockets)) File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run return loop.run_until_complete(main) File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete return future.result() File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 69, in serve await self._serve(sockets) File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 76, in _serve config.load() File "/usr/local/lib/python3.9/site-packages/uvicorn/config.py", line 433, in load self.loaded_app = import_from_string(self.app) File "/usr/local/lib/python3.9/site-packages/uvicorn/importer.py", line 19, in import_from_string module = importlib.import_module(module_str) File "/usr/local/lib/python3.9/importlib/init.py", line 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "", line 1030, in _gcd_import File "", line 1007, in _find_and_load File "", line 986, in _find_and_load_unlocked File "", line 680, in _load_unlocked File "", line 850, in exec_module File "", line 228, in _call_with_frames_removed File "/code/app.py", line 28, in dataset = load_dataset("Helsinki-NLP/opus_books", "en-fr") File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 2556, in load_dataset builder_instance = load_dataset_builder( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 2228, in load_dataset_builder dataset_module = dataset_module_factory( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1879, in dataset_module_factory raise e1 from None File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1854, in dataset_module_factory return HubDatasetModuleFactoryWithoutScript( File "/usr/local/lib/python3.9/site-packages/datasets/load.py", line 1206, in get_module dataset_readme_path = cached_path( File "/usr/local/lib/python3.9/site-packages/datasets/utils/file_utils.py", line 190, in cached_path output_path = get_from_cache( File "/usr/local/lib/python3.9/site-packages/datasets/utils/file_utils.py", line 493, in get_from_cache os.makedirs(cache_dir, exist_ok=True) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 215, in makedirs makedirs(head, exist_ok=exist_ok) File "/usr/local/lib/python3.9/os.py", line 225, in makedirs mkdir(name, mode) PermissionError: [Errno 13] Permission denied: '/.cache' - Assistant
The error message you're encountering,
PermissionError: [Errno 13] Permission denied: '/.cache', indicates that the process running your code does not have the necessary permissions to create or access the/.cachedirectory. This directory is typically used by many libraries, includingdatasetsfrom Hugging Face, to cache data or models locally to avoid re-downloading them in future uses.This problem can occur in restricted environments, like some containerized setups (Docker, for example) or managed environments where you have limited control over the filesystem. Here are several approaches to address this issue:
1. Specify a writable cache directory
Libraries often allow you to specify an alternative directory for cache purposes. For Hugging Face's
datasets, you can set theHF_HOMEenvironment variable to a writable directory. Before your code that loads the dataset, you might include:import os # Set HF_HOME to a directory where you have write permissions os.environ["HF_HOME"] = "/path/to/writable/cache/directory" # Then load your dataset from datasets import load_dataset dataset = load_dataset("Helsinki-NLP/opus_books", "en-fr")2. Use directory permissions
If you're working in a Docker container or a similar environment and you do have some control over it, ensure that the process running your Python script has the necessary permissions to write to the
/.cachedirectory or wherever you choose to store your cache.If changing permissions is an option:
# This command changes the permissions recursively for /.cache directory chmod -R 777 /.cacheOr, you can choose to run your script as a user with appropriate write permissions to the desired directory.
3. Configure the environment in a containerized setup
If you're using Docker, Kubernetes, or any containerized environment, you can map a host directory (which you have permissions for) to the container's
/.cachedirectory. For Docker, this can be done using volume mapping:docker run -v /host/directory/with/permissions:/.cache my_image4. Adjust the code to avoid using a global cache directory
If the issue persists or you're in an environment where you cannot easily change permissions or set environment variables, you might need to adjust your code to avoid operations requiring writing to the global cache directory. For example, you can manually download the dataset and load it from a local path or use APIs that allow specifying cache paths explicitly.
Always ensure that the path you choose to use as the cache directory exists and is writable by the process executing your code to avoid similar issues.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.