Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· openaiHugging Face Suzhen/CodeChat

import tensorflow as tf import numpy as np Hyperparameters batchsize = 96 blocksize = 20…

  1. User

    import tensorflow as tf import numpy as np

    Hyperparameters

    batch_size = 96 block_size = 20 max_iters = 200 eval_interval = 100 learning_rate = 1e-3 device = 'cuda:0' if tf.test.is_gpu_available() else 'cpu' eval_iters = 20 # Changed to 20 for consistency with eval_interval n_embd = 16 n_head = 2 n_layer = 2 dropout_rate = 0.0 # Renamed from dropout for clarity

    hidden_nodes = 20

    tf.random.set_seed(1337)

    with open('data/universe.txt', 'r', encoding='utf-8') as f: text = f.read()

    data = np.array([ord(c) for c in text], dtype=np.int32) n = int(0.9 * len(data)) train_data = data[:n] val_data = data[n:]

    @tf.function def get_batch(split): data = train_data if split == 'train' else val_data ix = np.random.randint(0, len(data) - block_size, size=(batch_size,)) x = np.array([data[i:i + block_size] for i in ix]) y = np.array([data[i + 1:i + block_size + 1] for i in ix]) return x, y

    class GPTLanguageModel(tf.keras.Model): def init(self, vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate=0.1): super(GPTLanguageModel, self).init() self.token_embedding_table = tf.keras.layers.Embedding(vocab_size, n_embd) self.position_embedding_table = tf.keras.layers.Embedding(block_size, n_embd) self.blocks = [self._build_block(n_embd, n_head) for _ in range(n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=1e-6) self.lm_head = tf.keras.layers.Dense(vocab_size)

    def _build_block(self, n_embd, n_head):
        head_size = n_embd // n_head  # Bug fixed, removed '+ 1'
        return tf.keras.Sequential([
            tf.keras.layers.LayerNormalization(epsilon=1e-6),
            tf.keras.layers.MultiHeadAttention(num_heads=n_head, key_dim=head_size),
            tf.keras.layers.Dense(4 * n_embd, activation='relu'),
            tf.keras.layers.Dense(n_embd)
        ])
    
    def call(self, idx, targets=None, training=False):
        B, T = tf.shape(idx)[0], tf.shape(idx)[1]
        tok_emb = self.token_embedding_table(idx)
        pos_emb = self.position_embedding_table(tf.range(T))
        x = tok_emb + tf.cast(pos_emb[None, :, :], tok_emb.dtype)  # Explicit casting to match dtype
    
        for block in self.blocks:
            x = block(x, training=training)
    
        x = self.ln_f(x)
        logits = self.lm_head(x)
    
        loss = None
        if targets is not None:
            loss = self.compute_loss(logits, targets)
    
        return logits, loss
    
    def compute_loss(self, logits, targets):
        loss = tf.keras.losses.sparse_categorical_crossentropy(targets, logits, from_logits=True)
        return tf.reduce_mean(loss)
    
    def generate(self, idx, max_new_tokens):
        for _ in range(max_new_tokens):
            logits, _ = self(idx[:, -block_size:], training=False)
            logits = logits[:, -1, :]
            idx_next = tf.random.categorical(logits, num_samples=1, dtype=tf.int32)
            idx = tf.concat([idx, idx_next], axis=1)
        return idx
    

    vocab_size = len(set(data))

    model = GPTLanguageModel(vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate)

    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

    train_loss_metric = tf.keras.metrics.Mean(name='train_loss') val_loss_metric = tf.keras.metrics.Mean(name='val_loss')

    @tf.function def train_step(inputs, targets): with tf.GradientTape() as tape: _, loss = model(inputs, targets, training=True) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) train_loss_metric.update_state(loss)

    @tf.function def val_step(inputs, targets): _, loss = model(inputs, targets, training=False) val_loss_metric.update_state(loss)

    Convert numpy operations to Tensor operations

    def numpy_to_tensor(numpy_function): def tensor_function(*args, **kwargs): tensor_args = [tf.convert_to_tensor(arg) for arg in args] return numpy_function(*tensor_args, **kwargs) return tensor_function

    get_batch = numpy_to_tensor(get_batch)

    for iteration in range(max_iters): for _ in range(eval_interval): inputs, targets = get_batch('train') train_step(inputs, targets)

    for _ in range(eval_iters):
        inputs, targets = get_batch('val')
        val_step(inputs, targets)
    
    if (iteration + 1) % eval_interval == 0:
        train_loss = train_loss_metric.result()
        val_loss = val_loss_metric.result()
        print(f'Iteration {iteration + 1}: Train Loss = {train_loss:.4f}, Val Loss = {val_loss:.4f}')
        train_loss_metric.reset_states()
        val_loss_metric.reset_states()
    

    model.save('Universe-GPT-tf.keras', save_format='tf')

  2. Assistant

    This code is creating a simple GPT (Generative Pretrained Transformer) style language model in TensorFlow and training it on a text dataset. The model is defined using the TensorFlow Keras API and is trained to predict the next character in a sequence of characters from the dataset. The dataset is split into training and validation sets, with 90% of the data used for training and the rest for validation.

    Here's a breakdown of the code:

    1. Hyperparameters and Device Selection: The code sets various hyperparameters like batch size, learning rate, number of layers and heads in the transformer, etc. It determines if a GPU is available and sets the device accordingly.

    2. Load Data: The text data is loaded from a local file universe.txt, converted to integer ASCII codes, and split into training and validation datasets.

    3. Batch Preparation: A function get_batch provides random batches of sequential character data for training the model. This function slices the input text to generate sequences (x) and their corresponding target characters (y).

    4. Model Definition: The GPT model (GPTLanguageModel) is created as a subclass of tf.keras.Model. It consists of token embeddings, positional embeddings, several transformer blocks, and a final layer normalization followed by a dense layer to predict logits for each character in the vocabulary.

    5. Training and Validation Steps: Separate functions train_step and val_step are defined using TensorFlow's @tf.function decorator to perform optimization and loss calculation for training and validation iterations, respectively.

    6. Training Loop: The main loop iterates through the specified number of iterations, calling train_step and val_step to update the model.

    7. Saving the Model: After training, the model is saved to the local disk.

    There are some points to note about the code:

    • device = 'cuda:0' if tf.test.is_gpu_available() else 'cpu' is a leftover line since the code uses TensorFlow's way of managing GPU usage automatically.
    • The code has corrected a bug concerning dividing n_embd by n_head to determine the size for each attention head.
    • The function numpy_to_tensor transforms the get_batch function to handle TensorFlow tensors instead of NumPy arrays, which could improve performance by ensuring compatibility with TensorFlow's graph execution.
    • This script does not have explicit statements to manage GPU memory or device placement, as TensorFlow 2+ is more automated in this aspect compared to TensorFlow 1.x.

    Please ensure that the data/universe.txt file is present in the specified path before running the script, otherwise, it will result in a file not found error.

    Remember to adjust hyperparameters and model complexity based on the available compute resources to avoid out-of-memory errors on your GPU or long training times. Also, the provided code snippet assumes a perfect scenario for simplicity and may require error handling and additional optimizations for production-level code.

  3. User

    when running this code: import tensorflow as tf import numpy as np

    block_size = 20

    with open('data/universe.txt', 'r', encoding='utf-8') as f: text = f.read()

    data = np.array([ord(c) for c in text], dtype=np.int32) vocab_size = len(set(data))

    Load the saved model

    model = tf.keras.models.load_model('Universe-GPT-tf.keras')

    Set the maximum number of new tokens to generate

    max_new_tokens = 100

    Initialize the input sequence with a random starting point

    input_seq = np.random.randint(0, vocab_size, size=(1, block_size))

    Generate new text

    generated_text = model.generate(input_seq, max_new_tokens)

    Convert the generated text back to characters

    generated_text = ''.join([chr(c) for c in generated_text[0]])

    print("Generated Text:") print(generated_text) i get this error: TypeError: Could not locate class 'GPTLanguageModel'. Make sure custom classes are decorated with @keras.saving.register_keras_serializable(). Full object config: {'module': None, 'class_name': 'GPTLanguageModel', 'config': {'dropout_rate': 0.0, 'vocab_size': 95, 'n_embd': 16, 'block_size': 20, 'n_head': 2, 'n_layer': 2}, 'registered_name': 'GPTLanguageModel'}

  4. Assistant

    The error you're encountering occurs because TensorFlow needs to know how to deserialize custom classes (like your GPTLanguageModel) when loading a model. In TensorFlow, models containing custom layers or classes must be provided with the necessary information to reconstruct those custom components upon loading.

    To resolve this issue, you can take the following steps:

    1. Ensure that the class with the same definition as when the model was saved is available in the environment. This means that you should define the GPTLanguageModel class exactly as it was during training before trying to load the model.

    2. Register the custom class with @tf.keras.utils.register_keras_serializable() decorator, so TensorFlow knows how to serialize and deserialize it.

    Here's how you could adjust your GPTLanguageModel class definition accordingly:

    import tensorflow as tf
    
    # Decorate the class to register it as serializable
    @tf.keras.utils.register_keras_serializable()
    class GPTLanguageModel(tf.keras.Model):
        def __init__(self, vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate=0.1):
            super(GPTLanguageModel, self).__init__()
            # ... [rest of your init code]
    
        # ... [rest of your class methods]
    

    After making this change, you can try saving your model again:

    # ... [code to train your model]
    model.save('Universe-GPT-tf.keras', save_format='tf')
    

    Now, when you load the model, you must ensure that the GPTLanguageModel class definition is present in the environment and TensorFlow knows how to deserialize it:

    import tensorflow as tf
    import numpy as np
    
    # ... [GPTLanguageModel class definition]
    
    # ... [rest of your code]
    
    # Load the saved model
    model = tf.keras.models.load_model('Universe-GPT-tf.keras', custom_objects={'GPTLanguageModel': GPTLanguageModel})
    

    By importing the necessary custom classes (or defining them in the script), and using the custom_objects argument in load_model, TensorFlow should be able to load your saved model without the TypeError. Make sure that the class definition is consistent between the time you save and load the model, or TensorFlow might not recognize it as the same class.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.