import tensorflow as tf import numpy as np Hyperparameters batchsize = 96 blocksize = 20…
- User
import tensorflow as tf import numpy as np
Hyperparameters
batch_size = 96 block_size = 20 max_iters = 200 eval_interval = 100 learning_rate = 1e-3 device = 'cuda:0' if tf.test.is_gpu_available() else 'cpu' eval_iters = 20 # Changed to 20 for consistency with eval_interval n_embd = 16 n_head = 2 n_layer = 2 dropout_rate = 0.0 # Renamed from dropout for clarity
hidden_nodes = 20
tf.random.set_seed(1337)
with open('data/universe.txt', 'r', encoding='utf-8') as f: text = f.read()
data = np.array([ord(c) for c in text], dtype=np.int32) n = int(0.9 * len(data)) train_data = data[:n] val_data = data[n:]
@tf.function def get_batch(split): data = train_data if split == 'train' else val_data ix = np.random.randint(0, len(data) - block_size, size=(batch_size,)) x = np.array([data[i:i + block_size] for i in ix]) y = np.array([data[i + 1:i + block_size + 1] for i in ix]) return x, y
class GPTLanguageModel(tf.keras.Model): def init(self, vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate=0.1): super(GPTLanguageModel, self).init() self.token_embedding_table = tf.keras.layers.Embedding(vocab_size, n_embd) self.position_embedding_table = tf.keras.layers.Embedding(block_size, n_embd) self.blocks = [self._build_block(n_embd, n_head) for _ in range(n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=1e-6) self.lm_head = tf.keras.layers.Dense(vocab_size)
def _build_block(self, n_embd, n_head): head_size = n_embd // n_head # Bug fixed, removed '+ 1' return tf.keras.Sequential([ tf.keras.layers.LayerNormalization(epsilon=1e-6), tf.keras.layers.MultiHeadAttention(num_heads=n_head, key_dim=head_size), tf.keras.layers.Dense(4 * n_embd, activation='relu'), tf.keras.layers.Dense(n_embd) ]) def call(self, idx, targets=None, training=False): B, T = tf.shape(idx)[0], tf.shape(idx)[1] tok_emb = self.token_embedding_table(idx) pos_emb = self.position_embedding_table(tf.range(T)) x = tok_emb + tf.cast(pos_emb[None, :, :], tok_emb.dtype) # Explicit casting to match dtype for block in self.blocks: x = block(x, training=training) x = self.ln_f(x) logits = self.lm_head(x) loss = None if targets is not None: loss = self.compute_loss(logits, targets) return logits, loss def compute_loss(self, logits, targets): loss = tf.keras.losses.sparse_categorical_crossentropy(targets, logits, from_logits=True) return tf.reduce_mean(loss) def generate(self, idx, max_new_tokens): for _ in range(max_new_tokens): logits, _ = self(idx[:, -block_size:], training=False) logits = logits[:, -1, :] idx_next = tf.random.categorical(logits, num_samples=1, dtype=tf.int32) idx = tf.concat([idx, idx_next], axis=1) return idxvocab_size = len(set(data))
model = GPTLanguageModel(vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate)
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
train_loss_metric = tf.keras.metrics.Mean(name='train_loss') val_loss_metric = tf.keras.metrics.Mean(name='val_loss')
@tf.function def train_step(inputs, targets): with tf.GradientTape() as tape: _, loss = model(inputs, targets, training=True) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) train_loss_metric.update_state(loss)
@tf.function def val_step(inputs, targets): _, loss = model(inputs, targets, training=False) val_loss_metric.update_state(loss)
Convert numpy operations to Tensor operations
def numpy_to_tensor(numpy_function): def tensor_function(*args, **kwargs): tensor_args = [tf.convert_to_tensor(arg) for arg in args] return numpy_function(*tensor_args, **kwargs) return tensor_function
get_batch = numpy_to_tensor(get_batch)
for iteration in range(max_iters): for _ in range(eval_interval): inputs, targets = get_batch('train') train_step(inputs, targets)
for _ in range(eval_iters): inputs, targets = get_batch('val') val_step(inputs, targets) if (iteration + 1) % eval_interval == 0: train_loss = train_loss_metric.result() val_loss = val_loss_metric.result() print(f'Iteration {iteration + 1}: Train Loss = {train_loss:.4f}, Val Loss = {val_loss:.4f}') train_loss_metric.reset_states() val_loss_metric.reset_states()model.save('Universe-GPT-tf.keras', save_format='tf')
- Assistant
This code is creating a simple GPT (Generative Pretrained Transformer) style language model in TensorFlow and training it on a text dataset. The model is defined using the TensorFlow Keras API and is trained to predict the next character in a sequence of characters from the dataset. The dataset is split into training and validation sets, with 90% of the data used for training and the rest for validation.
Here's a breakdown of the code:
-
Hyperparameters and Device Selection: The code sets various hyperparameters like batch size, learning rate, number of layers and heads in the transformer, etc. It determines if a GPU is available and sets the device accordingly.
-
Load Data: The text data is loaded from a local file
universe.txt, converted to integer ASCII codes, and split into training and validation datasets. -
Batch Preparation: A function
get_batchprovides random batches of sequential character data for training the model. This function slices the input text to generate sequences (x) and their corresponding target characters (y). -
Model Definition: The GPT model (
GPTLanguageModel) is created as a subclass oftf.keras.Model. It consists of token embeddings, positional embeddings, several transformer blocks, and a final layer normalization followed by a dense layer to predict logits for each character in the vocabulary. -
Training and Validation Steps: Separate functions
train_stepandval_stepare defined using TensorFlow's@tf.functiondecorator to perform optimization and loss calculation for training and validation iterations, respectively. -
Training Loop: The main loop iterates through the specified number of iterations, calling
train_stepandval_stepto update the model. -
Saving the Model: After training, the model is saved to the local disk.
There are some points to note about the code:
device = 'cuda:0' if tf.test.is_gpu_available() else 'cpu'is a leftover line since the code uses TensorFlow's way of managing GPU usage automatically.- The code has corrected a bug concerning dividing
n_embdbyn_headto determine the size for each attention head. - The function
numpy_to_tensortransforms theget_batchfunction to handle TensorFlow tensors instead of NumPy arrays, which could improve performance by ensuring compatibility with TensorFlow's graph execution. - This script does not have explicit statements to manage GPU memory or device placement, as TensorFlow 2+ is more automated in this aspect compared to TensorFlow 1.x.
Please ensure that the
data/universe.txtfile is present in the specified path before running the script, otherwise, it will result in a file not found error.Remember to adjust hyperparameters and model complexity based on the available compute resources to avoid out-of-memory errors on your GPU or long training times. Also, the provided code snippet assumes a perfect scenario for simplicity and may require error handling and additional optimizations for production-level code.
-
- User
when running this code: import tensorflow as tf import numpy as np
block_size = 20
with open('data/universe.txt', 'r', encoding='utf-8') as f: text = f.read()
data = np.array([ord(c) for c in text], dtype=np.int32) vocab_size = len(set(data))
Load the saved model
model = tf.keras.models.load_model('Universe-GPT-tf.keras')
Set the maximum number of new tokens to generate
max_new_tokens = 100
Initialize the input sequence with a random starting point
input_seq = np.random.randint(0, vocab_size, size=(1, block_size))
Generate new text
generated_text = model.generate(input_seq, max_new_tokens)
Convert the generated text back to characters
generated_text = ''.join([chr(c) for c in generated_text[0]])
print("Generated Text:") print(generated_text) i get this error: TypeError: Could not locate class 'GPTLanguageModel'. Make sure custom classes are decorated with
@keras.saving.register_keras_serializable(). Full object config: {'module': None, 'class_name': 'GPTLanguageModel', 'config': {'dropout_rate': 0.0, 'vocab_size': 95, 'n_embd': 16, 'block_size': 20, 'n_head': 2, 'n_layer': 2}, 'registered_name': 'GPTLanguageModel'} - Assistant
The error you're encountering occurs because TensorFlow needs to know how to deserialize custom classes (like your
GPTLanguageModel) when loading a model. In TensorFlow, models containing custom layers or classes must be provided with the necessary information to reconstruct those custom components upon loading.To resolve this issue, you can take the following steps:
-
Ensure that the class with the same definition as when the model was saved is available in the environment. This means that you should define the
GPTLanguageModelclass exactly as it was during training before trying to load the model. -
Register the custom class with
@tf.keras.utils.register_keras_serializable()decorator, so TensorFlow knows how to serialize and deserialize it.
Here's how you could adjust your
GPTLanguageModelclass definition accordingly:import tensorflow as tf # Decorate the class to register it as serializable @tf.keras.utils.register_keras_serializable() class GPTLanguageModel(tf.keras.Model): def __init__(self, vocab_size, n_embd, block_size, n_head, n_layer, dropout_rate=0.1): super(GPTLanguageModel, self).__init__() # ... [rest of your init code] # ... [rest of your class methods]After making this change, you can try saving your model again:
# ... [code to train your model] model.save('Universe-GPT-tf.keras', save_format='tf')Now, when you load the model, you must ensure that the
GPTLanguageModelclass definition is present in the environment and TensorFlow knows how to deserialize it:import tensorflow as tf import numpy as np # ... [GPTLanguageModel class definition] # ... [rest of your code] # Load the saved model model = tf.keras.models.load_model('Universe-GPT-tf.keras', custom_objects={'GPTLanguageModel': GPTLanguageModel})By importing the necessary custom classes (or defining them in the script), and using the
custom_objectsargument inload_model, TensorFlow should be able to load your saved model without theTypeError. Make sure that the class definition is consistent between the time you save and load the model, or TensorFlow might not recognize it as the same class. -
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.