gpt-3.5-turbo-0301· OpenAI WildChat-4.8M (AI2)
请给我一个简单的关于tensorflow使用进行transformer的英法翻译任务训练的示例代码
- User
请给我一个简单的关于tensorflow使用进行transformer的英法翻译任务训练的示例代码
- Assistant
以下是一个简单的使用TensorFlow训练Transformer进行英法翻译的示例代码:
import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers # 定义Transformer模型 class Transformer(keras.Model): def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, dropout_rate=0.1): super(Transformer, self).__init__() self.encoder = layers.Embedding(input_vocab_size, d_model) self.encoder_dropout = layers.Dropout(dropout_rate) self.encoder_layers = [EncoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)] self.encoder_norm = layers.LayerNormalization(epsilon=1e-6) self.decoder = layers.Embedding(target_vocab_size, d_model) self.decoder_dropout = layers.Dropout(dropout_rate) self.decoder_layers = [DecoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)] self.decoder_norm = layers.LayerNormalization(epsilon=1e-6) self.final_layer = layers.Dense(target_vocab_size) def call(self, inputs, targets, training, encoder_padding_mask, look_ahead_mask, decoder_padding_mask): # 编码器部分 enc_output = self.encoder(inputs) enc_output = self.encoder_dropout(enc_output, training=training) for encoder_layer in self.encoder_layers: enc_output = encoder_layer(enc_output, encoder_padding_mask) enc_output = self.encoder_norm(enc_output) # 解码器部分 dec_output = self.decoder(targets) dec_output = self.decoder_dropout(dec_output, training=training) for decoder_layer in self.decoder_layers: dec_output = decoder_layer(dec_output, enc_output, look_ahead_mask, decoder_padding_mask) dec_output = self.decoder_norm(dec_output) # 输出部分 final_output = self.final_layer(dec_output) return final_output # 定义编码器层 class EncoderLayer(layers.Layer): def __init__(self, d_model, num_heads, dff, dropout_rate): super(EncoderLayer, self).__init__() self.multi_head_attention = layers.MultiHeadAttention(num_heads, d_model) self.dropout1 = layers.Dropout(dropout_rate) self.norm1 = layers.LayerNormalization(epsilon=1e-6) self.position_wise_feed_forward = keras.Sequential([ layers.Dense(dff, activation='relu'), layers.Dense(d_model) ]) self.dropout2 = layers.Dropout(dropout_rate) self.norm2 = layers.LayerNormalization(epsilon=1e-6) def call(self, inputs, mask): attention_output = self.multi_head_attention(inputs, inputs, inputs, mask) attention_output = self.dropout1(attention_output) attention_output = self.norm1(inputs + attention_output) feed_forward_output = self.position_wise_feed_forward(attention_output) feed_forward_output = self.dropout2(feed_forward_output) feed_forward_output = self.norm2(attention_output + feed_forward_output) return feed_forward_output # 定义解码器层 class DecoderLayer(layers.Layer): def __init__(self, d_model, num_heads, dff, dropout_rate): super(DecoderLayer, self).__init__() self.multi_head_attention1 = layers.MultiHeadAttention(num_heads, d_model) self.dropout1 = layers.Dropout(dropout_rate) self.norm1 = layers.LayerNormalization(epsilon=1e-6) self.multi_head_attention2 = layers.MultiHeadAttention(num_heads, d_model) self.dropout2 = layers.Dropout(dropout_rate) self.norm2 = layers.LayerNormalization(epsilon=1e-6) self.position_wise_feed_forward = keras.Sequential([ layers.Dense(dff, activation='relu'), layers.Dense(d_model) ]) self.dropout3 = layers.Dropout(dropout_rate) self.norm3 = layers.LayerNormalization(epsilon=1e-6) def call(self, inputs, enc_output, look_ahead_mask, padding_mask): attention_output1 = self.multi_head_attention1(inputs, inputs, inputs, look_ahead_mask) attention_output1 = self.dropout1(attention_output1) attention_output1 = self.norm1(inputs + attention_output1) attention_output2 = self.multi_head_attention2(attention_output1, enc_output, enc_output, padding_mask) attention_output2 = self.dropout2(attention_output2) attention_output2 = self.norm2(attention_output1 + attention_output2) feed_forward_output = self.position_wise_feed_forward(attention_output2) feed_forward_output = self.dropout3(feed_forward_output) feed_forward_output = self.norm3(attention_output2 + feed_forward_output) return feed_forward_output # 定义损失函数 def loss_function(y_true, y_pred): mask = tf.math.logical_not(tf.math.equal(y_true, 0)) loss_ = loss_object(y_true, y_pred) mask = tf.cast(mask, dtype=loss_.dtype) loss_ *= mask return tf.reduce_mean(loss_) # 定义学习率衰减函数 class CustomSchedule(keras.optimizers.schedules.LearningRateSchedule): def __init__(self, d_model, warmup_steps=4000): super(CustomSchedule, self).__init__() self.d_model = tf.cast(d_model, tf.float32) self.warmup_steps = warmup_steps def __call__(self, step): arg1 = tf.math.rsqrt(step) arg2 = step * (self.warmup_steps ** -1.5) return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2) # 定义训练过程 def train(model, dataset, epochs): optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.98, epsilon=1e-9) train_loss = keras.metrics.Mean(name='train_loss') for epoch in range(epochs): train_loss.reset_states() for (batch, (inputs, targets)) in enumerate(dataset): # 定义掩蔽 encoder_padding_mask, combined_mask, decoder_padding_mask = create_masks(inputs, targets) with tf.GradientTape() as tape: predictions = model(inputs, targets, True, encoder_padding_mask, combined_mask, decoder_padding_mask) loss = loss_function(targets[:, 1:], predictions) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) train_loss(loss) if batch % 50 == 0: print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, train_loss.result())) print('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result())) # 加载数据集并预处理 input_tensor, target_tensor, input_tokenizer, target_tokenizer = load_dataset(PATH_TO_DATASET) input_tensor_train, target_tensor_train, input_tensor_val, target_tensor_val = train_val_split(input_tensor, target_tensor) buffer_size = len(input_tensor_train) batch_size = 64 num_batches = buffer_size // batch_size vocab_inp_size = len(input_tokenizer.word_index) + 1 vocab_tar_size = len(target_tokenizer.word_index) + 1 # 根据数据集创建掩蔽 def create_masks(inputs, targets): encoder_padding_mask = create_padding_mask(inputs) decoder_padding_mask = create_padding_mask(inputs) look_ahead_mask = create_look_ahead_mask(tf.shape(targets)[1]) decoder_target_padding_mask = create_padding_mask(targets) combined_mask = tf.maximum(decoder_target_padding_mask, look_ahead_mask) return encoder_padding_mask, combined_mask, decoder_padding_mask # 创建模型并训练 transformer = Transformer(num_layers=4, d_model=128, num_heads=8, dff=512, input_vocab_size=vocab_inp_size, target_vocab_size=vocab_tar_size, dropout_rate=0.1) train_dataset = create_dataset(input_tensor_train, target_tensor_train, batch_size) train(transformer, train_dataset, EPOCHS) - User
请继续
- Assistant
以下是完整的代码,包括数据集预处理等部分:
import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers import numpy as np import re import os import time # 下载并解压数据集 path_to_zip = tf.keras.utils.get_file('spa-eng.zip', origin='http://download.tensorflow.org/data/spa-eng.zip', extract=True) path_to_dataset = os.path.join(os.path.dirname(path_to_zip), "spa-eng", "spa.txt") # 定义处理特殊字符的函数 def preprocess_sentence(sentence): sentence = sentence.lower().strip() sentence = re.sub(r"([?.!,¿])", r" \1 ", sentence) sentence = re.sub(r'[" "]+', " ", sentence) sentence = re.sub(r"[^a-zA-Z?.!,¿]+", " ", sentence) sentence = sentence.strip() sentence = '<start> ' + sentence + ' <end>' return sentence # 加载并处理数据集 def load_dataset(path, num_examples=None): lines = open(path, encoding='UTF-8').read().strip().split('\n') word_pairs = [[preprocess_sentence(w) for w in l.split('\t')] for l in lines[:num_examples]] input_tensor = [wp[0] for wp in word_pairs] target_tensor = [wp[1] for wp in word_pairs] input_tokenizer = keras.preprocessing.text.Tokenizer(filters='') input_tokenizer.fit_on_texts(input_tensor) input_tensor = input_tokenizer.texts_to_sequences(input_tensor) input_tensor = keras.preprocessing.sequence.pad_sequences(input_tensor, padding='post') target_tokenizer = keras.preprocessing.text.Tokenizer(filters='') target_tokenizer.fit_on_texts(target_tensor) target_tensor = target_tokenizer.texts_to_sequences(target_tensor) target_tensor = keras.preprocessing.sequence.pad_sequences(target_tensor, padding='post') return input_tensor, target_tensor, input_tokenizer, target_tokenizer # 划分训练集和验证集 def train_val_split(input_tensor, target_tensor, train_ratio=0.8): num_examples = len(input_tensor) indices = np.arange(num_examples) np.random.shuffle(indices) train_size = int(train_ratio * num_examples) train_indices = indices[:train_size] val_indices = indices[train_size:] input_tensor_train, target_tensor_train = input_tensor[train_indices], target_tensor[train_indices] input_tensor_val, target_tensor_val = input_tensor[val_indices], target_tensor[val_indices] return input_tensor_train, target_tensor_train, input_tensor_val, target_tensor_val # 创建掩蔽 def create_padding_mask(seq): seq = tf.cast(tf.math.equal(seq, 0), tf.float32) return seq[:, tf.newaxis, tf.newaxis, :] def create_look_ahead_mask(size): mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0) return mask # 创建数据集 def create_dataset(input_tensor, target_tensor, batch_size): dataset = tf.data.Dataset.from_tensor_slices((input_tensor, target_tensor)).shuffle(buffer_size) dataset = dataset.batch(batch_size, drop_remainder=True) return dataset # 定义Transformer模型 class Transformer(keras.Model): def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, dropout_rate=0.1): super(Transformer, self).__init__() self.encoder = layers.Embedding(input_vocab_size, d_model) self.encoder_dropout = layers.Dropout(dropout_rate) self.encoder_layers = [EncoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)] self.encoder_norm = layers.LayerNormalization(epsilon=1e-6) self.decoder = layers.Embedding(target_vocab_size, d_model) self.decoder_dropout = layers.Dropout(dropout_rate) self.decoder_layers = [DecoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)] self.decoder_norm = layers.LayerNormalization(epsilon=1e-6) self.final_layer = layers.Dense(target_vocab_size) def call(self, inputs, targets, training, encoder_padding_mask, look_ahead_mask, decoder_padding_mask): # 编码器部分 enc_output = self.encoder(inputs) enc_output = self.encoder_dropout(enc_output, training=training) for encoder_layer in self.encoder_layers: enc_output = encoder_layer(enc_output, encoder_padding_mask) enc_output = self.encoder_norm(enc_output) # 解码器部分 dec_output = self.decoder(targets) dec_output = self.decoder_dropout(dec_output, training=training) for decoder_layer in self.decoder_layers: dec_output = decoder_layer(dec_output, enc_output, look_ahead_mask, decoder_padding_mask) dec_output = self.decoder_norm(dec_output) # 输出部分 final_output = self.final_layer(dec_output) return final_output # 定义编码器层 class EncoderLayer(layers.Layer): def __init__(self, d_model, num_heads, dff, dropout_rate): super(EncoderLayer, self).__init__() self.multi_head_attention = layers.MultiHeadAttention(num_heads, d_model) self.dropout1 = layers.Dropout(dropout_rate) self.norm1 = layers.LayerNormalization(epsilon=1e-6) self.position_wise_feed_forward = keras.Sequential([ layers.Dense(dff, activation='relu'), layers.Dense(d_model) ]) self.dropout2 = layers.Dropout(dropout_rate) self.norm2 = layers.LayerNormalization(epsilon=1e-6) def call(self, inputs, mask): attention_output = self.multi_head_attention(inputs, inputs, inputs, mask) attention_output = self.dropout1(attention_output) attention_output = self.norm1(inputs + attention_output) feed_forward_output = self.position_wise_feed_forward(attention_output) feed_forward_output = self.dropout2(feed_forward_output) feed_forward_output = self.norm2(attention_output + feed_forward_output) return feed_forward_output # 定义解码器层 class DecoderLayer(layers.Layer): def __init__(self, d_model, num_heads, dff, dropout_rate): super(DecoderLayer, self).__init__() self.multi_head_attention1 = layers.MultiHeadAttention(num_heads, d_model) self.dropout1 = layers.Dropout(dropout_rate) self.norm1 = layers.LayerNormalization(epsilon=1e-6) self.multi_head_attention2 = layers.MultiHeadAttention(num_heads, d_model) self.dropout2 = layers.Dropout(dropout_rate) self.norm2 = layers.LayerNormalization(epsilon=1e-6) self.position_wise_feed_forward = keras.Sequential([ layers.Dense(dff, activation='relu'), layers.Dense(d_model) ]) self.dropout3 = layers.Dropout(dropout_rate) self.norm3 = layers.LayerNormalization(epsilon=1e-6) def call(self, inputs, enc_output, look_ahead_mask, padding_mask): attention_output1 = self.multi_head_attention1(inputs, inputs, inputs, look_ahead_mask) attention_output1 = self.dropout1(attention_output1) attention_output1 = self.norm1(inputs + attention_output1) attention_output2 = self.multi_head_attention2(attention_output1, enc_output, enc_output, padding_mask) attention_output2 = self.dropout2(attention_output2) attention_output2 = self.norm2(attention_output1 + attention_output2) feed_forward_output = self.position_wise_feed_forward(attention_output2) feed_forward_output = self.dropout3(feed_forward_output) feed_forward_output = self.norm3(attention_output2 + feed_forward_output) return feed_forward_output # 定义损失函数 def loss_function(y_true, y_pred): mask = tf.math.logical_not(tf.math.equal(y_true, 0)) loss_ = loss_object(y_true, y_pred) mask = tf.cast(mask, dtype=loss_.dtype) loss_ *= mask return tf.reduce_mean(loss_) # 定义学习率衰减函数 class CustomSchedule(keras.optimizers.schedules.LearningRateSchedule): def __init__(self, d_model, warmup_steps=4000): super(CustomSchedule, self).__init__() self.d_model = tf.cast(d_model, tf.float32) self.warmup_steps = warmup_steps def __call__(self, step): arg1 = tf.math.rsqrt(step) arg2 = step * (self.warmup_steps ** -1.5) return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2) # 定义训练过程 def train(model, dataset, epochs): optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.98, epsilon=1e-9) train_loss = keras.metrics.Mean(name='train_loss') for epoch in range(epochs): train_loss.reset_states() for (batch, (inputs, targets)) in enumerate(dataset): # 定义掩蔽 encoder_padding_mask, combined_mask, decoder_padding_mask = create_masks(inputs, targets) with tf.GradientTape() as tape: predictions = model(inputs, targets, True, encoder_padding_mask, combined_mask, decoder_padding_mask) loss = loss_function(targets[:, 1:], predictions) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) train_loss(loss) if batch % 50 == 0: print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, train_loss.result())) print('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result())) # 定义翻译函数 def translate(input_sentence, model, input_tokenizer, target_tokenizer, max_length_input, max_length_target): input_sequence = input_tokenizer.texts_to_sequences([input_sentence]) input_sequence = keras.preprocessing.sequence.pad_sequences(input_sequence, maxlen=max_length_input, padding='post') input_sequence = tf.convert_to_tensor(input_sequence) start_token = target_tokenizer.word_index['<start>'] end_token = target_tokenizer.word_index['<end>'] output