Skip to content
The Chat Archive
gpt-3.5-turbo-0301· OpenAI WildChat-4.8M (AI2)

请给我一个简单的关于tensorflow使用进行transformer的英法翻译任务训练的示例代码

  1. User

    请给我一个简单的关于tensorflow使用进行transformer的英法翻译任务训练的示例代码

  2. Assistant

    以下是一个简单的使用TensorFlow训练Transformer进行英法翻译的示例代码:

    import tensorflow as tf
    from tensorflow import keras
    from tensorflow.keras import layers
    
    # 定义Transformer模型
    class Transformer(keras.Model):
        def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, dropout_rate=0.1):
            super(Transformer, self).__init__()
    
            self.encoder = layers.Embedding(input_vocab_size, d_model)
            self.encoder_dropout = layers.Dropout(dropout_rate)
            self.encoder_layers = [EncoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]
            self.encoder_norm = layers.LayerNormalization(epsilon=1e-6)
    
            self.decoder = layers.Embedding(target_vocab_size, d_model)
            self.decoder_dropout = layers.Dropout(dropout_rate)
            self.decoder_layers = [DecoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]
            self.decoder_norm = layers.LayerNormalization(epsilon=1e-6)
    
            self.final_layer = layers.Dense(target_vocab_size)
    
        def call(self, inputs, targets, training, encoder_padding_mask, look_ahead_mask, decoder_padding_mask):
            # 编码器部分
            enc_output = self.encoder(inputs)
            enc_output = self.encoder_dropout(enc_output, training=training)
    
            for encoder_layer in self.encoder_layers:
                enc_output = encoder_layer(enc_output, encoder_padding_mask)
    
            enc_output = self.encoder_norm(enc_output)
    
            # 解码器部分
            dec_output = self.decoder(targets)
            dec_output = self.decoder_dropout(dec_output, training=training)
    
            for decoder_layer in self.decoder_layers:
                dec_output = decoder_layer(dec_output, enc_output, look_ahead_mask, decoder_padding_mask)
    
            dec_output = self.decoder_norm(dec_output)
    
            # 输出部分
            final_output = self.final_layer(dec_output)
    
            return final_output
    
    # 定义编码器层
    class EncoderLayer(layers.Layer):
        def __init__(self, d_model, num_heads, dff, dropout_rate):
            super(EncoderLayer, self).__init__()
    
            self.multi_head_attention = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout1 = layers.Dropout(dropout_rate)
            self.norm1 = layers.LayerNormalization(epsilon=1e-6)
    
            self.position_wise_feed_forward = keras.Sequential([
                layers.Dense(dff, activation='relu'),
                layers.Dense(d_model)
            ])
            self.dropout2 = layers.Dropout(dropout_rate)
            self.norm2 = layers.LayerNormalization(epsilon=1e-6)
    
        def call(self, inputs, mask):
            attention_output = self.multi_head_attention(inputs, inputs, inputs, mask)
            attention_output = self.dropout1(attention_output)
            attention_output = self.norm1(inputs + attention_output)
    
            feed_forward_output = self.position_wise_feed_forward(attention_output)
            feed_forward_output = self.dropout2(feed_forward_output)
            feed_forward_output = self.norm2(attention_output + feed_forward_output)
    
            return feed_forward_output
    
    # 定义解码器层
    class DecoderLayer(layers.Layer):
        def __init__(self, d_model, num_heads, dff, dropout_rate):
            super(DecoderLayer, self).__init__()
    
            self.multi_head_attention1 = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout1 = layers.Dropout(dropout_rate)
            self.norm1 = layers.LayerNormalization(epsilon=1e-6)
    
            self.multi_head_attention2 = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout2 = layers.Dropout(dropout_rate)
            self.norm2 = layers.LayerNormalization(epsilon=1e-6)
    
            self.position_wise_feed_forward = keras.Sequential([
                layers.Dense(dff, activation='relu'),
                layers.Dense(d_model)
            ])
            self.dropout3 = layers.Dropout(dropout_rate)
            self.norm3 = layers.LayerNormalization(epsilon=1e-6)
    
        def call(self, inputs, enc_output, look_ahead_mask, padding_mask):
            attention_output1 = self.multi_head_attention1(inputs, inputs, inputs, look_ahead_mask)
            attention_output1 = self.dropout1(attention_output1)
            attention_output1 = self.norm1(inputs + attention_output1)
    
            attention_output2 = self.multi_head_attention2(attention_output1, enc_output, enc_output, padding_mask)
            attention_output2 = self.dropout2(attention_output2)
            attention_output2 = self.norm2(attention_output1 + attention_output2)
    
            feed_forward_output = self.position_wise_feed_forward(attention_output2)
            feed_forward_output = self.dropout3(feed_forward_output)
            feed_forward_output = self.norm3(attention_output2 + feed_forward_output)
    
            return feed_forward_output
    
    # 定义损失函数
    def loss_function(y_true, y_pred):
        mask = tf.math.logical_not(tf.math.equal(y_true, 0))
        loss_ = loss_object(y_true, y_pred)
    
        mask = tf.cast(mask, dtype=loss_.dtype)
        loss_ *= mask
    
        return tf.reduce_mean(loss_)
    
    # 定义学习率衰减函数
    class CustomSchedule(keras.optimizers.schedules.LearningRateSchedule):
        def __init__(self, d_model, warmup_steps=4000):
            super(CustomSchedule, self).__init__()
    
            self.d_model = tf.cast(d_model, tf.float32)
            self.warmup_steps = warmup_steps
    
        def __call__(self, step):
            arg1 = tf.math.rsqrt(step)
            arg2 = step * (self.warmup_steps ** -1.5)
    
            return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
    
    # 定义训练过程
    def train(model, dataset, epochs):
        optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.98, epsilon=1e-9)
        train_loss = keras.metrics.Mean(name='train_loss')
    
        for epoch in range(epochs):
            train_loss.reset_states()
    
            for (batch, (inputs, targets)) in enumerate(dataset):
                # 定义掩蔽
                encoder_padding_mask, combined_mask, decoder_padding_mask = create_masks(inputs, targets)
    
                with tf.GradientTape() as tape:
                    predictions = model(inputs, targets, True, encoder_padding_mask, combined_mask, decoder_padding_mask)
                    loss = loss_function(targets[:, 1:], predictions)
    
                gradients = tape.gradient(loss, model.trainable_variables)
                optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    
                train_loss(loss)
    
                if batch % 50 == 0:
                    print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, train_loss.result()))
    
        print('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result()))
    
    # 加载数据集并预处理
    input_tensor, target_tensor, input_tokenizer, target_tokenizer = load_dataset(PATH_TO_DATASET)
    input_tensor_train, target_tensor_train, input_tensor_val, target_tensor_val = train_val_split(input_tensor, target_tensor)
    
    buffer_size = len(input_tensor_train)
    batch_size = 64
    num_batches = buffer_size // batch_size
    
    vocab_inp_size = len(input_tokenizer.word_index) + 1
    vocab_tar_size = len(target_tokenizer.word_index) + 1
    
    # 根据数据集创建掩蔽
    def create_masks(inputs, targets):
        encoder_padding_mask = create_padding_mask(inputs)
        decoder_padding_mask = create_padding_mask(inputs)
    
        look_ahead_mask = create_look_ahead_mask(tf.shape(targets)[1])
        decoder_target_padding_mask = create_padding_mask(targets)
        combined_mask = tf.maximum(decoder_target_padding_mask, look_ahead_mask)
    
        return encoder_padding_mask, combined_mask, decoder_padding_mask
    
    # 创建模型并训练
    transformer = Transformer(num_layers=4, d_model=128, num_heads=8, dff=512,
                              input_vocab_size=vocab_inp_size, target_vocab_size=vocab_tar_size,
                              dropout_rate=0.1)
    train_dataset = create_dataset(input_tensor_train, target_tensor_train, batch_size)
    train(transformer, train_dataset, EPOCHS)
    
  3. User

    请继续

  4. Assistant

    以下是完整的代码,包括数据集预处理等部分:

    import tensorflow as tf
    from tensorflow import keras
    from tensorflow.keras import layers
    import numpy as np
    import re
    import os
    import time
    
    # 下载并解压数据集
    path_to_zip = tf.keras.utils.get_file('spa-eng.zip', origin='http://download.tensorflow.org/data/spa-eng.zip', extract=True)
    path_to_dataset = os.path.join(os.path.dirname(path_to_zip), "spa-eng", "spa.txt")
    
    # 定义处理特殊字符的函数
    def preprocess_sentence(sentence):
        sentence = sentence.lower().strip()
    
        sentence = re.sub(r"([?.!,¿])", r" \1 ", sentence)
        sentence = re.sub(r'[" "]+', " ", sentence)
        sentence = re.sub(r"[^a-zA-Z?.!,¿]+", " ", sentence)
    
        sentence = sentence.strip()
    
        sentence = '<start> ' + sentence + ' <end>'
        return sentence
    
    # 加载并处理数据集
    def load_dataset(path, num_examples=None):
        lines = open(path, encoding='UTF-8').read().strip().split('\n')
    
        word_pairs = [[preprocess_sentence(w) for w in l.split('\t')] for l in lines[:num_examples]]
    
        input_tensor = [wp[0] for wp in word_pairs]
        target_tensor = [wp[1] for wp in word_pairs]
    
        input_tokenizer = keras.preprocessing.text.Tokenizer(filters='')
        input_tokenizer.fit_on_texts(input_tensor)
        input_tensor = input_tokenizer.texts_to_sequences(input_tensor)
        input_tensor = keras.preprocessing.sequence.pad_sequences(input_tensor, padding='post')
    
        target_tokenizer = keras.preprocessing.text.Tokenizer(filters='')
        target_tokenizer.fit_on_texts(target_tensor)
        target_tensor = target_tokenizer.texts_to_sequences(target_tensor)
        target_tensor = keras.preprocessing.sequence.pad_sequences(target_tensor, padding='post')
    
        return input_tensor, target_tensor, input_tokenizer, target_tokenizer
    
    # 划分训练集和验证集
    def train_val_split(input_tensor, target_tensor, train_ratio=0.8):
        num_examples = len(input_tensor)
        indices = np.arange(num_examples)
        np.random.shuffle(indices)
    
        train_size = int(train_ratio * num_examples)
    
        train_indices = indices[:train_size]
        val_indices = indices[train_size:]
    
        input_tensor_train, target_tensor_train = input_tensor[train_indices], target_tensor[train_indices]
        input_tensor_val, target_tensor_val = input_tensor[val_indices], target_tensor[val_indices]
    
        return input_tensor_train, target_tensor_train, input_tensor_val, target_tensor_val
    
    # 创建掩蔽
    def create_padding_mask(seq):
        seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
    
        return seq[:, tf.newaxis, tf.newaxis, :]
    
    def create_look_ahead_mask(size):
        mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
        return mask
    
    # 创建数据集
    def create_dataset(input_tensor, target_tensor, batch_size):
        dataset = tf.data.Dataset.from_tensor_slices((input_tensor, target_tensor)).shuffle(buffer_size)
        dataset = dataset.batch(batch_size, drop_remainder=True)
        return dataset
    
    # 定义Transformer模型
    class Transformer(keras.Model):
        def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, dropout_rate=0.1):
            super(Transformer, self).__init__()
    
            self.encoder = layers.Embedding(input_vocab_size, d_model)
            self.encoder_dropout = layers.Dropout(dropout_rate)
            self.encoder_layers = [EncoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]
            self.encoder_norm = layers.LayerNormalization(epsilon=1e-6)
    
            self.decoder = layers.Embedding(target_vocab_size, d_model)
            self.decoder_dropout = layers.Dropout(dropout_rate)
            self.decoder_layers = [DecoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]
            self.decoder_norm = layers.LayerNormalization(epsilon=1e-6)
    
            self.final_layer = layers.Dense(target_vocab_size)
    
        def call(self, inputs, targets, training, encoder_padding_mask, look_ahead_mask, decoder_padding_mask):
            # 编码器部分
            enc_output = self.encoder(inputs)
            enc_output = self.encoder_dropout(enc_output, training=training)
    
            for encoder_layer in self.encoder_layers:
                enc_output = encoder_layer(enc_output, encoder_padding_mask)
    
            enc_output = self.encoder_norm(enc_output)
    
            # 解码器部分
            dec_output = self.decoder(targets)
            dec_output = self.decoder_dropout(dec_output, training=training)
    
            for decoder_layer in self.decoder_layers:
                dec_output = decoder_layer(dec_output, enc_output, look_ahead_mask, decoder_padding_mask)
    
            dec_output = self.decoder_norm(dec_output)
    
            # 输出部分
            final_output = self.final_layer(dec_output)
    
            return final_output
    
    # 定义编码器层
    class EncoderLayer(layers.Layer):
        def __init__(self, d_model, num_heads, dff, dropout_rate):
            super(EncoderLayer, self).__init__()
    
            self.multi_head_attention = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout1 = layers.Dropout(dropout_rate)
            self.norm1 = layers.LayerNormalization(epsilon=1e-6)
    
            self.position_wise_feed_forward = keras.Sequential([
                layers.Dense(dff, activation='relu'),
                layers.Dense(d_model)
            ])
            self.dropout2 = layers.Dropout(dropout_rate)
            self.norm2 = layers.LayerNormalization(epsilon=1e-6)
    
        def call(self, inputs, mask):
            attention_output = self.multi_head_attention(inputs, inputs, inputs, mask)
            attention_output = self.dropout1(attention_output)
            attention_output = self.norm1(inputs + attention_output)
    
            feed_forward_output = self.position_wise_feed_forward(attention_output)
            feed_forward_output = self.dropout2(feed_forward_output)
            feed_forward_output = self.norm2(attention_output + feed_forward_output)
    
            return feed_forward_output
    
    # 定义解码器层
    class DecoderLayer(layers.Layer):
        def __init__(self, d_model, num_heads, dff, dropout_rate):
            super(DecoderLayer, self).__init__()
    
            self.multi_head_attention1 = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout1 = layers.Dropout(dropout_rate)
            self.norm1 = layers.LayerNormalization(epsilon=1e-6)
    
            self.multi_head_attention2 = layers.MultiHeadAttention(num_heads, d_model)
            self.dropout2 = layers.Dropout(dropout_rate)
            self.norm2 = layers.LayerNormalization(epsilon=1e-6)
    
            self.position_wise_feed_forward = keras.Sequential([
                layers.Dense(dff, activation='relu'),
                layers.Dense(d_model)
            ])
            self.dropout3 = layers.Dropout(dropout_rate)
            self.norm3 = layers.LayerNormalization(epsilon=1e-6)
    
        def call(self, inputs, enc_output, look_ahead_mask, padding_mask):
            attention_output1 = self.multi_head_attention1(inputs, inputs, inputs, look_ahead_mask)
            attention_output1 = self.dropout1(attention_output1)
            attention_output1 = self.norm1(inputs + attention_output1)
    
            attention_output2 = self.multi_head_attention2(attention_output1, enc_output, enc_output, padding_mask)
            attention_output2 = self.dropout2(attention_output2)
            attention_output2 = self.norm2(attention_output1 + attention_output2)
    
            feed_forward_output = self.position_wise_feed_forward(attention_output2)
            feed_forward_output = self.dropout3(feed_forward_output)
            feed_forward_output = self.norm3(attention_output2 + feed_forward_output)
    
            return feed_forward_output
    
    # 定义损失函数
    def loss_function(y_true, y_pred):
        mask = tf.math.logical_not(tf.math.equal(y_true, 0))
        loss_ = loss_object(y_true, y_pred)
    
        mask = tf.cast(mask, dtype=loss_.dtype)
        loss_ *= mask
    
        return tf.reduce_mean(loss_)
    
    # 定义学习率衰减函数
    class CustomSchedule(keras.optimizers.schedules.LearningRateSchedule):
        def __init__(self, d_model, warmup_steps=4000):
            super(CustomSchedule, self).__init__()
    
            self.d_model = tf.cast(d_model, tf.float32)
            self.warmup_steps = warmup_steps
    
        def __call__(self, step):
            arg1 = tf.math.rsqrt(step)
            arg2 = step * (self.warmup_steps ** -1.5)
    
            return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
    
    # 定义训练过程
    def train(model, dataset, epochs):
        optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.98, epsilon=1e-9)
        train_loss = keras.metrics.Mean(name='train_loss')
    
        for epoch in range(epochs):
            train_loss.reset_states()
    
            for (batch, (inputs, targets)) in enumerate(dataset):
                # 定义掩蔽
                encoder_padding_mask, combined_mask, decoder_padding_mask = create_masks(inputs, targets)
    
                with tf.GradientTape() as tape:
                    predictions = model(inputs, targets, True, encoder_padding_mask, combined_mask, decoder_padding_mask)
                    loss = loss_function(targets[:, 1:], predictions)
    
                gradients = tape.gradient(loss, model.trainable_variables)
                optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    
                train_loss(loss)
    
                if batch % 50 == 0:
                    print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, train_loss.result()))
    
        print('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result()))
    
    # 定义翻译函数
    def translate(input_sentence, model, input_tokenizer, target_tokenizer, max_length_input, max_length_target):
        input_sequence = input_tokenizer.texts_to_sequences([input_sentence])
        input_sequence = keras.preprocessing.sequence.pad_sequences(input_sequence, maxlen=max_length_input, padding='post')
        input_sequence = tf.convert_to_tensor(input_sequence)
    
        start_token = target_tokenizer.word_index['<start>']
        end_token = target_tokenizer.word_index['<end>']
    
        output