import numpy as np from keras.models import Model from keras.layers import Input, LSTM,…
- User
import numpy as np from keras.models import Model from keras.layers import Input, LSTM, Bidirectional, Concatenate, Dot,Embedding,Activation,Dense from keras.preprocessing.text import Tokenizer from tensorflow.keras.preprocessing.sequence import pad_sequences
Define hyperparameters
max_sequence_length = 100 embedding_dim = 100 latent_dim = 256
Load train, validate, and test data from text files
with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Train.txt', 'r', encoding='utf-8') as f: train_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Validate.txt', 'r', encoding='utf-8') as f: validate_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines()
Tokenize sentences and convert them into numerical representations
tokenizer_en = Tokenizer() tokenizer_en.fit_on_texts(train_data) train_data_en = tokenizer_en.texts_to_sequences(train_data) train_data_en = pad_sequences(train_data_en, maxlen=max_sequence_length)
tokenizer_ar = Tokenizer() tokenizer_ar.fit_on_texts(train_data) train_data_ar = tokenizer_ar.texts_to_sequences(train_data) train_data_ar = pad_sequences(train_data_ar, maxlen=max_sequence_length)
Define encoder inputs and LSTM layer
encoder_inputs = Input(shape=(max_sequence_length,)) encoder_embedding = Embedding(len(tokenizer_en.word_index) + 1, embedding_dim)(encoder_inputs) encoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(encoder_embedding)
Define decoder inputs and LSTM layer
decoder_inputs = Input(shape=(max_sequence_length,)) decoder_embedding = Embedding(len(tokenizer_ar.word_index) + 1, embedding_dim)(decoder_inputs) decoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(decoder_embedding)
Apply attention mechanism
attention = Dot(axes=[2, 2])([decoder_lstm, encoder_lstm]) attention = Activation('softmax')(attention) context = Dot(axes=[2, 1])([attention, encoder_lstm]) decoder_combined_context = Concatenate(axis=-1)([context, decoder_lstm])
Define decoder output layer
decoder_dense = Dense(len(tokenizer_ar.word_index) + 1, activation='softmax') decoder_outputs = decoder_dense(decoder_combined_context)
Define the model
model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary()
Train the model
model.fit([train_data_en, train_data_ar], np.expand_dims(train_data_ar, -1), batch_size=64, epochs=10, validation_split=0.2)
Use test data to evaluate the model
with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines()
test_data_en = tokenizer_en.texts_to_sequences(test_data) test_data_en = pad_sequences(test_data_en, maxlen=max_sequence_length)
predicted_arabic = model.predict(test_data_en) decoded_arabic = tokenizer_ar.sequences_to_texts(np.argmax(predicted_arabic, axis=-1))
Print example prediction translation from test data
print("English sentence: ", test_data[0]) print("Predicted Arabic sentence: ", decoded_arabic[0])
- Assistant
I have made some modifications to the code to resolve the error related to the tokenizer. Here is the updated code:
import numpy as np import tensorflow as tf from keras.models import Model from keras.layers import Input, LSTM, Bidirectional, Concatenate, Dot, Embedding, Activation, Dense from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences # Define hyperparameters max_sequence_length = 100 embedding_dim = 100 latent_dim = 256 # Load train, validate, and test data from text files with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Train.txt', 'r', encoding='utf-8') as f: train_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Validate.txt', 'r', encoding='utf-8') as f: validate_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines() # Tokenize sentences and convert them into numerical representations tokenizer_en = Tokenizer() tokenizer_en.fit_on_texts(train_data) train_data_en = tokenizer_en.texts_to_sequences(train_data) train_data_en = pad_sequences(train_data_en, maxlen=max_sequence_length) tokenizer_ar = Tokenizer() tokenizer_ar.fit_on_texts(train_data) train_data_ar = tokenizer_ar.texts_to_sequences(train_data) train_data_ar = pad_sequences(train_data_ar, maxlen=max_sequence_length) # Define encoder inputs and LSTM layer encoder_inputs = Input(shape=(max_sequence_length,)) encoder_embedding = Embedding(len(tokenizer_en.word_index) + 1, embedding_dim)(encoder_inputs) encoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(encoder_embedding) # Define decoder inputs and LSTM layer decoder_inputs = Input(shape=(max_sequence_length,)) decoder_embedding = Embedding(len(tokenizer_ar.word_index) + 1, embedding_dim)(decoder_inputs) decoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(decoder_embedding) # Apply attention mechanism attention = Dot(axes=[2, 2])([decoder_lstm, encoder_lstm]) attention = Activation('softmax')(attention) context = Dot(axes=[2, 1])([attention, encoder_lstm]) decoder_combined_context = Concatenate(axis=-1)([context, decoder_lstm]) # Define decoder output layer decoder_dense = Dense(len(tokenizer_ar.word_index) + 1, activation='softmax') decoder_outputs = decoder_dense(decoder_combined_context) # Define the model model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # Train the model model.fit([train_data_en, train_data_ar], np.expand_dims(train_data_ar, -1), batch_size=64, epochs=10, validation_split=0.2) # Use test data to evaluate the model test_data_en = tokenizer_en.texts_to_sequences(test_data) test_data_en = pad_sequences(test_data_en, maxlen=max_sequence_length) predicted_arabic = model.predict([test_data_en, np.zeros_like(test_data_en)]) decoded_arabic = tokenizer_ar.sequences_to_texts(np.argmax(predicted_arabic, axis=-1)) # Print example prediction translation from test data print("English sentence: ", test_data[0]) print("Predicted Arabic sentence: ", decoded_arabic[0])Please make sure to update the file paths for the train, validate, and test text files in the code. Also, ensure that you have the necessary packages installed.
- User
ValueError: Shapes (None, 100, 1) and (None, 100, 9715) are incompatible
- Assistant
The error you're encountering is caused by the mismatch between the shape of the predicted output tensor and the shape of the target output tensor.
To resolve this issue, you can modify the model's output and the target output tensors to have the same shape. Instead of using
np.expand_dims(train_data_ar, -1)as the target output, you can use one-hot encoding to convert the target sequences into a binary matrix.Here's an updated version of the code that implements one-hot encoding for the target output:
import numpy as np from keras.models import Model from keras.layers import Input, LSTM, Bidirectional, Concatenate, Dot, Embedding, Activation, Dense from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences from keras.utils import to_categorical # Define hyperparameters max_sequence_length = 100 embedding_dim = 100 latent_dim = 256 # Load train, validate, and test data from text files with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Train.txt', 'r', encoding='utf-8') as f: train_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Validate.txt', 'r', encoding='utf-8') as f: validate_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines() # Tokenize sentences and convert them into numerical representations tokenizer_en = Tokenizer() tokenizer_en.fit_on_texts(train_data) train_data_en = tokenizer_en.texts_to_sequences(train_data) train_data_en = pad_sequences(train_data_en, maxlen=max_sequence_length) tokenizer_ar = Tokenizer() tokenizer_ar.fit_on_texts(train_data) train_data_ar = tokenizer_ar.texts_to_sequences(train_data) train_data_ar = pad_sequences(train_data_ar, maxlen=max_sequence_length) # Define encoder inputs and LSTM layer encoder_inputs = Input(shape=(max_sequence_length,)) encoder_embedding = Embedding(len(tokenizer_en.word_index) + 1, embedding_dim)(encoder_inputs) encoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(encoder_embedding) # Define decoder inputs and LSTM layer decoder_inputs = Input(shape=(max_sequence_length,)) decoder_embedding = Embedding(len(tokenizer_ar.word_index) + 1, embedding_dim)(decoder_inputs) decoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(decoder_embedding) # Apply attention mechanism attention = Dot(axes=[2, 2])([decoder_lstm, encoder_lstm]) attention = Activation('softmax')(attention) context = Dot(axes=[2, 1])([attention, encoder_lstm]) decoder_combined_context = Concatenate(axis=-1)([context, decoder_lstm]) # Define decoder output layer decoder_dense = Dense(len(tokenizer_ar.word_index) + 1, activation='softmax') decoder_outputs = decoder_dense(decoder_combined_context) # Define the model model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # Convert target sequences to binary matrix train_data_ar = to_categorical(train_data_ar, num_classes=len(tokenizer_ar.word_index) + 1) # Train the model model.fit([train_data_en, train_data_ar], train_data_ar, batch_size=64, epochs=10, validation_split=0.2) # Use test data to evaluate the model test_data_en = tokenizer_en.texts_to_sequences(test_data) test_data_en = pad_sequences(test_data_en, maxlen=max_sequence_length) predicted_arabic = model.predict([test_data_en, np.zeros_like(test_data_en)]) decoded_arabic = tokenizer_ar.sequences_to_texts(np.argmax(predicted_arabic, axis=-1)) # Print example prediction translation from test data print("English sentence: ", test_data[0]) print("Predicted Arabic sentence: ", decoded_arabic[0])Hope this helps! Let me know if you have any other questions.
- User
ValueError: Shapes (None, 100) and (None, 100, 9715) are incompatible
- Assistant
Apologies for the confusion. It seems that there is still an issue with the shape compatibility between the predicted output and the target output. Here's an updated version of the code that resolves this issue:
import numpy as np from keras.models import Model from keras.layers import Input, LSTM, Bidirectional, Concatenate, Dot, Embedding, Activation, Dense from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences # Define hyperparameters max_sequence_length = 100 embedding_dim = 100 latent_dim = 256 # Load train, validate, and test data from text files with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Train.txt', 'r', encoding='utf-8') as f: train_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Validate.txt', 'r', encoding='utf-8') as f: validate_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines() # Tokenize sentences and convert them into numerical representations tokenizer_en = Tokenizer() tokenizer_en.fit_on_texts(train_data) train_data_en = tokenizer_en.texts_to_sequences(train_data) train_data_en = pad_sequences(train_data_en, maxlen=max_sequence_length) tokenizer_ar = Tokenizer() tokenizer_ar.fit_on_texts(train_data) train_data_ar = tokenizer_ar.texts_to_sequences(train_data) train_data_ar = pad_sequences(train_data_ar, maxlen=max_sequence_length) # Define encoder inputs and LSTM layer encoder_inputs = Input(shape=(max_sequence_length,)) encoder_embedding = Embedding(len(tokenizer_en.word_index) + 1, embedding_dim)(encoder_inputs) encoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(encoder_embedding) # Define decoder inputs and LSTM layer decoder_inputs = Input(shape=(max_sequence_length,)) decoder_embedding = Embedding(len(tokenizer_ar.word_index) + 1, embedding_dim)(decoder_inputs) decoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(decoder_embedding) # Apply attention mechanism attention = Dot(axes=[2, 2])([decoder_lstm, encoder_lstm]) attention = Activation('softmax')(attention) context = Dot(axes=[2, 1])([attention, encoder_lstm]) decoder_combined_context = Concatenate(axis=-1)([context, decoder_lstm]) # Define decoder output layer decoder_dense = Dense(len(tokenizer_ar.word_index) + 1, activation='softmax') decoder_outputs = decoder_dense(decoder_combined_context) # Define the model model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.summary() # Train the model model.fit([train_data_en, train_data_ar[:, :-1]], train_data_ar[:, 1:], batch_size=64, epochs=10, validation_split=0.2) # Use test data to evaluate the model test_data_en = tokenizer_en.texts_to_sequences(test_data) test_data_en = pad_sequences(test_data_en, maxlen=max_sequence_length) predicted_arabic = model.predict([test_data_en, np.zeros((test_data_en.shape[0], max_sequence_length))]) decoded_arabic = tokenizer_ar.sequences_to_texts(np.argmax(predicted_arabic, axis=-1)) # Print example prediction translation from test data print("English sentence: ", test_data[0]) print("Predicted Arabic sentence: ", decoded_arabic[0])In this updated version, I've made the following changes:
- Modified the loss function to
sparse_categorical_crossentropyto handle the integer target sequences. - Adjusted the target output to
train_data_ar[:, 1:], which removes the first token from each target sequence. - Removed the use of one-hot encoding for the target output and used integer sequences directly.
- Added
np.zeros((test_data_en.shape[0], max_sequence_length))as the initial state for the decoder during prediction.
Hopefully, this resolves the shape incompatibility issue. Let me know if you encounter any further issues.
- Modified the loss function to
- User
ValueError: Input 1 of layer "model" is incompatible with the layer: expected shape=(None, 100), found shape=(None, 99)
- Assistant
Apologies for the confusion. The error is caused by a mismatch in the shape of the target output and the model's expected output shape.
To resolve this issue, you need to make a small adjustment to the target output when fitting the model. Instead of using
train_data_ar[:, 1:]as the target output, you should usetrain_data_ar[:, 1:]andtrain_data_ar[:, 1:]with a shift of one position.Here's the updated code:
import numpy as np from keras.models import Model from keras.layers import Input, LSTM, Bidirectional, Concatenate, Dot, Embedding, Activation, Dense from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences # Define hyperparameters max_sequence_length = 100 embedding_dim = 100 latent_dim = 256 # Load train, validate, and test data from text files with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Train.txt', 'r', encoding='utf-8') as f: train_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Validate.txt', 'r', encoding='utf-8') as f: validate_data = f.read().splitlines() with open('D:/PHD Papers/scientific translation/GPT/First Code BiLSTM/NewResult/Test.txt', 'r', encoding='utf-8') as f: test_data = f.read().splitlines() # Tokenize sentences and convert them into numerical representations tokenizer_en = Tokenizer() tokenizer_en.fit_on_texts(train_data) train_data_en = tokenizer_en.texts_to_sequences(train_data) train_data_en = pad_sequences(train_data_en, maxlen=max_sequence_length) tokenizer_ar = Tokenizer() tokenizer_ar.fit_on_texts(train_data) train_data_ar = tokenizer_ar.texts_to_sequences(train_data) train_data_ar = pad_sequences(train_data_ar, maxlen=max_sequence_length) # Define encoder inputs and LSTM layer encoder_inputs = Input(shape=(max_sequence_length,)) encoder_embedding = Embedding(len(tokenizer_en.word_index) + 1, embedding_dim)(encoder_inputs) encoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(encoder_embedding) # Define decoder inputs and LSTM layer decoder_inputs = Input(shape=(max_sequence_length,)) decoder_embedding = Embedding(len(tokenizer_ar.word_index) + 1, embedding_dim)(decoder_inputs) decoder_lstm = Bidirectional(LSTM(latent_dim, return_sequences=True))(decoder_embedding) # Apply attention mechanism attention = Dot(axes=[2, 2])([decoder_lstm, encoder_lstm]) attention = Activation('softmax')(attention) context = Dot(axes=[2, 1])([attention, encoder_lstm]) decoder_combined_context = Concatenate(axis=-1)([context, decoder_lstm]) # Define decoder output layer decoder_dense = Dense(len(tokenizer_ar.word