from transformers import BertTokenizer, TFBertModel import numpy as np import pandas as…
- User
from transformers import BertTokenizer, TFBertModel import numpy as np import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split from sklearn.svm import SVC from sklearn.metrics import classification_report
Load Quora dataset
quora_data = pd.read_csv('questions.csv', nrows=6000)
Select a subset of the dataset for training
#train_data = quora_data.sample(n=80) # Adjust the number of samples based on your requirements
Get the minimum length between question1 and question2
min_length = min(len(quora_data['question1']), len(quora_data['question2']))
Adjust the number of samples based on the minimum length
train_data = quora_data.sample(n=min_length)
Convert the question pairs to list format
question1_list = list(train_data['question1']) question2_list = list(train_data['question2']) is_duplicate = list(train_data['is_duplicate'])
Create DataFrame for the first pair
data1 = pd.DataFrame({'Sentences': question1_list, 'Labels': is_duplicate})
Create DataFrame for the second pair
data2 = pd.DataFrame({'Sentences': question2_list, 'Labels': is_duplicate})
Concatenate the two DataFrames
data = pd.concat([data1, data2], ignore_index=True)
Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(data['Sentences'], data['Labels'], test_size=0.2, random_state=42)
Initialize the TF-IDF vectorizer
vectorizer = TfidfVectorizer()
Vectorize the sentences in train set
X_train_vectors = vectorizer.fit_transform(X_train)
Vectorize the sentences in test set
X_test_vectors = vectorizer.transform(X_test)
Initialize the SVM classifier
svm = SVC()
Fit the classifier to the training data
svm.fit(X_train_vectors, y_train)
Predict labels for the test data
y_pred = svm.predict(X_test_vectors)
Print classification report
print(classification_report(y_test, y_pred))
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, f1_score, recall_score
Calculate metrics
accuracy = accuracy_score(y_test, y_pred) cm = confusion_matrix(y_test, y_pred) precision = precision_score(y_test, y_pred) f1 = f1_score(y_test, y_pred) recall = recall_score(y_test, y_pred)
Print the results:
print("Accuracy:", accuracy) print("Confusion Matrix:") print(cm) print("Precision:", precision) print("F1 Score:", f1) print("Recall:", recall)
save the output the is_duplicated column in an excel file
- Assistant
To save the
is_duplicatecolumn in an Excel file, you can use thepandaslibrary. Here's how you can modify the code to achieve that:# Load Quora dataset quora_data = pd.read_csv('questions.csv', nrows=6000) # Select a subset of the dataset for training min_length = min(len(quora_data['question1']), len(quora_data['question2'])) train_data = quora_data.sample(n=min_length) # Convert the question pairs to list format question1_list = list(train_data['question1']) question2_list = list(train_data['question2']) is_duplicate = list(train_data['is_duplicate']) # Create DataFrame for the first pair data1 = pd.DataFrame({'Sentences': question1_list, 'Labels': is_duplicate}) # Create DataFrame for the second pair data2 = pd.DataFrame({'Sentences': question2_list, 'Labels': is_duplicate}) # Concatenate the two DataFrames data = pd.concat([data1, data2], ignore_index=True) # Split data into train and test sets X_train, X_test, y_train, y_test = train_test_split(data['Sentences'], data['Labels'], test_size=0.2, random_state=42) # Initialize the TF-IDF vectorizer vectorizer = TfidfVectorizer() # Vectorize the sentences in train set X_train_vectors = vectorizer.fit_transform(X_train) # Vectorize the sentences in test set X_test_vectors = vectorizer.transform(X_test) # Initialize the SVM classifier svm = SVC() # Fit the classifier to the training data svm.fit(X_train_vectors, y_train) # Predict labels for the test data y_pred = svm.predict(X_test_vectors) # Save the 'is_duplicate' column in an Excel file result_df = pd.DataFrame({'is_duplicate': y_pred}) result_df.to_excel('output.xlsx', index=False)This code will save the
is_duplicatecolumn in an Excel file namedoutput.xlsxin the same directory as your Python script. Make sure you have thepandaslibrary installed. - User
from transformers import BertTokenizer, TFBertModel import numpy as np import pandas as pd from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense
Load BERT tokenizer and model
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') bert_model = TFBertModel.from_pretrained('bert-base-uncased')
Load Quora dataset
quora_data = pd.read_csv('questions.csv',nrows=6000)
Select a subset of the dataset for training
train_data = quora_data.sample(n=5000) # Adjust the number of samples based on your requirements
Convert the question pairs to list format
question1_list = list(train_data['question1']) question2_list = list(train_data['question2'])
Tokenize and convert sentences to BERT inputs
max_len = 25 # Define the maximum length for padding/truncating encoded_inputs = tokenizer(question1_list, question2_list, padding=True, truncation=True, max_length=max_len, return_tensors='tf' ) #attention_mask = encoded_inputs['attention_mask']
Get BERT embeddings
#outputs = bert_model(input_ids, attention_mask=attention_mask) #embeddings = outputs[0] # Shape: (num_samples, max_len, 768) #outputs = bert_model(encoded_inputs.input_ids, attention_mask=encoded_inputs.attention_mask) #embeddings = outputs.last_hidden_state # Shape: (num_samples, max_len, 768)
inputs = { 'input_ids': encoded_inputs['input_ids'], 'attention_mask': encoded_inputs['attention_mask'] }
outputs = bert_model(inputs) embeddings = outputs.last_hidden_state # Shape: (num_samples, max_len, 768)
Define LSTM model
model = Sequential() model.add(LSTM(64, return_sequences=True, input_shape=(max_len, 768))) model.add(LSTM(64)) model.add(Dense(1, activation='sigmoid'))
Compile the model
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
Prepare labels for training
labels = np.array(train_data['is_duplicate']) # Assuming the dataset has a “is_duplicate” column indicating similarity/dissimilarity
Train the model
model.fit(embeddings, labels, epochs=10)
#question1_list = list(test_data['question1']) #question2_list = list(test_data['question2'])
Load test data and preprocess
test_data = quora_data.sample(n=1000) # Adjust the number of samples for testing encoded_test = tokenizer(list(test_data['question1']), list(test_data['question2']), padding=True, truncation=True, max_length=max_len, return_tensors='tf') test_input_ids = encoded_test['input_ids'] test_attention_mask = encoded_test['attention_mask']
Get BERT embeddings for test data
test_outputs = bert_model(test_input_ids, attention_mask=test_attention_mask) test_embeddings = test_outputs[0] # Shape: (num_samples, max_len, 768)
Make predictions on test data
predictions = model.predict(test_embeddings)
Print test predictions
for i, pred in enumerate(predictions): print(f"Question 1: {test_data.iloc[i]['question1']}") print(f"Question 2: {test_data.iloc[i]['question2']}") print(f"Similarity Prediction: {pred}") print()
save the output the is_duplicated column in an excel file
- Assistant
To save the
is_duplicatecolumn in an Excel file, you can use thepandaslibrary. Here's how you can modify the code to achieve that:# Load BERT tokenizer and model tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') bert_model = TFBertModel.from_pretrained('bert-base-uncased') # Load Quora dataset quora_data = pd.read_csv('questions.csv', nrows=6000) # Select a subset of the dataset for training train_data = quora_data.sample(n=5000) # Adjust the number of samples based on your requirements # Convert the question pairs to list format question1_list = list(train_data['question1']) question2_list = list(train_data['question2']) # Tokenize and convert sentences to BERT inputs max_len = 25 # Define the maximum length for padding/truncating encoded_inputs = tokenizer(question1_list, question2_list, padding=True, truncation=True, max_length=max_len, return_tensors='tf') inputs = { 'input_ids': encoded_inputs['input_ids'], 'attention_mask': encoded_inputs['attention_mask'] } outputs = bert_model(inputs) embeddings = outputs.last_hidden_state # Shape: (num_samples, max_len, 768) # Define LSTM model model = Sequential() model.add(LSTM(64, return_sequences=True, input_shape=(max_len, 768))) model.add(LSTM(64)) model.add(Dense(1, activation='sigmoid')) # Compile the model model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # Prepare labels for training labels = np.array(train_data['is_duplicate']) # Assuming the dataset has a “is_duplicate” column indicating similarity/dissimilarity # Train the model model.fit(embeddings, labels, epochs=10) # Load test data and preprocess test_data = quora_data.sample(n=1000) # Adjust the number of samples for testing encoded_test = tokenizer(list(test_data['question1']), list(test_data['question2']), padding=True, truncation=True, max_length=max_len, return_tensors='tf') test_input_ids = encoded_test['input_ids'] test_attention_mask = encoded_test['attention_mask'] test_outputs = bert_model(test_input_ids, attention_mask=test_attention_mask) test_embeddings = test_outputs.last_hidden_state # Shape: (num_samples, max_len, 768) # Make predictions on test data predictions = model.predict(test_embeddings) # Create DataFrame for test predictions test_predictions = pd.DataFrame({'Question1': test_data['question1'], 'Question2': test_data['question2'], 'is_duplicate': predictions}) # Save the test predictions in an Excel file test_predictions.to_excel('test_predictions.xlsx', index=False)This code will save the test predictions including the
is_duplicatecolumn in an Excel file namedtest_predictions.xlsxin the same directory as your Python script. Make sure you have thepandaslibrary installed. - User
Create DataFrame for test predictions
test_predictions = pd.DataFrame({‘Question1’: test_data[‘question1’], ‘Question2’: test_data[‘question2’], ‘is_duplicate’: predictions})
ValueError: Per-column arrays must each be 1-dimensional
- Assistant
Apologies for the error in the code. It seems there is an issue with the format of the dictionary keys. Please try modifying the code as follows:
# Create DataFrame for test predictions test_predictions = pd.DataFrame({'Question1': list(test_data['question1']), 'Question2': list(test_data['question2']), 'is_duplicate': predictions})The above code will convert the 'question1' and 'question2' columns from the
test_dataDataFrame into lists, ensuring they are one-dimensional arrays as required by thepd.DataFramefunction. This should resolve theValueErroryou encountered. - User
ValueError: Per-column arrays must each be 1-dimensional
- Assistant
Apologies once again for the oversight. The error is occurring because
predictionsis a 2-dimensional array. To resolve the issue, you can flatten thepredictionsarray before creating the DataFrame. Here's the updated code:# Create DataFrame for test predictions test_predictions = pd.DataFrame({ 'Question1': list(test_data['question1']), 'Question2': list(test_data['question2']), 'is_duplicate': predictions.flatten() })By using
predictions.flatten(), we ensure that thepredictionsarray is flattened into a 1-dimensional array before creating the DataFrame. This should resolve theValueErrorand allow you to save the test predictions in an Excel file. - User
compare the output of the three models saved in excel and make a voting model
- Assistant
To compare the outputs of the three models saved in Excel and create a voting model, you can follow these steps:
- Load the outputs of the three models from their respective Excel files into pandas DataFrames.
- Merge the DataFrames on a common identifier (e.g., index or question pair ID).
- Calculate the majority vote for each question pair by selecting the most frequent label from the three models' outputs.
- Create a final DataFrame with the question pairs and the majority vote.
- Save the final DataFrame to an Excel file for further analysis or evaluation.
Here's an example of how you can implement these steps:
import pandas as pd # Load the outputs of the three models from their Excel files model1_predictions = pd.read_excel('model1_predictions.xlsx') # Replace with the actual file names model2_predictions = pd.read_excel('model2_predictions.xlsx') model3_predictions = pd.read_excel('model3_predictions.xlsx') # Merge the DataFrames on a common identifier merged_predictions = pd.merge(model1_predictions, model2_predictions, on='QuestionID') merged_predictions = pd.merge(merged_predictions, model3_predictions, on='QuestionID') # Calculate the majority vote for each question pair merged_predictions['MajorityVote'] = merged_predictions[['is_duplicate_x', 'is_duplicate_y', 'is_duplicate']].mode(axis=1)[0] # Create the final DataFrame with the question pairs and the majority vote final_predictions = merged_predictions[['QuestionID', 'Question1', 'Question2', 'MajorityVote']] # Save the final predictions to an Excel file final_predictions.to_excel('voting_predictions.xlsx', index=False)Make sure to replace the file names ('model1_predictions.xlsx', 'model2_predictions.xlsx', 'model3_predictions.xlsx') with the actual names of the Excel files containing the predictions of the three models.
The
final_predictionsDataFrame will contain the question pairs along with the majority vote from the three models. You can save this DataFrame to an Excel file named 'voting_predictions.xlsx' for further analysis or evaluation. - User
from transformers import BertTokenizer, TFBertModel import numpy as np import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split from sklearn.svm import SVC from sklearn.metrics import classification_report
Load Quora dataset
quora_data = pd.read_csv('questions.csv', nrows=6000)
Select a subset of the dataset for training
#train_data = quora_data.sample(n=80) # Adjust the number of samples based on your requirements
Get the minimum length between question1 and question2
min_length = min(len(quora_data['question1']), len(quora_data['question2']))
Adjust the number of samples based on the minimum length
train_data = quora_data.sample(n=min_length)
Convert the question pairs to list format
question1_list = list(train_data['question1']) question2_list = list(train_data['question2']) is_duplicate = list(train_data['is_duplicate'])
Create DataFrame for the first pair
data1 = pd.DataFrame({'Sentences': question1_list, 'Labels': is_duplicate})
Create DataFrame for the second pair
data2 = pd.DataFrame({'Sentences': question2_list, 'Labels': is_duplicate})
Concatenate the two DataFrames
data = pd.concat([data1, data2], ignore_index=True)
Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(data['Sentences'], data['Labels'], test_size=0.2, random_state=42)
Initialize the TF-IDF vectorizer
vectorizer = TfidfVectorizer()
Vectorize the sentences in train set
X_train_vectors = vectorizer.fit_transform(X_train)
Vectorize the sentences in test set
X_test_vectors = vectorizer.transform(X_test)
Initialize the SVM classifier
svm = SVC()
Fit the classifier to the training data
svm.fit(X_train_vectors, y_train)
Predict labels for the test data
y_pred = svm.predict(X_test_vectors)
Print classification report
print(classification_report(y_test, y_pred))
#svm_predictions = y_pred
Save the ‘is_duplicate’ column in an Excel file
result_df = pd.DataFrame({'is_duplicate': y_pred}) result_df.to_excel('output.xlsx', index=False)
save the two questions and the classification in an excel file
- Assistant
To save the two questions and their corresponding classification in an Excel file, you can modify the code as follows:
# Load Quora dataset quora_data = pd.read_csv('questions.csv', nrows=6000) # Select a subset of the dataset for training min_length = min(len(quora_data['question1']), len(quora_data['question2'])) train_data = quora_data.sample(n=min_length) # Convert the question pairs to list format question1_list = list(train_data['question1']) question2_list = list(train_data['question2']) is_duplicate = list(train_data['is_duplicate']) # Create DataFrame for the first pair data1 = pd.DataFrame({'Question1': question1_list, 'Question2': question2_list, 'Label': is_duplicate}) # Create DataFrame for the second pair data2 = pd.DataFrame({'Question1': question2_list, 'Question2': question1_list, 'Label': is_duplicate}) # Concatenate the two DataFrames data = pd.concat([data1, data2], ignore_index=True) # Save the questions and classification in an Excel file data.to_excel('questions_classification.xlsx', index=False)This code will save the questions and their corresponding classification (is_duplicate) in an Excel file named 'questions_classification.xlsx'. The file will contain two columns: 'Question1' and 'Question2' representing the questions, and 'Label' representing the classification. Make sure you have the
pandaslibrary installed. - User
the output data to be saved
- Assistant
Apologies for the confusion. If you would like to save the questions and their corresponding predictions generated by the SVM classifier, you can modify the code as follows:
# Load Quora dataset quora_data = pd.read_csv('questions.csv', nrows=6000) #