import torch import torch.nn as nn import torch.nn.functional as F from…
- User
import torch import torch.nn as nn import torch.nn.functional as F from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter from tqdm import tqdm
Expert LSTM Model
class LSTMExpert(nn.Module): def init(self, input_size, hidden_size, output_size): super(LSTMExpert, self).init() self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True) self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x): h, _ = self.lstm(x) # Shape of h: [batch_size, sequence_length, hidden_size] # No longer taking the last sequence output, transform for all time steps h = h.reshape(-1, h.shape[2]) # Flatten to two dimensions: [batch_size * sequence_length, hidden_size] return self.fc(h)Gating Network
class GatingNetwork(nn.Module): def init(self, input_size, num_experts): super(GatingNetwork, self).init() self.fc = nn.Linear(input_size, num_experts) self.softmax = nn.Softmax(dim=1)
def forward(self, x): if x.dim() > 2: x = x.view(x.size(0), -1) # Flatten the tensor correctly x = self.fc(x) return self.softmax(x)Mixture of Experts Model
class MixtureOfExperts(nn.Module): def init(self, input_size, hidden_size, output_size, num_experts): super(MixtureOfExperts, self).init() self.num_experts = num_experts self.experts = nn.ModuleList([LSTMExpert(input_size, hidden_size, output_size) for _ in range(num_experts)]) self.gating_network = GatingNetwork(input_size, num_experts)
def forward(self, x): # Flatten x for gating input: [batch_size * sequence_length, input_size] gating_input = x.contiguous().view(-1, x.shape[-1]) # Get gating scores for each expert: [batch_size * sequence_length, num_experts, 1] gating_scores = self.gating_network(gating_input).view(-1, self.num_experts, 1) # Compute outputs from each expert expert_outputs = [expert(x) for expert in self.experts] # List of [batch_size * sequence_length, vocab_size] tensors # Stack along a new dimension for experts: [batch_size * sequence_length, vocab_size, num_experts] expert_outputs = torch.stack(expert_outputs, dim=2) # Calculate weighted sum of expert outputs: [batch_size * sequence_length, vocab_size] mixed_output = torch.bmm(expert_outputs, gating_scores) # [batch_size * sequence_length, vocab_size, 1] mixed_output = mixed_output.squeeze(-1) # Remove last singleton dimension return mixed_outputclass TextDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len with open(path, "r", encoding="utf-8") as f: text = f.read() words = text.split() self.vocab, self.idx2token = self.build_vocab(words) self.tokens = [self.vocab.get(w, 0) for w in words] # Default to 0 for unknown tokens
def build_vocab(self, words): counts = Counter(words) vocab = {word: i for i, (word, _) in enumerate(counts.most_common(), 1)} vocab["<pad>"] = 0 idx2token = {i: word for word, i in vocab.items()} return vocab, idx2token def __len__(self): return len(self.tokens) // self.seq_len def __getitem__(self, idx): chunk = self.tokens[idx * self.seq_len: (idx + 1) * self.seq_len] x = chunk[:-1] y = chunk[1:] return torch.tensor(x), torch.tensor(y)def collate_fn(batch): inputs, targets = zip(*batch) inputs = pad_sequence(inputs, batch_first=True, padding_value=0) targets = pad_sequence(targets, batch_first=True, padding_value=0) return inputs, targets
Set the path to your text file and define sequence length
path_to_text = 'C:/Users/Dell-PC/Desktop/Projets/2-The-Beyond\Training-data/Summaries.txt' # replace with the path to your text file seq_len = 128 # sequence length
Create a dataset and data loader
dataset = TextDataset(path_to_text, seq_len) data_loader = DataLoader(dataset, batch_size=96, shuffle=True, collate_fn=collate_fn)
def train_model_TXT(model, criterion, optimizer, num_epochs, data_loader): model.train() for epoch in range(num_epochs): total_loss = 0 progress_bar = tqdm(data_loader, desc=f"Epoch {epoch+1}/{num_epochs}")
for inputs, targets in progress_bar: optimizer.zero_grad() # Forward pass predictions = model(inputs) # Flatten targets to be [batch_size * sequence_length] targets = targets.view(-1) # Flatten predictions to be [batch_size * sequence_length, num_classes] predictions = predictions.view(-1, predictions.size(-1)) # Loss computation loss = criterion(predictions, targets) loss.backward() optimizer.step() total_loss += loss.item() # Update the progress bar with the loss information progress_bar.set_postfix(loss=loss.item()) average_loss = total_loss / len(data_loader) print(f"Epoch {epoch+1}, Avg Loss: {average_loss:.4f}")def generate_text(model, dataset, seed_text, num_generate, temperature=1.0): model.eval() # Put the model in evaluation mode
# List to store the generated tokens generated_tokens = [] # Initial sequence (prefix) to start the generation process input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()] # Convert to token IDs current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0) # Generate num_generate tokens for _ in range(num_generate): # Forward pass through the model with torch.no_grad(): output = model(current_sequence) # Get probabilities, apply temperature scaling, and sample from the distribution probabilities = F.softmax(output[-1] / temperature, dim=0).detach() next_token_idx = torch.multinomial(probabilities, 1).item() # Append token to the current sequence and to the generated tokens generated_tokens.append(next_token_idx) current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1) # Convert tokens to words generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens]) # Use .get() to provide a default value for missing keys return generated_textIntegration of MoE with the rest of the code
vocab_size = len(dataset.vocab) # The size of the vocabulary embedding_dim = 144 # Size of embeddings hidden_size = 576 # Number of features in the hidden state of the LSTM num_experts = 2 # Define the number of LSTM experts you want in your MoE model
Instantiate the MixtureOfExperts model
mixture_of_experts_model = MixtureOfExperts(embedding_dim, hidden_size, vocab_size, num_experts)
Wrap the experts and the embedding layer into a new combined model, replacing SingleLSTM_TXT
class MoEModel(nn.Module): def init(self, embedding_dim, vocab_size, MoE): super(MoEModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim) self.MoE = MoE
def forward(self, x): embedded = self.embedding(x) output = self.MoE(embedded) return outputInstantiate the MoEModel with embeddings and mixture of experts
moe_model = MoEModel(embedding_dim, vocab_size, mixture_of_experts_model)
def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)
Example usage with your model:
total_params = count_parameters(moe_model) print(f"Total trainable parameters: {total_params}")
Training parameters
num_epochs = 10 batch_size = 96 learning_rate = 1e-2
Define Loss Function and Optimizer for MoE model
criterion = nn.CrossEntropyLoss() # Use CrossEntropyLoss for classification tasks optimizer = torch.optim.Adam(moe_model.parameters(), lr=learning_rate)
Replace references to single_lstm_txt_model with moe_model
Train the model with the text data
train_model_TXT(moe_model, criterion, optimizer, num_epochs, data_loader)
Start a loop for the interactive chat-like text generation
while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to stop): ")
# Check if user wants to quit the interaction if seed_text.lower() == "quit": print("Exiting text generation chat.") break # User input is not empty and not “quit”, generate text if seed_text.strip(): num_generate = 16 # Number of words to generate temperature = 0.1 # Sampling temperature, higher will increase diversity # Use the trained model to generate text generated_text = generate_text(moe_model, dataset, seed_text, num_generate, temperature) print("Generated Text:", generated_text) else: print("Seed text cannot be empty.") except KeyboardInterrupt: # Handle KeyboardInterrupt (Ctrl+C) to gracefully exit print("\nExiting text generation chat.") break - Assistant
Your script consists of several parts that define a Mixture of Experts with LSTM experts for text generation. The core components of your script are as follows:
-
LSTMExpert: A class that defines an LSTM model to be used as an expert in the mixture of experts model. It consists of an LSTM layer followed by a fully connected layer.
-
GatingNetwork: A neural network that takes the input sequence and computes the softmax weights for each of the LSTM experts.
-
MixtureOfExperts: An ensemble model that incorporates multiple LSTMExpert models and a GatingNetwork. It computes the final output by combining the outputs of each expert weighted by the gating network's output.
-
TextDataset: A PyTorch
Datasetclass for tokenizing the text data and serving it in chunks suitable for LSTM training. -
MoEModel: A wrapper model that includes an embedding layer and the MixtureOfExperts model. This will be trainable end-to-end.
The script also includes training and text generation functions:
-
train_model_TXT: Function for training the model using the provided dataset and data loader. It utilizes a loss function and an optimizer.
-
generate_text: Function for generating text with the trained model, taking into account a starting seed text, sequence length, and sampling temperature.
Finally, the script sets up the training process, instantiates a model object, and enters an interactive loop allowing a user to input seed text and receive generated text.
Here are a few things to watch out for in the script:
-
Ensure that the paths and parameters such as sequence length are set appropriately for your use case.
-
Verify that the dataset and tokenization process match your needs and that unknown tokens are handled correctly.
-
The
generate_textfunction makes use of the temperature parameter when sampling. This can affect the diversity of the generated text, and you may need to adjust this to suit your preferences. -
It is good practice to use
torch.no_grad()during inference (text generation) to disable gradient calculations and reduce memory consumption. -
The script uses a simple greedy decoding strategy. For more advanced or nuanced text generation, you could consider using other techniques such as beam search.
-
It's important to handle cases where the sequence length of the input text doesn't match your model's expectations, which may occur in
generate_text.
Overall, the script is well-organized and appears to cover the necessary components and steps for training a Mixture of Experts model on a text dataset and using it to generate new text sequences based on user-provided seed text. Remember to monitor the training process and evaluate generated text quality to adjust hyperparameters and potentially improve model performance.
-
- User
Help me change how datasets are loaded and what the model is trained on, instead of loading a txt dataset, we'll load a jsonl dataset, with two columns, Document, which are supposed to be the inputs of the model, and Summary, which is what the model should be able to predict, please modify all the necessary part of the code to affect the change
- Assistant
To adapt your code to load a JSON Lines (jsonl) dataset with "Document" and "Summary" columns, you will need to modify the
TextDatasetclass and any part of the code that references the dataset format. Here's how to proceed:- Modify the
TextDatasetclass to load the JSONL file and handle the "Document" and "Summary" pairs. - Update the collate function to handle variable-length input and output sequences, and use
pad_sequenceas needed. - Modify the training function so it works with the modified dataset and collate function.
Here are the changes applied to the
TextDataset,collate_fn, andtrain_model_TXTcode blocks:import json class SummaryDataset(Dataset): def __init__(self, path): with open(path, "r", encoding="utf-8") as f: self.data = [json.loads(line) for line in f] self.vocab, self.idx2token = self.build_vocab([d["Document"] + " " + d["Summary"] for d in self.data]) def build_vocab(self, docs): words = [word for doc in docs for word in doc.split()] counts = Counter(words) vocab = {word: i for i, (word, _) in enumerate(counts.most_common(), 1)} vocab["<pad>"] = 0 idx2token = {i: word for word, i in vocab.items()} return vocab, idx2token def __len__(self): return len(self.data) def __getitem__(self, idx): document = self.data[idx]["Document"] summary = self.data[idx]["Summary"] document_tokens = [self.vocab.get(w, 0) for w in document.split()] summary_tokens = [self.vocab.get(w, 0) for w in summary.split()] return torch.tensor(document_tokens), torch.tensor(summary_tokens) def collate_fn(batch): documents, summaries = zip(*batch) documents_padded = pad_sequence(documents, batch_first=True, padding_value=0) summaries_padded = pad_sequence(summaries, batch_first=True, padding_value=0) return documents_padded, summaries_padded # Modify the training function if needed to accommodate the new data structure def train_model_TXT(model, criterion, optimizer, num_epochs, data_loader): # ... (rest of the training code remains the same) # Load the dataset (path_to_jsonl should be the path to your jsonl file) path_to_jsonl = 'path_to_your_jsonl_file.jsonl' dataset = SummaryDataset(path_to_jsonl) data_loader = DataLoader(dataset, batch_size=96, shuffle=True, collate_fn=collate_fn) # Now you can train your model with the new dataset and dataloaderThese changes will load a JSONL file where each line contains a JSON object with the keys "Document" and "Summary." The
SummaryDatasetclass tokenizes both parts, builds a vocabulary, and serves them as input and target sequences for the model.Remember that you need to instantiate your model, loss function, and optimizer, as well as call the
train_model_TXTfunction with these parameters. The modification assumes that the model expects the input sequence and predicts the output sequence (the summary) directly, without the use of an encoder-decoder architecture like in some sequence-to-sequence models.If you need to transform the inputs and outputs to a different representation (e.g., sequences of embeddings or encoded states), you would need to adapt the model definition and forward pass correspondingly.
Also, don't forget to adjust paths, batch sizes, and any other parameters according to your specific requirements and the characteristics of the new dataset.
- Modify the