import torch import torch.nn as nn import torch.nn.functional as F from…
- User
import torch import torch.nn as nn import torch.nn.functional as F from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter import json from tqdm import tqdm import math
def positional_encoding(seq_len, d_model, device): pos = torch.arange(seq_len, dtype=torch.float, device=device).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)).to(device) pe = torch.zeros(seq_len, d_model, device=device) pe[:, 0::2] = torch.sin(pos * div_term) pe[:, 1::2] = torch.cos(pos * div_term) return pe.unsqueeze(0)
Expert Transformer Model
class TransformerExpert(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1): super(TransformerExpert, self).init() self.d_model = d_model self.input_fc = nn.Linear(input_size, d_model) encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers) self.output_fc = nn.Linear(d_model, output_size)
def forward(self, x): x = self.input_fc(x) + positional_encoding(x.size(1), self.d_model, x.device) transformer_output = self.transformer_encoder(x) output = self.output_fc(transformer_output[:, -1, :]) return outputGating Network
class GatingNetwork(nn.Module): def init(self, input_size, num_experts): super(GatingNetwork, self).init() self.fc = nn.Linear(input_size, num_experts) self.softmax = nn.Softmax(dim=1)
def forward(self, x): if x.dim() > 2: x = x.view(x.size(0), -1)Flatten the tensor correctly
x = self.fc(x) return self.softmax(x)Mixture of Experts Model
class MixtureOfTransformerExperts(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_experts, num_encoder_layers=1): super(MixtureOfTransformerExperts, self).init() self.num_experts = num_experts self.experts = nn.ModuleList([TransformerExpert(input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers) for _ in range(num_experts)]) self.gating_network = GatingNetwork(d_model, num_experts)
def forward(self, x): gating_scores = self.gating_network(x) expert_outputs = [expert(x).unsqueeze(1) for expert in self.experts] expert_outputs = torch.cat(expert_outputs, dim=1) # shape: [batch, num_experts, output_size] mixed_output = torch.bmm(expert_outputs.permute(0, 2, 1), gating_scores.unsqueeze(-1)).squeeze(-1) return mixed_outputclass QAJsonlDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len self.pairs = self.load_data(path) # Flatten the pairs completely before passing them to build_vocab self.vocab, self.idx2token = self.build_vocab([word for pair in self.pairs for sublist in pair for word in sublist]) self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]
def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("question", ""), data.get("answer", "") pairs.append((question.split(), answer.split())) return pairs def tokenize(self, words): # Tokenize a sentence and pad if necessary # Add <eos> token at the end if there’s room tokens = [self.vocab.get(w, self.vocab["<unk>"]) for w in words] if len(tokens) < self.seq_len: tokens.append(self.vocab["<eos>"]) # Add <eos> token tokens.extend([self.vocab["<pad>"]] * (self.seq_len - len(tokens))) # Pad the rest else: tokens = tokens[:self.seq_len - 1] + [self.vocab["<eos>"]] return tokens def build_vocab(self, words): # Start with special tokens with fixed indices vocab = {"<unk>": 0, "<pad>": 1, "<eos>": 2} start_index = len(vocab) # Use Counter to count word frequencies in the corpus counts = Counter(words) # Create the vocab dictionary with all words, starting indices after the special tokens for word, _ in counts.most_common(): if word not in vocab: # Skip special tokens vocab[word] = len(vocab) # Create the reverse mapping from indices to words idx2token = {idx: token for token, idx in vocab.items()} return vocab, idx2token def __len__(self): return len(self.tokenized_pairs) def __getitem__(self, idx): tokenized_question, tokenized_answer = self.tokenized_pairs[idx] return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)class MoETransformerModel(nn.Module): def init(self, vocab_size, d_model, moe): super(MoETransformerModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=d_model) self.moe = moe self.dropout = nn.Dropout(p=0.1) # Dropout added for regularization
def forward(self, x): embedded = self.dropout(self.embedding(x)) embedded_with_position = embedded + positional_encoding(x.size(1), d_model, x.device) return self.moe(embedded_with_position)def collate_fn(batch): questions, answers = zip(*batch) questions = pad_sequence(questions, batch_first=True, padding_value=0) answers = pad_sequence(answers, batch_first=True, padding_value=0) return questions, answers
Set the path to your text file and define sequence length
path_to_text = 'GSM2K.jsonl' # replace with the path to your text file seq_len = 64 # sequence length
Create a dataset and data loader
dataset = QAJsonlDataset(path_to_text, seq_len) data_loader = DataLoader(dataset, batch_size=128, shuffle=True, collate_fn=collate_fn, num_workers=4)
Training loop - added gradient clipping to avoid exploding gradients
def train_model(model, criterion, optimizer, num_epochs, data_loader): model.train() for epoch in range(num_epochs): total_loss = 0 progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False) for i, (inputs, targets) in progress_bar: optimizer.zero_grad() predictions = model(inputs) loss = criterion(predictions.view(-1, predictions.size(-1)), targets.view(-1)) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) # Gradient clipping optimizer.step() total_loss += loss.item() progress_bar.set_postfix({"Loss": loss.item()}) average_loss = total_loss / len(data_loader.dataset) print(f"Epoch {epoch+1}, Average Loss: {average_loss}")
def generate_text(model, dataset, seed_text, num_generate, temperature=1.0): model.eval() # Put the model in evaluation mode
# List to store the generated tokens generated_tokens = [] # Initial sequence (prefix) to start the generation process input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()] # Convert to token IDs current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0) # Generate num_generate tokens for _ in range(num_generate): # Forward pass through the model with torch.no_grad(): output = model(current_sequence) # Get probabilities, apply temperature scaling, and sample from the distribution probabilities = F.softmax(output[-1] / temperature, dim=0).detach() next_token_idx = torch.multinomial(probabilities, 1).item() # Append token to the current sequence and to the generated tokens generated_tokens.append(next_token_idx) current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1) # Convert tokens to words generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens]) # Use .get() to provide a default value for missing keys return generated_textDefine hyperparameters specific to the transformer
d_model = 128 nhead = 8 dim_feedforward = 256 num_encoder_layers = 2 num_experts = 2 vocab_size = len(dataset.vocab) # Assume dataset.vocab is defined in the QAJsonlDataset class
Instantiate resulting MoE transformer model
moe = MixtureOfTransformerExperts( input_size=d_model, # Assuming each word is represented by d_model features d_model=d_model, output_size=vocab_size, # Output size is the vocab size for token generation nhead=nhead, dim_feedforward=dim_feedforward, num_experts=num_experts, num_encoder_layers=num_encoder_layers )
Instantiate the MoE transformer model
moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe)
def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)
Example usage with your model:
total_params = count_parameters(moe_transformer_model) print(f"Total trainable parameters: {total_params}")
Training parameters
num_epochs = 4 learning_rate = 1e-4
Define Loss Function and Optimizer for MoE model - using Label Smoothing for better generalization
criterion = nn.CrossEntropyLoss(label_smoothing=0.1) optimizer = torch.optim.AdamW(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=0.01) # Using AdamW with weight decay scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, "min")
Replace references to single_lstm_txt_model with moe_model
Train the model with the text data
train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader)
Start a loop for the interactive chat-like text generation
while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to stop): ")
# Check if user wants to quit the interaction if seed_text.lower() == "quit": print("Exiting text generation chat.") break # User input is not empty and not “quit”, generate text if seed_text.strip(): num_generate = 16 # Number of words to generate temperature = 1.0 # Sampling temperature, higher will increase diversity # Use the trained model to generate text generated_text = generate_text(moe_transformer_model, dataset, seed_text, num_generate, temperature) print("Generated Text:", generated_text) else: print("Seed text cannot be empty.") except KeyboardInterrupt: # Handle KeyboardInterrupt (Ctrl+C) to gracefully exit print("\nExiting text generation chat.") breaktorch.save(moe_transformer_model.state_dict(), "MoE_Transformer-X2-9M-QA.pth")
- Assistant
Your script defines a comprehensive and flexible architecture for a Mixture of Transformer Experts (MoE) used for text generation and includes a training loop, text generation function, and interactive chat-like text generation loop. The script also properly demonstrates how to define, train, and interact with the model. Below is a brief explanation of each component in your script.
-
positional_encoding: A function to add positional encodings to a sequence of embeddings. This allows a transformer network to take into account the order of items. -
TransformerExpert: Defines a single expert transformer with a specified number of encoder layers, model dimensions, heads, feed-forward size, and an input/output layer for mapping input features to the desired output size. -
GatingNetwork: A simple network that takes an input tensor, flattens it if necessary, and outputs a softmax distribution over the experts. -
MixtureOfTransformerExperts: This is the MoE model that contains multiple TransformerExpert models and a GatingNetwork. It calculates weighted outputs from each expert based on the gating scores and combines these outputs to produce a final prediction. -
QAJsonlDataset: A PyTorch Dataset class that reads from a JSONL file containing question-answer pairs, builds a vocabulary, and tokenizes the pairs. It also handles padding sequences. -
MoETransformerModel: A wrapper model that includes an embedding layer for token embeddings, the MoE model, and dropout for regularization. -
collate_fn: A custom function for collating data into batches. It pads sequences to the same length to create a batch. -
train_model: Trains the model for the given number of epochs, with added gradient clipping. -
generate_text: Generates text from a trained MoE Transformer Model, using progressive generation and sampling. -
count_parameters: Function to count the total number of trainable parameters in the model. -
At the end, it initializes all necessary components (dataset, model, optimizer, etc.), starts the training loop, and enters an interactive chat-like loop where the user can input text and the model will generate additional text following the seed.
The model is trained using cross-entropy loss with label smoothing, the AdamW optimizer, and a ReduceLROnPlateau scheduler. After training, the user can interact with the model in a chat-like text generation setup, and the model's state is saved for later use.
Here are some potential improvements or considerations for the script:
- Ensure that you have a suitable dataset (named 'GSM2K.jsonl') for the specific use case (the file is referenced in the script but not provided).
- Properly handle the early stopping conditions for training based on the validation loss or another criterion.
- Experiment with different hyperparameters, such as learning rates, number of experts, and the d_model, to find the best configuration for your specific task.
- Since the script saves the model after each epoch, you can add a checkpointing feature that only saves the model if there's an improvement in terms of validation loss or accuracy.
- The hardware requirements for training such models can be significant; ensure you have access to GPUs if the model size and dataset are substantial.
- The quality of the text generation might greatly depend on the temperature parameter during sampling; tuning this hyperparameter can help in generating more coherent or creative text.
By incorporating these additional features, the script would be more robust and flexible for various use cases and datasets.
-
- User
Total trainable parameters: 8897328 Traceback (most recent call last): File "", line 1, in File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 116, in spawn_main exitcode = _main(fd, parent_sentinel) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 125, in _main prepare(preparation_data) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 236, in prepare _fixup_main_from_path(data['init_main_from_path']) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path main_content = runpy.run_path(main_path, File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 289, in run_path return _run_module_code(code, init_globals, run_name, File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 96, in _run_module_code _run_code(code, mod_globals, init_globals, File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in _run_code exec(code, run_globals) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 230, in train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 149, in train_model progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\utils\data\dataloader.py", line 438, in iter return self._get_iterator() File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\utils\data\dataloader.py", line 386, in _get_iterator return _MultiProcessingDataLoaderIter(self) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\utils\data\dataloader.py", line 1039, in init w.start() File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 121, in start self._popen = self._Popen(self) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\context.py", line 224, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\context.py", line 336, in _Popen return Popen(process_obj) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\popen_spawn_win32.py", line 45, in init prep_data = spawn.get_preparation_data(process_obj._name) File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 154, in get_preparation_data _check_not_importing_main() File "C:\Users\Dell-PC\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main raise RuntimeError(''' RuntimeError: An attempt has been made to start a new process before the current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable. - Assistant
The error you are seeing is due to attempting to create a new process in a way that is not safe during the startup phase of the program. This can happen when using multiprocessing in Python, particularly on Windows, where the default start method is "spawn" rather than "fork".
To fix this issue, you should protect the entry point of the program by using the following idiom:
if __name__ == '__main__': # Put the code that launches processes hereThis way, the code that creates new processes will only be executed if the script is run as a main program, and not if it's imported as a module. Here is how you should adjust your existing script. You can move all the code that should run on the main process to under this check:
if __name__ == '__main__': # Initialize dataset, model, optimizer, scheduler, etc. # ... # Define the training loop code def train_model(model, criterion, optimizer, num_epochs, data_loader): # ... # Training the model train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader) # Interactive text generation loop while True: # ... # Save the model torch.save(moe_transformer_model.state_dict(), "MoE_Transformer-X2-9M-QA.pth")This should be placed at the end of your script. This way, you ensure that the multiprocessing code is run safely. You should encapsulate any top-level code that starts processes or that is not function or class definitions, ensuring that
train_modeland any other functionality that uses multiprocessing is called within this check.This pattern is particularly important on Windows because of the lack of
fork()system call which behaves differently from Unix-like systems. It ensures that the modules are not imported anew in each child process, which would otherwise cause the child processes to inadvertently execute code that's not meant to be run multiple times or in parallel. - User
Total trainable parameters: 8897328 Traceback (most recent call last): File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 231, in train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 153, in train_model predictions = model(inputs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 129, in forward return self.moe(embedded_with_position) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 61, in forward gating_scores = self.gating_network(x) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "c:\Users\Dell-PC\Desktop\Projets\Mamba\MoE-Transformer_Q&A.py", line 49, in forward x = self.fc(x) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "C:\Users\Dell-PC\Desktop\Projets\Mamba\Arithword\lib\site-packages\torch\nn\modules\linear.py", line 114, in forward return F.linear(input, self.weight, self.bias) RuntimeError: mat1 and mat2 shapes cannot be multiplied (128x8192 and 128x2)
- Assistant
The error you're experiencing indicates that there's a mismatch in the matrix dimensions that are being multiplied together in the linear transformation within your
GatingNetwork. According to the error message, you're trying to multiply a matrix with shape(128x8192)with another matrix of shape(128x2), which is not compatible for matrix multiplication.This issue likely arises within the
GatingNetworkforward function:def forward(self, x): if x.dim() > 2: x = x.view(x.size(0), -1) x = self.fc(x) return self.softmax(x)The
view(x.size(0), -1)is flattening the inputxinto a two-dimensional array with shape(batch_size, -1), where-1is automatically inferred to ensure all the input elements are covered. This process should produce an array suitable for passing through the fc (fully connected) layer.However, the shape
(128x8192)indicates that the batch size is 128 and the flattened features per sample are 8192, which should match the first dimension of the weight matrix in the fc layer. In your case, the fc layer's weight matrix has a shape(128x2), which shows the input features are expected to be 128, but you get 8192 features instead, hence the mismatch.Let's investigate what might be going wrong and how to solve it:
-
Verify the input size to the
GatingNetwork: Ensure that when instantiating theGatingNetwork, the input size is correct and matches the flattened feature dimensions of your input tensor just before it's passed to the gating network. Based on the error message, it seems that the expected input size should be 8192 instead of 128. -
Check the shape of
xbefore the linex = self.fc(x)in theforwardmethod of theGatingNetwork. You might need to adjust the model so that the correct shape is passed to theGatingNetwork. -
The error suggests that your model's sequential length times the
d_model(which should define the number of features for each token) may be resulting in a tensor with a shape that's incompatible with what yourGatingNetworkexpects. You need to ensure the output size of the transformer and the input size of theGatingNetworkconsistently agree.
Here's an adjusted version of the
GatingNetwork:class GatingNetwork(nn.Module): def __init__(self, input_feature_dim, num_experts): super(GatingNetwork, self).__init__() self.fc = nn.Linear(input_feature_dim, num_experts) self.softmax = nn.Softmax(dim=1) def forward(self, x): # Assuming x is of shape [batch_size, seq_len, d_model] you may want to aggregate # across the sequence length dimension before the gating network x = x.mean(dim=1) # Take mean or another operation that reduces seq_len dimension x = self.fc(x) # Now the shape of x should match the weight matrix of the fc layer return self.softmax(x)This assumes that you want to collapse the sequence length dimension before the gating network. If you have another intention in mind, ensure that you preprocess
xaccordingly. Themean(dim=1)is just an example of dimension reduction, but the correct approach depends on the logic of your gating network. -