can you add to the conditionals for optimizertype to add GERU support to the code:…
- User
can you add to the conditionals for optimizer_type to add GERU support to the code: import torch import torch.nn as nn import torch.nn.functional as F import json import math from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter from tqdm import tqdm import matplotlib.pyplot as plt from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score
---------- Device Configuration ----------
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
---------- Utility Functions ----------
def positional_encoding(seq_len, d_model, device): pos = torch.arange(seq_len, dtype=torch.float, device=device).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)).to(device) pe = torch.zeros(seq_len, d_model, device=device) pe[:, 0::2] = torch.sin(pos * div_term) pe[:, 1::2] = torch.cos(pos * div_term) return pe.unsqueeze(0)
-------- Performance ----------
def evaluate_model(model, data_loader, device): model.eval() all_preds, all_targets = [], []
with torch.no_grad(): for inputs, targets in data_loader: inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) predictions = torch.argmax(outputs, dim=-1).view(-1) # Flatten predicted indices all_preds.extend(predictions.cpu().numpy()) all_targets.extend(targets.view(-1).cpu().numpy()) # Ensure targets are also flattened # Calculate precision, recall, and F1 score after ensuring all_preds and all_targets are correctly aligned. precision = precision_score(all_targets, all_preds, average='macro', zero_division=0) recall = recall_score(all_targets, all_preds, average='macro', zero_division=0) f1 = f1_score(all_targets, all_preds, average='macro', zero_division=0) print(f"Precision: {precision:.4f}") print(f"Recall: {recall:.4f}") print(f"F1 Score: {f1:.4f}") return precision, recall, f1Function to plot loss over time
def plot_loss(loss_history): plt.figure(figsize=(10, 5)) plt.plot(loss_history, label='Training Loss') plt.xlabel('Batches') plt.ylabel('Loss') plt.title('Training Loss Over Time') plt.legend() plt.show()
---------- Model Definitions ----------
class TransformerExpert(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1): super(TransformerExpert, self).init() self.d_model = d_model self.input_fc = nn.Linear(input_size, d_model) self.pos_encoder = nn.Parameter(positional_encoding(1, d_model, device), requires_grad=True) encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True, norm_first=True) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers) self.output_fc = nn.Linear(d_model, output_size) self.norm = nn.LayerNorm(d_model)
def forward(self, x): x = self.norm(self.input_fc(x)) + self.pos_encoder transformer_output = self.transformer_encoder(x) output = self.output_fc(transformer_output) return outputclass GatingNetwork(nn.Module): def init(self, input_feature_dim, num_experts, hidden_dims=[512], dropout_rate=0.1): super(GatingNetwork, self).init() layers = [] last_dim = input_feature_dim
for hidden_dim in hidden_dims: layers.extend([ nn.Linear(last_dim, hidden_dim), nn.GELU(), nn.Dropout(dropout_rate), ]) last_dim = hidden_dim layers.append(nn.Linear(last_dim, num_experts)) self.fc_layers = nn.Sequential(*layers) self.softmax = nn.Softmax(dim=1) def forward(self, x): x = x.mean(dim=1) # To ensure gating is based on overall features across the sequence x = self.fc_layers(x) return self.softmax(x)class MixtureOfTransformerExperts(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_experts, num_encoder_layers=1): super(MixtureOfTransformerExperts, self).init() self.num_experts = num_experts self.output_size = output_size self.experts = nn.ModuleList([TransformerExpert(input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers) for _ in range(num_experts)]) self.gating_network = GatingNetwork(d_model, num_experts)
def forward(self, x): gating_scores = self.gating_network(x) expert_outputs = [expert(x) for expert in self.experts] stacked_expert_outputs = torch.stack(expert_outputs) expanded_gating_scores = gating_scores.unsqueeze(2).unsqueeze(3) expanded_gating_scores = expanded_gating_scores.expand(-1, -1, x.size(1), self.output_size) expanded_gating_scores = expanded_gating_scores.transpose(0, 1) mixed_output = torch.sum(stacked_expert_outputs * expanded_gating_scores, dim=0) return mixed_outputclass MoETransformerModel(nn.Module): def init(self, vocab_size, d_model, moe): super(MoETransformerModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=d_model) self.moe = moe self.dropout = nn.Dropout(p=0.125)
def forward(self, x): embedded = self.dropout(self.embedding(x)) return self.moe(embedded)---------- Dataset Definitions ----------
class QAJsonlDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len self.pairs = self.load_data(path) self.vocab, self.idx2token = self.build_vocab([word for pair in self.pairs for sublist in pair for word in sublist]) self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]
def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("user", ""), data.get("content", "") pairs.append((question.split(), answer.split())) return pairs def tokenize(self, words): tokens = [self.vocab.get(w, self.vocab["<unk>"]) for w in words] if len(tokens) < self.seq_len: tokens.append(self.vocab["<eos>"]) tokens.extend([self.vocab["<pad>"]] * (self.seq_len - len(tokens))) else: tokens = tokens[:self.seq_len - 1] + [self.vocab["<eos>"]] return tokens def build_vocab(self, words): vocab = {"<unk>": 0, "<pad>": 1, "<eos>": 2} start_index = len(vocab) counts = Counter(words) for word, _ in counts.most_common(): if word not in vocab: vocab[word] = len(vocab) idx2token = {idx: token for token, idx in vocab.items()} return vocab, idx2token def __len__(self): return len(self.tokenized_pairs) def __getitem__(self, idx): tokenized_question, tokenized_answer = self.tokenized_pairs[idx] return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)def collate_fn(batch): questions, answers = zip(*batch) questions = pad_sequence(questions, batch_first=True, padding_value=0) answers = pad_sequence(answers, batch_first=True, padding_value=0) return questions, answers
---------- Training and Inference Functions ----------
def train_model(model, criterion, optimizer, num_epochs, data_loader): model.train() loss_history = [] # Initialize a list to keep track of losses for epoch in range(num_epochs): total_loss = 0 total_items = 0 # Keep track of total items processed progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False) for i, (inputs, targets) in progress_bar: inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad()
# Predict predictions = model(inputs) predictions = predictions.view(-1, predictions.size(-1)) # Make sure predictions are the right shape targets = targets.view(-1) # Flatten targets to match prediction shape if necessary # Calculate loss loss = criterion(predictions, targets) loss.backward() # Gradient clipping for stabilization torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) optimizer.step() scheduler.step() # Update total loss and the number of items total_loss += loss.item() * inputs.size(0) # Multiply loss by batch size total_items += inputs.size(0) loss_history.append(loss.item()) progress_bar.set_postfix({"Loss": loss.item()}) average_loss = total_loss / total_items # Correctly compute average loss print(f"Epoch {epoch+1}, Average Loss: {average_loss:.6f}") return loss_historyclass WarmupLR(torch.optim.lr_scheduler._LRScheduler): def init(self, optimizer, warmup_steps, scheduler_step_lr): self.warmup_steps = warmup_steps self.scheduler_step_lr = scheduler_step_lr # The subsequent scheduler super(WarmupLR, self).init(optimizer)
def get_lr(self): if self._step_count <= self.warmup_steps: warmup_factor = float(self._step_count) / float(max(1, self.warmup_steps)) for base_lr in self.base_lrs: yield base_lr * warmup_factor else: self.scheduler_step_lr.step() # Update the subsequent scheduler for param_group in self.optimizer.param_groups: yield param_group['lr']def generate_text(model, dataset, seed_text, num_generate, temperature=1.0): model.eval()
generated_tokens = [] # Initial sequence (prefix) to start the generation process input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()] current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0) current_sequence = current_sequence.to(device) # Generate num_generate tokens for _ in range(num_generate): # Forward pass through the model with torch.no_grad(): output = model(current_sequence) # Get probabilities, apply temperature scaling, and sample from the distribution probabilities = F.softmax(output[:, -1, :] / temperature, dim=-1).detach() next_token_idx = torch.multinomial(probabilities, 1).item() # Append token to the current sequence and to the generated tokens generated_tokens.append(next_token_idx) current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1).to(device) # Convert tokens to words generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens]) return generated_textdef count_tokens_in_dataset(dataset): return sum([len(pair[0]) + len(pair[1]) for pair in dataset.pairs])
def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)
---------- Hyperparameters and Model Instantiation ----------
Transformer :
d_model = 256 nhead = 8 dim_feedforward = 512 num_encoder_layers = 4 num_experts = 1 hidden_dims = [] dropout_rate = 0.5
Gating Network :
gating_network = GatingNetwork( input_feature_dim=d_model, num_experts=num_experts, hidden_dims=hidden_dims, dropout_rate=dropout_rate, )
Training Parameters
batch_size = 32 # Adjustable batch size warmup_steps = 6000 # Warmup steps for learning rate optimizer_type = "Adam" # Could be “SGD”, “RMSprop”, etc. learning_rate = 1e-2 weight_decay = 1e-5 # For L2 regularization num_epochs = 200
Dataset :
path_to_dataset = "C:/Users/L14/Documents/Projets/Easy-MoE/Easy-MoE/data/Real_talk.jsonl" seq_len = 8 dataset = QAJsonlDataset(path_to_dataset, seq_len) data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn, pin_memory=True)
num_tokens = count_tokens_in_dataset(dataset) print(f"Total number of tokens in the dataset: {num_tokens}")
vocab_size = len(dataset.vocab)
moe = MixtureOfTransformerExperts( input_size=d_model, d_model=d_model, output_size=vocab_size, nhead=nhead, dim_feedforward=dim_feedforward, num_experts=num_experts, num_encoder_layers=num_encoder_layers ).to(device)
moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe).to(device)
Count of total parameters :
total_params = count_parameters(moe_transformer_model) print(f"Total trainable parameters: {total_params}")
---------- Training ----------
Adjusting optimizer setup to include weight decay and allow switching between types
if optimizer_type == "AdamW": optimizer = torch.optim.AdamW(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay) elif optimizer_type == "SGD": optimizer = torch.optim.SGD(moe_transformer_model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=weight_decay) elif optimizer_type == "Adam": optimizer = torch.optim.Adam(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay)
Instantiate your main scheduler (StepLR)
step_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=1.15)
Wrap it with WarmupLR
scheduler = WarmupLR(optimizer, warmup_steps=warmup_steps, scheduler_step_lr=step_lr_scheduler)
criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
Train the model
loss_history = train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader)
Evaluating the model
plot_loss(loss_history) train_accuracy = evaluate_model(moe_transformer_model, data_loader, device)
---------- Inference ----------
def interactive_text_generation(model, dataset, max_length=32, temperature=0.1): while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to exit and save the model): ").strip()
# Check if user wants to quit the interaction if seed_text.lower() == 'quit': print("Exiting text generation mode.") break # Check if the seed text is not empty if seed_text: generated_text = generate_text(model, dataset, seed_text, max_length, temperature) print("Generated Text: ", generated_text) else: print("Seed text cannot be empty. Please enter some text.") except KeyboardInterrupt: # Handle the interrupt signal to exit gracefully print("\nReceived interrupt signal. Exiting text generation mode.") break except Exception as e: # Handle other exceptions and prevent the loop from crashing print(f"An error occurred: {e}. Try again.")interactive_text_generation(moe_transformer_model, dataset)
---------- Save Trained Model ----------
torch.save(moe_transformer_model.state_dict(), "Transformer-Alpha-v02.pth")
- Assistant
To add GEometric Rectified Unit (GeRU) support to your code, you'll need to modify the sections of the code where activation functions are utilized. Specifically, you will have to define a GEometric Rectified Linear Unit (GERU) activation function and apply it in places where you intend to use GERU instead of the previously defined activation functions, such as GELU in your
GatingNetwork.Geometric Rectified Linear Unit (GeRU) is not a standard activation function in PyTorch, so we'll need to implement it based on its definition. Assuming GERU behaves similarly to ReLU but with a twist that incorporates geometric operations or modifications, I'll provide a custom implementation based on this hypothetical description.
Assuming GERU function ( GERU(x) = \max(0, x) + \alpha * \min(0, x) ), with ( \alpha ) as a learnable parameter for negative values:
- Implement GERU Activation Function:
Let's implement a basic
'GERU'class inheriting fromnn.Module. This assumes the hypothetical formula provided; adjust based on the actual definition or mathematical operations of GERU as needed.
class GERU(nn.Module): def __init__(self, in_features): super(GERU, self).__init__() self.alpha = nn.Parameter(torch.rand(in_features)) def forward(self, x): return torch.max(x, torch.zeros_like(x)) + self.alpha * torch.min(x, torch.zeros_like(x))- Modify GatingNetwork to Use GERU:
In your
GatingNetworkreplace all instances ofnn.GELU()with instances of yourGERUfunction. You'll need to pass the correct dimension (hidden_dim) to instantiateGERU.
class GatingNetwork(nn.Module): def __init__(self, input_feature_dim, num_experts, hidden_dims=[512], dropout_rate=0.1): super(GatingNetwork, self).__init__() layers = [] last_dim = input_feature_dim for hidden_dim in hidden_dims: layers.extend([ nn.Linear(last_dim, hidden_dim), GERU(hidden_dim), # Use GERU here nn.Dropout(dropout_rate), ]) last_dim = hidden_dim layers.append(nn.Linear(last_dim, num_experts)) self.fc_layers = nn.Sequential(*layers) self.softmax = nn.Softmax(dim=1)- Activation Parameter Adjustment:
Note that the
GERUclass as defined above introduces learnable parameters (self.alpha). This means that wherever you use GERU, additional parameters are being learned by your model, potentially affecting optimization dynamics.
Ensure you test the modified architecture for stability and performance due to these changes. If GERU has a specific mathematical definition or characteristic I haven't covered, adjust the
GERUclass implementation accordingly.This modification introduces GERU as an activation function in your gating network, replacing GELU. You can also replace other instances of activation functions throughout your model's architecture with GERU in a similar manner if desired.
- Implement GERU Activation Function:
Let's implement a basic
- User
please modify the encoding scheme to be pair-byte based
- Assistant
To implement a pair-byte based encoding scheme, which I presume you meant to be Byte Pair Encoding (BPE), you would traditionally utilize it in the preprocessing stage to tokenize your text into subwords or symbol pairs, which significantly helps in dealing with unknown words or extending the vocabulary in a more efficient manner.
However, the direct implementation of BPE from scratch in the context of the PyTorch code you provided could be quite involved since it demands modifications to your tokenization approach and potentially the structure of your dataset. Typically, BPE tokenization is performed before feeding data into the model. Tools like SentencePiece or libraries like Hugging Face’s Tokenizers are often used for this purpose because they provide efficient implementations of BPE and other tokenization methods.
To switch to a BPE scheme within your current setup, follow the generalized steps below:
1. Preprocess Your Dataset with BPE Tokenization
Here's a simplified approach, assuming use of the
tokenizerslibrary from Hugging Face, which needs to be installed (pip install tokenizers).First, you need to train a BPE tokenizer on your dataset:
from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import Whitespace # Initialize a tokenizer tokenizer = Tokenizer(BPE()) # Initialize a pre-tokenizer to split the input into words tokenizer.pre_tokenizer = Whitespace() # Initialize a trainer trainer = BpeTrainer(special_tokens=["<unk>", "<pad>", "<eos>"]) # Your texts to train on texts = [pair[0] + " " + pair[1] for pair in pairs] # Assuming 'pairs' are (question, answer) from your data tokenizer.train_from_iterator(texts, trainer=trainer) # Save the tokenizer tokenizer.save("bpe_tokenizer.json")You’d train this tokenizer on your entire dataset text.
2. Modify the Dataset to Use BPE Tokenization
Adjust your dataset's
__init__and tokenization to leverage the trained BPE tokenizer.class QAJsonlDataset(Dataset): def __init__(self, path, seq_len): # Load a pretrained tokenizer self.tokenizer = Tokenizer.from_file("bpe_tokenizer.json") self.seq_len = seq_len self.pairs = self.load_data(path) self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs] def tokenize(self, text): # Tokenize and pad/truncate sequences here output = self.tokenizer.encode(text) tokens = output.ids if len(tokens) < self.seq_len: tokens.append(self.tokenizer.token_to_id("<eos>")) tokens.extend([self.tokenizer.token_to_id("<pad>")] * (self.seq_len - len(tokens))) else: tokens = tokens[:self.seq_len - 1] + [self.tokenizer.token_to_id("<eos>")] return tokens3. Adjust the Rest of Your Code as Needed
Ensure that your model and training routine can handle the input from the new tokenization scheme. Since BPE might change the distribution and identity of tokens, it's crucial to update any parts of your code that assumed a specific vocabulary size or token IDs.
Additional Notes:
-
Vocabulary Size: Depending on the number of merges you allow during BPE training, the resulting vocabulary size can significantly differ. Make sure your model’s embedding layers are defined to accommodate this size.
-
Tokenization Scheme: You must adjust the original
build_vocabandtokenizemethods in your dataset class since BPE replaces them. -
Embeddings Size: Your embeddings layer will need to know the vocabulary size which could change your
vocab_sizeparameter in model instantiation.
These steps should offer a starting point for incorporating a Byte Pair Encoding approach into your PyTorch language model. Further adjustments may be necessary based on the specific requirements of your project and the data you're working with.
-
- User
from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import Whitespace from torch.utils.data import DataLoader, Dataset from collections import Counter from torch.nn.utils.rnn import pad_sequence import torch import json
class QAJsonlDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len self.pairs = self.load_data(path) self.vocab, self.idx2token = self.build_vocab([word for pair in self.pairs for sublist in pair for word in sublist]) self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]
def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("user", ""), data.get("content", "") pairs.append((question.split(), answer.split())) return pairs def tokenize(self, words): tokens = [self.vocab.get(w, self.vocab["<unk>"]) for w in words] if len(tokens) < self.seq_len: tokens.append(self.vocab["<eos>"]) tokens.extend([self.vocab["<pad>"]] * (self.seq_len - len(tokens))) else: tokens = tokens[:self.seq_len - 1] + [self.vocab["<eos>"]] return tokens def build_vocab(self, words): vocab = {"<unk>": 0, "<pad>": 1, "<eos>": 2} start_index = len(vocab) counts = Counter(words) for word, _ in counts.most_common(): if word not in vocab: vocab[word] = len(vocab) idx2token = {idx: token for token, idx in vocab.items()} return vocab, idx2token def __len__(self): return len(self.tokenized_pairs) def __getitem__(self, idx): tokenized_question, tokenized_answer = self.tokenized_pairs[idx] return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)def collate_fn(batch): questions, answers = zip(*batch) questions = pad_sequence(questions, batch_first=True, padding_value=0) answers = pad_sequence(answers, batch_first=True, padding_value=0) return questions, answers
Dataset :
path_to_dataset = "C:/Users/L14/Documents/Projets/Easy-MoE/Easy-MoE/data/Real_talk.jsonl" seq_len = 8 pairs = QAJsonlDataset(path_to_dataset, seq_len)
Initialize a tokenizer
tokenizer = Tokenizer(BPE())
Initialize a pre-tokenizer to split the input into words
tokenizer.pre_tokenizer = Whitespace()
Initialize a trainer
trainer = BpeTrainer(special_tokens=["", "", ""])
Your texts to train on
texts = [pair[0] + " " + pair[1] for pair in pairs] # Assuming ‘pairs’ are (question, answer) from your data tokenizer.train_from_iterator(texts, trainer=trainer)
Save the tokenizer
tokenizer.save("bpe_tokenizer.json")
- Assistant
Your implementation attempts to integrate Byte Pair Encoding (BPE) into your dataset preparation process, which is a good approach for handling vocabularies in NLP applications more efficiently and dealing with out-of-vocabulary (OOV) words. However, there are a few areas in the given code that need correction or further clarification:
-
Training the BPE Tokenizer on Your Dataset: To train a BPE tokenizer, you must provide a list of strings where each string is a single example from your dataset. Your current implementation attempts to generate this list by iterating over your
QAJsonlDatasetinstance as if it's iterable, directly combining question and answer pairs. This approach won't work directly without adjustments because theQAJsonlDatasetreturns tensors from its__getitem__method, not plain text. -
Data Preparation for BPE Training: Before training the tokenizer, you need to prepare your data properly. You must extract all questions and answers from your dataset and flatten them into a list of strings where each string is either a complete question or answer.
Let's correct these issues and ensure the BPE tokenizer is trained with the correct inputs:
Adjusting the Dataset Loading and BPE Tokenizer Training
from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import Whitespace import json class QAJsonlDataset: def __init__(self, path): self.pairs = self.load_data(path) def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("user", ""), data.get("content", "") pairs.append((question, answer)) return pairs def preprocess_data_for_bpe(path_to_dataset): # Initialize dataset dataset = QAJsonlDataset(path_to_dataset) # Prepare data for BPE texts = [question + " " + answer for question, answer in dataset.pairs] return texts # Preprocess the data path_to_dataset = "path/to/your/dataset.jsonl" texts = preprocess_data_for_bpe(path_to_dataset) # Initialize a tokenizer tokenizer = Tokenizer(BPE()) # Initialize a pre-tokenizer tokenizer.pre_tokenizer = Whitespace() # Initialize a trainer trainer = BpeTrainer(special_tokens=["<unk>", "<pad>", "<eos>"]) # Train the tokenizer tokenizer.train_from_iterator(texts, trainer=trainer) # Save the tokenizer tokenizer.save("bpe_tokenizer.json")Key Points:
- Dataset Preprocessing: The code has been adjusted to ensure that texts are prepared correctly for BPE tokenizer training. We concatenate questions and answers directly when preparing
texts, assuming that each line in your JSON lines (.jsonl) file has a single question-answer pair. - Training the Tokenizer: After preparing
texts, the tokenizer is properly trained on these combinations of questions and answers. - Tokenizer Saving: Once trained, the tokenizer is saved to
bpe_tokenizer.json, which you can later load for tokenizing new texts or for integrating into the__getitem__method of your dataset class.
Remember, after training your BPE tokenizer, you should adjust the tokenization process in your dataset to use this tokenizer. This involves loading the trained tokenizer and then applying it within the
tokenizemethod rather than using the previously defined vocabulary-based method. -
- User
now that i have the tokenizer, i need you to modify the QAjsonldataset class to use the bpe tokenizer now, here is the code for reference: import torch import torch.nn as nn import torch.nn.functional as F import json import math from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter from tqdm import tqdm import matplotlib.pyplot as plt from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score
---------- Device Configuration ----------
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
---------- Utility Functions ----------
def positional_encoding(seq_len, d_model, device): pos = torch.arange(seq_len, dtype=torch.float, device=device).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)).to(device) pe = torch.zeros(seq_len, d_model, device=device) pe[:, 0::2] = torch.sin(pos * div_term) pe[:, 1::2] = torch.cos(pos * div_term) return pe.unsqueeze(0)
-------- Performance ----------
def evaluate_model(model, data_loader, device): model.eval() all_preds, all_targets = [], []
with torch.no_grad(): for inputs, targets in data_loader: inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) predictions = torch.argmax(outputs, dim=-1).view(-1) # Flatten predicted indices all_preds.extend(predictions.cpu().numpy()) all_targets.extend(targets.view(-1).cpu().numpy()) # Ensure targets are also flattened # Calculate precision, recall, and F1 score after ensuring all_preds and all_targets are correctly aligned. precision = precision_score(all_targets, all_preds, average='macro', zero_division=0) recall = recall_score(all_targets, all_preds, average='macro', zero_division=0) f1 = f1_score(all_targets, all_preds, average='macro', zero_division=0) print(f"Precision: {precision:.4f}") print(f"Recall: {recall:.4f}") print(f"F1 Score: {f1:.4f}") return precision, recall, f1Function to plot loss over time
def plot_loss(loss_history): plt.figure(figsize=(10, 5)) plt.plot(loss_history, label='Training Loss') plt.xlabel('Batches') plt.ylabel('Loss') plt.title('Training Loss Over Time') plt.legend() plt.show()
---------- Model Definitions ----------
class TransformerExpert(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1): super(TransformerExpert, self).init() self.d_model = d_model self.input_fc = nn.Linear(input_size, d_model) self.pos_encoder = nn.Parameter(positional_encoding(1, d_model, device), requires_grad=True) encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True, norm_first=True) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers) self.output_fc = nn.Linear(d_model, output_size) self.norm = nn.LayerNorm(d_model)
def forward(self, x): x = self.norm(self.input_fc(x)) + self.pos_encoder transformer_output = self.transformer_encoder(x) output = self.output_fc(transformer_output) return outputclass GatingNetwork(nn.Module): def init(self, input_feature_dim, num_experts, hidden_dims=[512], dropout_rate=0.1): super(GatingNetwork, self).init() layers = [] last_dim = input_feature_dim
for hidden_dim in hidden_dims: layers.extend([ nn.Linear(last_dim, hidden_dim), nn.GELU(), nn.Dropout(dropout_rate), ]) last_dim = hidden_dim layers.append(nn.Linear(last_dim, num_experts)) self.fc_layers = nn.Sequential(*layers) self.softmax = nn.Softmax(dim=1) def forward(self, x): x = x.mean(dim=1) # To ensure gating is based on overall features across the sequence x = self.fc_layers(x) return self.softmax(x)class MixtureOfTransformerExperts(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_experts, num_encoder_layers=1): super(MixtureOfTransformerExperts, self).init() self.num_experts = num_experts self.output_size = output_size self.experts = nn.ModuleList([TransformerExpert(input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers) for _ in range(num_experts)]) self.gating_network = GatingNetwork(d_model, num_experts)
def forward(self, x): gating_scores = self.gating_network(x) expert_outputs = [expert(x) for expert in self.experts] stacked_expert_outputs = torch.stack(expert_outputs) expanded_gating_scores = gating_scores.unsqueeze(2).unsqueeze(3) expanded_gating_scores = expanded_gating_scores.expand(-1, -1, x.size(1), self.output_size) expanded_gating_scores = expanded_gating_scores.transpose(0, 1) mixed_output = torch.sum(stacked_expert_outputs * expanded_gating_scores, dim=0) return mixed_outputclass MoETransformerModel(nn.Module): def init(self, vocab_size, d_model, moe): super(MoETransformerModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=d_model) self.moe = moe self.dropout = nn.Dropout(p=0.125)
def forward(self, x): embedded = self.dropout(self.embedding(x)) return self.moe(embedded)---------- Dataset Definitions ----------
class QAJsonlDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len self.pairs = self.load_data(path) self.vocab, self.idx2token = self.build_vocab([word for pair in self.pairs for sublist in pair for word in sublist]) self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]
def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("user", ""), data.get("content", "") pairs.append((question.split(), answer.split())) return pairs def tokenize(self, words): tokens = [self.vocab.get(w, self.vocab["<unk>"]) for w in words] if len(tokens) < self.seq_len: tokens.append(self.vocab["<eos>"]) tokens.extend([self.vocab["<pad>"]] * (self.seq_len - len(tokens))) else: tokens = tokens[:self.seq_len - 1] + [self.vocab["<eos>"]] return tokens def build_vocab(self, words): vocab = {"<unk>": 0, "<pad>": 1, "<eos>": 2} start_index = len(vocab) counts = Counter(words) for word, _ in counts.most_common(): if word not in vocab: vocab[word] = len(vocab) idx2token = {idx: token for token, idx in vocab.items()} return vocab, idx2token def __len__(self): return len(self.tokenized_pairs) def __getitem__(self, idx): tokenized_question, tokenized_answer = self.tokenized_pairs[idx] return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)def collate_fn(batch): questions, answers = zip(*batch) questions = pad_sequence(questions, batch_first=True, padding_value=0) answers = pad_sequence(answers, batch_first=True, padding_value=0) return questions, answers
---------- Training and Inference Functions ----------
def train_model(model, criterion, optimizer, num_epochs, data_loader): model.train() loss_history = [] # Initialize a list to keep track of losses for epoch in range(num_epochs): total_loss = 0 total_items = 0 # Keep track of total items processed progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False) for i, (inputs, targets) in progress_bar: inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad()
# Predict predictions = model(inputs) predictions = predictions.view(-1, predictions.size(-1)) # Make sure predictions are the right shape targets = targets.view(-1) # Flatten targets to match prediction shape if necessary # Calculate loss loss = criterion(predictions, targets) loss.backward() # Gradient clipping for stabilization torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) optimizer.step() scheduler.step() # Update total loss and the number of items total_loss += loss.item() * inputs.size(0) # Multiply loss by batch size total_items += inputs.size(0) loss_history.append(loss.item()) progress_bar.set_postfix({"Loss": loss.item()}) average_loss = total_loss / total_items # Correctly compute average loss print(f"Epoch {epoch+1}, Average Loss: {average_loss:.6f}") return loss_historyclass WarmupLR(torch.optim.lr_scheduler._LRScheduler): def init(self, optimizer, warmup_steps, scheduler_step_lr): self.warmup_steps = warmup_steps self.scheduler_step_lr = scheduler_step_lr # The subsequent scheduler super(WarmupLR, self).init(optimizer)
def get_lr(self): if self._step_count <= self.warmup_steps: warmup_factor = float(self._step_count) / float(max(1, self.warmup_steps)) for base_lr in self.base_lrs: yield base_lr * warmup_factor else: self.scheduler_step_lr.step() # Update the subsequent scheduler for param_group in self.optimizer.param_groups: yield param_group['lr']class GERU(nn.Module): def init(self, in_features): super(GERU, self).init() self.alpha = nn.Parameter(torch.rand(in_features))
def forward(self, x): return torch.max(x, torch.zeros_like(x)) + self.alpha * torch.min(x, torch.zeros_like(x))def generate_text(model, dataset, seed_text, num_generate, temperature=1.0): model.eval()
generated_tokens = [] # Initial sequence (prefix) to start the generation process input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()] current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0) current_sequence = current_sequence.to(device) # Generate num_generate tokens for _ in range(num_generate): # Forward pass through the model with torch.no_grad(): output = model(current_sequence) # Get probabilities, apply temperature scaling, and sample from the distribution probabilities = F.softmax(output[:, -1, :] / temperature, dim=-1).detach() next_token_idx = torch.multinomial(probabilities, 1).item() # Append token to the current sequence and to the generated tokens generated_tokens.append(next_token_idx) current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1).to(device) # Convert tokens to words generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens]) return generated_textdef count_tokens_in_dataset(dataset): return sum([len(pair[0]) + len(pair[1]) for pair in dataset.pairs])
def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)
---------- Hyperparameters and Model Instantiation ----------
Transformer :
d_model = 256 nhead = 4 dim_feedforward = 1024 num_encoder_layers = 4 num_experts = 1 hidden_dims = [] dropout_rate = 0.5
Gating Network :
gating_network = GatingNetwork( input_feature_dim=d_model, num_experts=num_experts, hidden_dims=hidden_dims, dropout_rate=dropout_rate, )
Training Parameters
batch_size = 32 # Adjustable batch size warmup_steps = 6000 # Warmup steps for learning rate optimizer_type = "AdamW" # Could be “SGD”, “RMSprop”, etc. learning_rate = 1e-2 weight_decay = 1e-5 # For L2 regularization num_epochs = 100
Dataset :
path_to_dataset = "C:/Users/L14/Documents/Projets/Easy-MoE/Easy-MoE/data/Real_talk.jsonl" seq_len = 8 dataset = QAJsonlDataset(path_to_dataset, seq_len) data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn, pin_memory=True)
num_tokens = count_tokens_in_dataset(dataset) print(f"Total number of tokens in the dataset: {num_tokens}")
vocab_size = len(dataset.vocab)
moe = MixtureOfTransformerExperts( input_size=d_model, d_model=d_model, output_size=vocab_size, nhead=nhead, dim_feedforward=dim_feedforward, num_experts=num_experts, num_encoder_layers=num_encoder_layers ).to(device)
moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe).to(device)
Count of total parameters :
total_params = count_parameters(moe_transformer_model) print(f"Total trainable parameters: {total_params}")
---------- Training ----------
Adjusting optimizer setup to include weight decay and allow switching between types
if optimizer_type == "AdamW": optimizer = torch.optim.AdamW(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay) elif optimizer_type == "SGD": optimizer = torch.optim.SGD(moe_transformer_model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=weight_decay) elif optimizer_type == "Adam": optimizer = torch.optim.Adam(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay)
Instantiate your main scheduler (StepLR)
step_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=1.15)
Wrap it with WarmupLR
scheduler = WarmupLR(optimizer, warmup_steps=warmup_steps, scheduler_step_lr=step_lr_scheduler)
criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
Train the model
loss_history = train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader)
Evaluating the model
plot_loss(loss_history) train_accuracy = evaluate_model(moe_transformer_model, data_loader, device)
---------- Inference ----------
def interactive_text_generation(model, dataset, max_length=32, temperature=0.1): while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to exit and save the model): ").strip()
# Check if user wants to quit the interaction if seed_text.lower() == 'quit': print("Exiting text generation mode.") break # Check if the seed text is not empty if seed_text: generated_text = generate_text(model, dataset, seed_text, max_length, temperature) print("Generated Text: ", generated_text) else: print("Seed text cannot be empty. Please enter some text.") except KeyboardInterrupt: # Handle the interrupt signal to exit gracefully print("\nReceived interrupt signal. Exiting text generation mode.") break except Exception as e: # Handle other exceptions and prevent the loop from crashing print(f"An error occurred: {e}. Try again.")interactive_text_generation(moe_transformer_model, dataset)
---------- Save Trained Model ----------
torch.save(moe_transformer_model.state_dict(), "Transformer-Alpha-v02.pth")
- Assistant
To modify the
QAJsonlDatasetclass to use the Byte Pair Encoding (BPE) tokenizer that you have trained, you'll need to load the tokenizer, and adapt thetokenizemethod to use it. This involves changing the dataset to ignore the previously built vocabulary and instead tokenize inputs using BPE directly.Here's how you can modify the
QAJsonlDatasetclass:from tokenizers import Tokenizer from torch.utils.data import DataLoader, Dataset import torch import json class QAJsonlDataset(Dataset): def __init__(self, path, seq_len, tokenizer_path): # Load the trained tokenizer self.tokenizer = Tokenizer.from_file(tokenizer_path) self.seq_len = seq_len self.pairs = self.load_data(path) # Using BPE, so no need for manual vocab or idx2token. # Tokenization will now happen using self.tokenizer. self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs] def load_data(self, path): pairs = [] with open(path, "r", encoding="utf-8") as f: for line in f: data = json.loads(line.strip()) question, answer = data.get("user", ""), data.get("content", "") pairs.append((question, answer)) # Store questions and answers as raw strings return pairs def tokenize(self, text): # Tokenizing using the BPE tokenizer encoded = self.tokenizer.encode(text) tokens = encoded.ids # Padding/truncation if len(tokens) < self.seq_len: # Padding tokens += [self.tokenizer.token_to_id("<pad>")] * (self.seq_len - len(tokens)) else: # Truncation tokens = tokens[:self.seq_len - 1] + [self.tokenizer.token_to_id("<eos>")] return tokens def __len__(self): return len(self.tokenized_pairs) def __getitem__(self, idx): tokenized_question, tokenized_answer = self.tokenized_pairs[idx] return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long) # Example usage: path_to_dataset = "C:/Users/L14/Documents/Projets/Easy-MoE/Easy-MoE/data/Real_talk.jsonl" seq_len = 8 tokenizer_path = "bpe_tokenizer.json" # Update this to your tokenizer's file path dataset = QAJsonlDataset(path_to_dataset, seq_len, tokenizer_path)Please ensure that
tokenizer_pathis correctly pointing to your saved tokenizer.This modification will ensure that data passed through the
QAJsonlDatasetclass will be tokenized using your trained BPE tokenizer, allowing you to leverage the BPE method's advantages, such as handling of unknown words, in your model training and evaluation. - User
Traceback (most recent call last): File "c:\Users\L14\Documents\Projets\Easy-MoE\Easy-MoE\Easy-MoE_1.5.py", line 323, in vocab_size = len(dataset.vocab) AttributeError: 'QAJsonlDataset' object has no attribute 'vocab'
- Assistant
The error you're encountering arises because the modified
QAJsonlDatasetclass no longer defines or uses avocabattribute. In the previous version of your dataset class, bothvocabandidx2tokenwere used for your custom tokenization process. However, now that you've switched to using a pre-trained BPE tokenizer, these are no longer necessary, and the tokenizer handles everything related to vocabulary and token IDs internally.To resolve the issue, you need to determine the vocabulary size dynamically from the tokenizer when setting up your model that uses this dataset. Here’s how you can adjust your code to get the
vocab_sizefrom the BPE tokenizer:# Load the tokenizer tokenizer = Tokenizer.from_file("bpe_tokenizer.json") # Determine the vocabulary size vocab_size = tokenizer.get_vocab_size() # Now, you can use this `vocab_size` when defining your model moe = MixtureOfTransformerExperts( input_size=d_model, d_model=d_model, output_size=vocab_size, nhead=nhead, dim_feedforward=dim_feedforward, num_experts=num_experts, num_encoder_layers=num_encoder_layers ).to(device) moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe).to(device)Please ensure that any other place in your code that referred to
dataset.vocabor used the dataset's vocabulary directly is updated to use the vocabulary size or information obtained from the tokenizer.Remember, with BPE and similar subword tokenization methods, the vocabulary is implicitly handled by the tokenizer, and you usually work with token IDs directly after tokenization without manual mapping between tokens and indices. This change should simplify the handling of vocabularies in your code and make it more flexible for handling various texts, including those with previously unseen words.
- User
check this: Epoch 96, Average Loss: 1.623216
Epoch 97, Average Loss: 1.586082
Epoch 98, Average Loss: 1.572504
Epoch 99, Average Loss: 1.631836
Epoch 100, Average Loss: 1.599515
Accuracy: 0.7222 Precision: 0.7124 Recall: 0.7965 F1 Score: 0.7231 Enter seed text (type 'quit' to exit and save the model): what is the sun ? An error occurred: 'QAJsonlDataset' object has no attribute 'vocab'. Try again. Enter seed text (type 'quit' to exit and save the model): what is the sun ? An error occurred: 'QAJsonlDataset' object has no attribute 'vocab'. Try again. - Assistant
The error you're encountering during interactive text generation is due to the
generate_textfunction (or a similar function you're using for text generation) referencing an attributevocabon theQAJsonlDatasetobject, which, as per the recent modifications to use the BPE tokenizer, no longer exists.Given that you have switched to using the BPE tokenizer, the text generation process needs to be adjusted to utilize the BPE tokenizer for encoding the seed text and decoding the generated text tokens back to text.
Here is how you could adjust the text generation function to use the tokenizer correctly:
def generate_text(model, tokenizer, seed_text, num_generate, temperature=1.0): model.eval() generated_tokens = [] # Encode the seed text using the tokenizer encoded_input = tokenizer.encode(seed_text) input_ids = torch.tensor(encoded_input.ids, dtype=torch.long).unsqueeze(0).to(device) # Generate num_generate tokens with torch.no_grad(): for _ in range(num_generate): output = model(input_ids) # Get the last logits and apply temperature logits = output[:, -1, :] / temperature probabilities = F.softmax(logits, dim=-1) next_token_id = torch.multinomial(probabilities, num_samples=1).item() # Append generated token ID and prepare the new input_ids generated_tokens.append(next_token_id) input_ids = torch.cat([input_ids, torch.tensor([[next_token_id]], dtype=torch.long).to(device)], dim=1) # Decode the generated token IDs back to text generated_text = tokenizer.decode(generated_tokens) return generated_text # Interactive text generation setup tokenizer = Tokenizer.from_file("bpe_tokenizer.json") # Make sure this path is correct while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to exit and save the model): ").strip() # Check if the user wants to quit the interaction if seed_text.lower() == 'quit': print("Exiting text generation mode.") break # Generate text based on the seed text if seed_text: generated_text = generate_text(moe_transformer_model, tokenizer, seed_text, max_length=32, temperature=0.7) # Modify max_length/temperature as needed print("Generated Text:", generated_text) else: print("Seed text cannot be empty. Please enter some text.") except Exception as e: # Handle potential errors gracefully print(f"An error occurred: {e}. Try again.")Ensure you replace the references to vocab or idx2token with the appropriate method calls to the tokenizer. This adjustment uses the BPE tokenizer to encode the input seed text and decode the generated output, aligning with the changes made to rely on the BPE tokenizer for handling the vocabulary.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.