Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· openaiHugging Face Suzhen/CodeChat

what is wrong with my code : import torch import torch.nn as nn import…

  1. User

    what is wrong with my code : import torch import torch.nn as nn import torch.nn.functional as F import json import math from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter from tqdm import tqdm import matplotlib.pyplot as plt from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score from tokenizers import Tokenizer

    ---------- Device Configuration ----------

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    ---------- Utility Functions ----------

    def positional_encoding(seq_len, d_model, device): pos = torch.arange(seq_len, dtype=torch.float, device=device).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)).to(device) pe = torch.zeros(seq_len, d_model, device=device) pe[:, 0::2] = torch.sin(pos * div_term) pe[:, 1::2] = torch.cos(pos * div_term) return pe.unsqueeze(0)

    -------- Performance ----------

    def evaluate_model(model, data_loader, device): model.eval() all_preds, all_targets = [], []

    with torch.no_grad():
        for inputs, targets in data_loader:
            inputs, targets = inputs.to(device), targets.to(device)
    
            outputs = model(inputs)
            predictions = torch.argmax(outputs, dim=-1).view(-1)  # Flatten predicted indices
    
            all_preds.extend(predictions.cpu().numpy())
            all_targets.extend(targets.view(-1).cpu().numpy())  # Ensure targets are also flattened
    
    # Calculate precision, recall, and F1 score after ensuring all_preds and all_targets are correctly aligned.
    accuracy = accuracy_score(all_targets, all_preds)
    precision = precision_score(all_targets, all_preds, average='macro', zero_division=0)
    recall = recall_score(all_targets, all_preds, average='macro', zero_division=0)
    f1 = f1_score(all_targets, all_preds, average='macro', zero_division=0)
    
    print(f"Accuracy: {accuracy:.4f}")
    print(f"Precision: {precision:.4f}")
    print(f"Recall: {recall:.4f}")
    print(f"F1 Score: {f1:.4f}")
    
    return accuracy ,precision, recall, f1
    

    Function to plot loss over time

    def plot_loss(loss_history): plt.figure(figsize=(10, 5)) plt.plot(loss_history, label='Training Loss') plt.xlabel('Batches') plt.ylabel('Loss') plt.title('Training Loss Over Time') plt.legend() plt.show()

    ---------- Model Definitions ----------

    class TransformerExpert(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1): super(TransformerExpert, self).init() self.d_model = d_model self.input_fc = nn.Linear(input_size, d_model) self.pos_encoder = nn.Parameter(positional_encoding(1, d_model, device), requires_grad=True) encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True, norm_first=True) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers) self.output_fc = nn.Linear(d_model, output_size) self.norm = nn.LayerNorm(d_model)

    def forward(self, x):
        x = self.norm(self.input_fc(x)) + self.pos_encoder
        transformer_output = self.transformer_encoder(x)
        output = self.output_fc(transformer_output)
        return output
    

    class GatingNetwork(nn.Module): def init(self, input_feature_dim, num_experts, hidden_dims=[6144, 3072, 1536, 768], dropout_rate=0.2): super(GatingNetwork, self).init() layers = [] last_dim = input_feature_dim

        for hidden_dim in hidden_dims:
            layers.extend([
                nn.Linear(last_dim, hidden_dim),
                nn.GELU(),
                nn.Dropout(dropout_rate),
            ])
            last_dim = hidden_dim
    
        layers.append(nn.Linear(last_dim, num_experts))
        self.fc_layers = nn.Sequential(*layers)
        self.softmax = nn.Softmax(dim=1)
    
    def forward(self, x):
        x = x.mean(dim=1)  # To ensure gating is based on overall features across the sequence
        x = self.fc_layers(x)
        return self.softmax(x)
    

    class MixtureOfTransformerExperts(nn.Module): def init(self, input_size, d_model, output_size, nhead, dim_feedforward, num_experts, num_encoder_layers=1): super(MixtureOfTransformerExperts, self).init() self.num_experts = num_experts self.output_size = output_size self.experts = nn.ModuleList([TransformerExpert(input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers) for _ in range(num_experts)]) self.gating_network = GatingNetwork(d_model, num_experts)

    def forward(self, x):
        gating_scores = self.gating_network(x)
    
        expert_outputs = [expert(x) for expert in self.experts]
        stacked_expert_outputs = torch.stack(expert_outputs)
    
        expanded_gating_scores = gating_scores.unsqueeze(2).unsqueeze(3)
        expanded_gating_scores = expanded_gating_scores.expand(-1, -1, x.size(1), self.output_size)
        expanded_gating_scores = expanded_gating_scores.transpose(0, 1)
    
        mixed_output = torch.sum(stacked_expert_outputs * expanded_gating_scores, dim=0)
        return mixed_output
    

    class MoETransformerModel(nn.Module): def init(self, vocab_size, d_model, moe): super(MoETransformerModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=d_model) self.moe = moe self.dropout = nn.Dropout(p=0.1)

    def forward(self, x):
        embedded = self.dropout(self.embedding(x))
        return self.moe(embedded)
    

    ---------- Dataset Definitions ----------

    class QAJsonlDataset(Dataset): def init(self, path, seq_len, tokenizer_path): # Load the trained tokenizer self.tokenizer = Tokenizer.from_file(tokenizer_path)

        self.seq_len = seq_len
        self.pairs = self.load_data(path)
        # Using BPE, so no need for manual vocab or idx2token.
        # Tokenization will now happen using self.tokenizer.
        self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]
    
    def load_data(self, path):
        pairs = []
        with open(path, "r", encoding="utf-8") as f:
            for line in f:
                data = json.loads(line.strip())
                question, answer = data.get("user", ""), data.get("content", "")
                pairs.append((question, answer))  # Store questions and answers as raw strings
        return pairs
    
    def tokenize(self, text):
        # Tokenizing using the BPE tokenizer
        encoded = self.tokenizer.encode(text)
        tokens = encoded.ids
        # Padding/truncation
        if len(tokens) < self.seq_len:
            # Padding
            tokens += [self.tokenizer.token_to_id("<pad>")] * (self.seq_len - len(tokens))
        else:
            # Truncation
            tokens = tokens[:self.seq_len - 1] + [self.tokenizer.token_to_id("<eos>")]
        return tokens
        
    def __len__(self):
        return len(self.tokenized_pairs)
    
    def __getitem__(self, idx):
        tokenized_question, tokenized_answer = self.tokenized_pairs[idx]
        return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)
    

    def collate_fn(batch): questions, answers = zip(*batch) questions = pad_sequence(questions, batch_first=True, padding_value=0) answers = pad_sequence(answers, batch_first=True, padding_value=0) return questions, answers

    ---------- Training and Inference Functions ----------

    def train_model(model, criterion, optimizer, num_epochs, data_loader): model.train() loss_history = [] # Initialize a list to keep track of losses for epoch in range(num_epochs): total_loss = 0 total_items = 0 # Keep track of total items processed progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False) for i, (inputs, targets) in progress_bar: inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad()

            # Predict
            predictions = model(inputs)
            predictions = predictions.view(-1, predictions.size(-1))  # Make sure predictions are the right shape
            targets = targets.view(-1)  # Flatten targets to match prediction shape if necessary
            
            # Calculate loss
            loss = criterion(predictions, targets)
            loss.backward()
            
            # Gradient clipping for stabilization
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
            optimizer.step()
            scheduler.step()
            
            # Update total loss and the number of items
            total_loss += loss.item() * inputs.size(0)  # Multiply loss by batch size
            total_items += inputs.size(0)
            loss_history.append(loss.item())
            
            progress_bar.set_postfix({"Loss": loss.item()})
        
        average_loss = total_loss / total_items  # Correctly compute average loss
        print(f"Epoch {epoch+1}, Average Loss: {average_loss:.6f}")
    
    return loss_history
    

    class WarmupLR(torch.optim.lr_scheduler._LRScheduler): def init(self, optimizer, warmup_steps, scheduler_step_lr): self.warmup_steps = warmup_steps self.scheduler_step_lr = scheduler_step_lr # The subsequent scheduler super(WarmupLR, self).init(optimizer)

    def get_lr(self):
        if self._step_count <= self.warmup_steps:
            warmup_factor = float(self._step_count) / float(max(1, self.warmup_steps))
            for base_lr in self.base_lrs:
                yield base_lr * warmup_factor
        else:
            self.scheduler_step_lr.step()  # Update the subsequent scheduler
            for param_group in self.optimizer.param_groups:
                yield param_group['lr']
    

    class GERU(nn.Module): def init(self, in_features): super(GERU, self).init() self.alpha = nn.Parameter(torch.rand(in_features))

    def forward(self, x):
        return torch.max(x, torch.zeros_like(x)) + self.alpha * torch.min(x, torch.zeros_like(x))
    

    def generate_text(model, tokenizer, seed_text, num_generate, temperature=1.0): model.eval()

    generated_tokens = []
    
    # Encode the seed text using the tokenizer
    encoded_input = tokenizer.encode(seed_text)
    input_ids = torch.tensor(encoded_input.ids, dtype=torch.long).unsqueeze(0).to(device)
    
    # Generate num_generate tokens
    with torch.no_grad():
        for _ in range(num_generate):
            output = model(input_ids)
            
            # Get the last logits and apply temperature
            logits = output[:, -1, :] / temperature
            probabilities = F.softmax(logits, dim=-1)
            next_token_id = torch.multinomial(probabilities, num_samples=1).item()
    
            # Append generated token ID and prepare the new input_ids
            generated_tokens.append(next_token_id)
            input_ids = torch.cat([input_ids, torch.tensor([[next_token_id]], dtype=torch.long).to(device)], dim=1)
    
    # Decode the generated token IDs back to text
    generated_text = tokenizer.decode(generated_tokens)
    return generated_text
    

    def count_tokens_in_dataset(dataset): return sum([len(pair[0]) + len(pair[1]) for pair in dataset.pairs])

    def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)

    ---------- Hyperparameters and Model Instantiation ----------

    Transformer :

    d_model = 384 nhead = 12 dim_feedforward = 1536 num_encoder_layers = 12 num_experts = 2

    Training Parameters

    batch_size = 384 # Adjustable batch size warmup_steps = 2000 # Warmup steps for learning rate optimizer_type = "AdamW" # Could be “SGD”, “RMSprop”, etc. learning_rate = 1e-1 weight_decay = 1e-5 # For L2 regularization num_epochs = 10

    Dataset :

    path_to_dataset = "C:/Users/L14/Documents/Projets/Easy-MoE/Easy-MoE/data/Real_talk.jsonl" tokenizer_path = "BPE_tokenizer(Real-Talk).json" seq_len = 256 dataset = QAJsonlDataset(path_to_dataset, seq_len, tokenizer_path) data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn, pin_memory=True)

    num_tokens = count_tokens_in_dataset(dataset) print(f"Total number of tokens in the dataset: {num_tokens}")

    Load the tokenizer

    tokenizer = Tokenizer.from_file(tokenizer_path)

    Determine the vocabulary size

    vocab_size = tokenizer.get_vocab_size()

    moe = MixtureOfTransformerExperts( input_size=d_model, d_model=d_model, output_size=vocab_size, nhead=nhead, dim_feedforward=dim_feedforward, num_experts=num_experts, num_encoder_layers=num_encoder_layers ).to(device)

    moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe).to(device)

    Count of total parameters :

    total_params = count_parameters(moe_transformer_model) print(f"Total trainable parameters: {total_params}")

    ---------- Training ----------

    Adjusting optimizer setup to include weight decay and allow switching between types

    if optimizer_type == "AdamW": optimizer = torch.optim.AdamW(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay) elif optimizer_type == "SGD": optimizer = torch.optim.SGD(moe_transformer_model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=weight_decay) elif optimizer_type == "Adam": optimizer = torch.optim.Adam(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=weight_decay)

    Instantiate your main scheduler (StepLR)

    step_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.95)

    Wrap it with WarmupLR

    scheduler = WarmupLR(optimizer, warmup_steps=warmup_steps, scheduler_step_lr=step_lr_scheduler)

    criterion = nn.CrossEntropyLoss(label_smoothing=0.1)

    Train the model

    loss_history = train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader)

    Evaluating the model

    plot_loss(loss_history) train_accuracy = evaluate_model(moe_transformer_model, data_loader, device)

    ---------- Inference ----------

    def interactive_text_generation(model, dataset, max_length=32, temperature=1.0): while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to exit and save the model): ").strip()

            # Check if the user wants to quit the interaction
            if seed_text.lower() == 'quit':
                print("Exiting text generation mode.")
                break
    
            # Generate text based on the seed text
            if seed_text:
                generated_text = generate_text(model, dataset, seed_text, max_length, temperature)  # Modify max_length/temperature as needed
                print("Generated Text:", generated_text)
            else:
                print("Seed text cannot be empty. Please enter some text.")
                
        except Exception as e:
            # Handle potential errors gracefully
            print(f"An error occurred: {e}. Try again.")
    

    interactive_text_generation(moe_transformer_model, tokenizer)

    ---------- Save Trained Model ----------

    torch.save(moe_transformer_model.state_dict(), "Transformer-Alpha-v04.pth")

  2. Assistant

    Your code is largely well-structured and comprehensive, and it's evident that you've implemented a complex system involving a mixture of transformer experts model. However, there are several points I'd like to highlight which may not be "wrong" but could lead to issues or optimizations. Additionally, I found a couple of potential problems that could hinder the seamless execution of your code:

    1. In the positional_encoding function, the pos tensor and div_term calculations look accurate, but the dimensions for positional encodings are hardcoded, which means that it won't work if your sequence length changes after initialization. It would be more flexible to generate positional encodings dynamically based on the input batch size, although it might not be a "mistake" depending on your specific use case.

    2. In the TransformerExpert model, you use nn.Parameter to make your positional encodings learnable by setting requires_grad=True. This is not inherently wrong, but it's a deviation from the common practice where positional encodings are fixed. If this is an intentional design choice, it's fine, but it's something worth noting as it might lead to unexpected behavior compared to the traditional transformer model.

    3. You are using scheduler.step() inside your training loop without checking if scheduler is defined. This might throw a NameError if the code is modified in a way that scheduler is not defined or instantiated before its use. It would be safer to check if scheduler exists before calling its step method.

    4. When calling your training loop, data_loader is passed as a parameter, but there is no validation split or validation data loader passed to validate the model during training. Incorporating validation would help in monitoring overfitting during training.

    5. The train_model function references scheduler.step(), but scheduler is not passed as a parameter to the function. This would cause a NameError. Pass scheduler as a parameter to the train_model function to fix this issue.

    6. You are using softmax in the GatingNetwork forward method for your mixture of experts, which is correct for obtaining the weights for each expert, but be cautious of numerical instability. PyTorch's nn.Softmax handles this pretty well, though, so it's not a major concern.

    7. The collate_fn function correctly pads the sequences, which is good for varying lengths of input. However, make sure your model can efficiently handle padded sequences (which it seems like it does, given the transformer architecture).

    8. In interactive_text_generation, you are using dataset.tokenizer instead of just tokenizer in the function parameters, leading to a potential error because dataset may not have a tokenizer attribute. Revise the function definition to accept the tokenizer directly.

    9. Consider moving model-saving out of the interactive_text_generation loop. This loop is designed for interactive text generation, and it might not be intuitive for the user that exiting the loop saves the model.

    Finally, these are not necessarily "mistakes" but optimizations and best practices that can help refine your code and possibly pinpoint areas of improvement or potential bugs. Overall, your implementation is sophisticated and represents a deep understanding of PyTorch and NLP model architecture.

  3. User

    give me a fix for the first potential issue

  4. Assistant

    The first potential issue concerns the usage of hardcoded positional encodings that might not adapt if the sequence length changes dynamically. To address this, you can modify the positional_encoding function to dynamically generate positional encodings based on the sequence length of the inputs it receives during the forward pass. However, since you want your positional encoding to be a learnable parameter and initialized once, you can initiate it with a large enough sequence length and then adapt dynamically during the forward pass based on actual input lengths.

    Here’s how you can adapt your initial approach:

    Modifications in the Model Definition

    1. Initialization of Positional Encodings: During the model's initialization, define the maximum sequence length for which you would like to prepare your positional encodings.

    2. Modify Forward Pass to Adjust Pos Encodings Dynamically: In your model's forward pass, you will then use a portion of these pre-computed encodings based on the actual sequence length of the input.

    First, adjust the TransformerExpert class to initialize with a max sequence length:

    class TransformerExpert(nn.Module):
        def __init__(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1, max_seq_len=5000):
            # Other initializations...
            self.max_seq_len = max_seq_len
            self.pos_encoder = nn.Parameter(self._positional_encoding(self.max_seq_len, d_model), requires_grad=True)
            # Other parts of __init__
    
        def _positional_encoding(self, seq_len, d_model):
            # Place your positional_encoding logic here but with self args.
            # This will generate a pos encoding matrix for seq_len.
            # Ensure to remove the .unsqueeze(0) at the end from your current implementation
            # code to generate pos goes here...
            return pe
    
        def forward(self, x):
            seq_len = x.size(1)
            pos_encoding = self.pos_encoder[:seq_len, :].unsqueeze(0)
            # Your forward pass using the adapted pos_encoding
    

    Now, adjust the _positional_encoding method to be a static method inside your TransformerExpert class:

    class TransformerExpert(nn.Module):
        @staticmethod
        def _positional_encoding(seq_len, d_model):
            # Your existing positional encoding logic here...
    

    Usage

    With these changes, the TransformerExpert model will be initialized with positional encodings up to max_seq_len, and during each forward pass, it dynamically selects the appropriate slice of the full positional encoding matrix based on the actual length of the inputs.

    Note:

    • Ensure the max_seq_len you choose is greater than or equal to the maximum sequence length you expect to encounter. If longer sequences come up, you'll either need to truncate them or reinitialize the model with a larger max_seq_len.
    • As your positional encodings are learnable parameters, having a large max_seq_len might increase your model’s parameter count significantly. Be cautious of the trade-offs.

    This approach allows your model to be flexible with variable input lengths while still benefiting from learnable positional encodings.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.