Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· openaiHugging Face Suzhen/CodeChat

import torch import torch.nn as nn import torch.nn.functional as F from…

  1. User

    import torch import torch.nn as nn import torch.nn.functional as F from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader, Dataset from collections import Counter from tqdm import tqdm

    Expert LSTM Model

    class LSTMExpert(nn.Module): def init(self, input_size, hidden_size, output_size): super(LSTMExpert, self).init() self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True) self.fc = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        h, _ = self.lstm(x)
        # Shape of h: [batch_size, sequence_length, hidden_size]
        # No longer taking the last sequence output, transform for all time steps
        h = h.reshape(-1, h.shape[2])  # Flatten to two dimensions: [batch_size * sequence_length, hidden_size]
        return self.fc(h)
    

    Gating Network

    class GatingNetwork(nn.Module): def init(self, input_size, num_experts): super(GatingNetwork, self).init() self.fc = nn.Linear(input_size, num_experts) self.softmax = nn.Softmax(dim=1)

    def forward(self, x):
        if x.dim() > 2:
            x = x.view(x.size(0), -1)  # Flatten the tensor correctly
        x = self.fc(x)
        return self.softmax(x)
    

    Mixture of Experts Model

    class MixtureOfExperts(nn.Module): def init(self, input_size, hidden_size, output_size, num_experts): super(MixtureOfExperts, self).init() self.num_experts = num_experts self.experts = nn.ModuleList([LSTMExpert(input_size, hidden_size, output_size) for _ in range(num_experts)]) self.gating_network = GatingNetwork(input_size, num_experts)

    def forward(self, x):
        # Flatten x for gating input: [batch_size * sequence_length, input_size]
        gating_input = x.contiguous().view(-1, x.shape[-1])
        # Get gating scores for each expert: [batch_size * sequence_length, num_experts, 1]
        gating_scores = self.gating_network(gating_input).view(-1, self.num_experts, 1)
    
        # Compute outputs from each expert
        expert_outputs = [expert(x) for expert in self.experts]  # List of [batch_size * sequence_length, vocab_size] tensors
        # Stack along a new dimension for experts: [batch_size * sequence_length, vocab_size, num_experts]
        expert_outputs = torch.stack(expert_outputs, dim=2)
    
        # Calculate weighted sum of expert outputs: [batch_size * sequence_length, vocab_size]
        mixed_output = torch.bmm(expert_outputs, gating_scores)  # [batch_size * sequence_length, vocab_size, 1]
        mixed_output = mixed_output.squeeze(-1)  # Remove last singleton dimension
    
        return mixed_output
    

    class TextDataset(Dataset): def init(self, path, seq_len): self.seq_len = seq_len with open(path, "r", encoding="utf-8") as f: text = f.read() words = text.split() self.vocab, self.idx2token = self.build_vocab(words) self.tokens = [self.vocab.get(w, 0) for w in words] # Default to 0 for unknown tokens

    def build_vocab(self, words):
        counts = Counter(words)
        vocab = {word: i for i, (word, _) in enumerate(counts.most_common(), 1)}
        vocab["<pad>"] = 0
        idx2token = {i: word for word, i in vocab.items()}
        return vocab, idx2token
        
    def __len__(self):
        return len(self.tokens) // self.seq_len
    
    def __getitem__(self, idx):
        chunk = self.tokens[idx * self.seq_len: (idx + 1) * self.seq_len]
        x = chunk[:-1]
        y = chunk[1:]
        return torch.tensor(x), torch.tensor(y)
    

    def collate_fn(batch): inputs, targets = zip(*batch) inputs = pad_sequence(inputs, batch_first=True, padding_value=0) targets = pad_sequence(targets, batch_first=True, padding_value=0) return inputs, targets

    Set the path to your text file and define sequence length

    path_to_text = 'C:/Users/Dell-PC/Desktop/Projets/2-The-Beyond\Training-data/Summaries.txt' # replace with the path to your text file seq_len = 128 # sequence length

    Create a dataset and data loader

    dataset = TextDataset(path_to_text, seq_len) data_loader = DataLoader(dataset, batch_size=96, shuffle=True, collate_fn=collate_fn)

    def train_model_TXT(model, criterion, optimizer, num_epochs, data_loader): model.train() for epoch in range(num_epochs): total_loss = 0 progress_bar = tqdm(data_loader, desc=f"Epoch {epoch+1}/{num_epochs}")

        for inputs, targets in progress_bar:
            optimizer.zero_grad()
    
            # Forward pass
            predictions = model(inputs)
    
            # Flatten targets to be [batch_size * sequence_length]
            targets = targets.view(-1)
    
            # Flatten predictions to be [batch_size * sequence_length, num_classes]
            predictions = predictions.view(-1, predictions.size(-1))
    
            # Loss computation
            loss = criterion(predictions, targets)
            loss.backward()
            optimizer.step()
    
            total_loss += loss.item()
    
            # Update the progress bar with the loss information
            progress_bar.set_postfix(loss=loss.item())
    
        average_loss = total_loss / len(data_loader)
        print(f"Epoch {epoch+1}, Avg Loss: {average_loss:.4f}")
    

    def generate_text(model, dataset, seed_text, num_generate, temperature=1.0): model.eval() # Put the model in evaluation mode

    # List to store the generated tokens
    generated_tokens = []
    
    # Initial sequence (prefix) to start the generation process
    input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()]  # Convert to token IDs
    current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0)
    
    # Generate num_generate tokens
    for _ in range(num_generate):
        # Forward pass through the model
        with torch.no_grad():
            output = model(current_sequence)
        
        # Get probabilities, apply temperature scaling, and sample from the distribution
        probabilities = F.softmax(output[-1] / temperature, dim=0).detach()
        next_token_idx = torch.multinomial(probabilities, 1).item()
    
        # Append token to the current sequence and to the generated tokens
        generated_tokens.append(next_token_idx)
        current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1)
    
    # Convert tokens to words
    generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens])  # Use .get() to provide a default value for missing keys
    return generated_text
    

    Integration of MoE with the rest of the code

    vocab_size = len(dataset.vocab) # The size of the vocabulary embedding_dim = 144 # Size of embeddings hidden_size = 576 # Number of features in the hidden state of the LSTM num_experts = 2 # Define the number of LSTM experts you want in your MoE model

    Instantiate the MixtureOfExperts model

    mixture_of_experts_model = MixtureOfExperts(embedding_dim, hidden_size, vocab_size, num_experts)

    Wrap the experts and the embedding layer into a new combined model, replacing SingleLSTM_TXT

    class MoEModel(nn.Module): def init(self, embedding_dim, vocab_size, MoE): super(MoEModel, self).init() self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim) self.MoE = MoE

    def forward(self, x):
        embedded = self.embedding(x)
        output = self.MoE(embedded)
        return output
    

    Instantiate the MoEModel with embeddings and mixture of experts

    moe_model = MoEModel(embedding_dim, vocab_size, mixture_of_experts_model)

    def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad)

    Example usage with your model:

    total_params = count_parameters(moe_model) print(f"Total trainable parameters: {total_params}")

    Training parameters

    num_epochs = 10 batch_size = 96 learning_rate = 1e-2

    Define Loss Function and Optimizer for MoE model

    criterion = nn.CrossEntropyLoss() # Use CrossEntropyLoss for classification tasks optimizer = torch.optim.Adam(moe_model.parameters(), lr=learning_rate)

    Replace references to single_lstm_txt_model with moe_model

    Train the model with the text data

    train_model_TXT(moe_model, criterion, optimizer, num_epochs, data_loader)

    Start a loop for the interactive chat-like text generation

    while True: try: # Get user input seed_text = input("Enter seed text (type 'quit' to stop): ")

        # Check if user wants to quit the interaction
        if seed_text.lower() == "quit":
            print("Exiting text generation chat.")
            break
        
        # User input is not empty and not “quit”, generate text
        if seed_text.strip():
            num_generate = 16  # Number of words to generate
            temperature = 0.1  # Sampling temperature, higher will increase diversity
            
            # Use the trained model to generate text
            generated_text = generate_text(moe_model, dataset, seed_text, num_generate, temperature)
            print("Generated Text:", generated_text)
        else:
            print("Seed text cannot be empty.")
    except KeyboardInterrupt:
        # Handle KeyboardInterrupt (Ctrl+C) to gracefully exit
        print("\nExiting text generation chat.")
        break
    
  2. Assistant

    Your script consists of several parts that define a Mixture of Experts with LSTM experts for text generation. The core components of your script are as follows:

    1. LSTMExpert: A class that defines an LSTM model to be used as an expert in the mixture of experts model. It consists of an LSTM layer followed by a fully connected layer.

    2. GatingNetwork: A neural network that takes the input sequence and computes the softmax weights for each of the LSTM experts.

    3. MixtureOfExperts: An ensemble model that incorporates multiple LSTMExpert models and a GatingNetwork. It computes the final output by combining the outputs of each expert weighted by the gating network's output.

    4. TextDataset: A PyTorch Dataset class for tokenizing the text data and serving it in chunks suitable for LSTM training.

    5. MoEModel: A wrapper model that includes an embedding layer and the MixtureOfExperts model. This will be trainable end-to-end.

    The script also includes training and text generation functions:

    • train_model_TXT: Function for training the model using the provided dataset and data loader. It utilizes a loss function and an optimizer.

    • generate_text: Function for generating text with the trained model, taking into account a starting seed text, sequence length, and sampling temperature.

    Finally, the script sets up the training process, instantiates a model object, and enters an interactive loop allowing a user to input seed text and receive generated text.

    Here are a few things to watch out for in the script:

    1. Ensure that the paths and parameters such as sequence length are set appropriately for your use case.

    2. Verify that the dataset and tokenization process match your needs and that unknown tokens are handled correctly.

    3. The generate_text function makes use of the temperature parameter when sampling. This can affect the diversity of the generated text, and you may need to adjust this to suit your preferences.

    4. It is good practice to use torch.no_grad() during inference (text generation) to disable gradient calculations and reduce memory consumption.

    5. The script uses a simple greedy decoding strategy. For more advanced or nuanced text generation, you could consider using other techniques such as beam search.

    6. It's important to handle cases where the sequence length of the input text doesn't match your model's expectations, which may occur in generate_text.

    Overall, the script is well-organized and appears to cover the necessary components and steps for training a Mixture of Experts model on a text dataset and using it to generate new text sequences based on user-provided seed text. Remember to monitor the training process and evaluate generated text quality to adjust hyperparameters and potentially improve model performance.

  3. User

    Help me change how datasets are loaded and what the model is trained on, instead of loading a txt dataset, we'll load a jsonl dataset, with two columns, Document, which are supposed to be the inputs of the model, and Summary, which is what the model should be able to predict, please modify all the necessary part of the code to affect the change

  4. Assistant

    To adapt your code to load a JSON Lines (jsonl) dataset with "Document" and "Summary" columns, you will need to modify the TextDataset class and any part of the code that references the dataset format. Here's how to proceed:

    1. Modify the TextDataset class to load the JSONL file and handle the "Document" and "Summary" pairs.
    2. Update the collate function to handle variable-length input and output sequences, and use pad_sequence as needed.
    3. Modify the training function so it works with the modified dataset and collate function.

    Here are the changes applied to the TextDataset, collate_fn, and train_model_TXT code blocks:

    import json
    
    class SummaryDataset(Dataset):
        def __init__(self, path):
            with open(path, "r", encoding="utf-8") as f:
                self.data = [json.loads(line) for line in f]
            self.vocab, self.idx2token = self.build_vocab([d["Document"] + " " + d["Summary"] for d in self.data])
            
        def build_vocab(self, docs):
            words = [word for doc in docs for word in doc.split()]
            counts = Counter(words)
            vocab = {word: i for i, (word, _) in enumerate(counts.most_common(), 1)}
            vocab["<pad>"] = 0
            idx2token = {i: word for word, i in vocab.items()}
            return vocab, idx2token
    
        def __len__(self):
            return len(self.data)
    
        def __getitem__(self, idx):
            document = self.data[idx]["Document"]
            summary = self.data[idx]["Summary"]
            document_tokens = [self.vocab.get(w, 0) for w in document.split()]
            summary_tokens = [self.vocab.get(w, 0) for w in summary.split()]
            return torch.tensor(document_tokens), torch.tensor(summary_tokens)
        
    def collate_fn(batch):
        documents, summaries = zip(*batch)
        documents_padded = pad_sequence(documents, batch_first=True, padding_value=0)
        summaries_padded = pad_sequence(summaries, batch_first=True, padding_value=0)
        return documents_padded, summaries_padded
    
    # Modify the training function if needed to accommodate the new data structure
    def train_model_TXT(model, criterion, optimizer, num_epochs, data_loader):
        # ... (rest of the training code remains the same)
    
    # Load the dataset (path_to_jsonl should be the path to your jsonl file)
    path_to_jsonl = 'path_to_your_jsonl_file.jsonl'
    dataset = SummaryDataset(path_to_jsonl)
    data_loader = DataLoader(dataset, batch_size=96, shuffle=True, collate_fn=collate_fn)
    
    # Now you can train your model with the new dataset and dataloader
    

    These changes will load a JSONL file where each line contains a JSON object with the keys "Document" and "Summary." The SummaryDataset class tokenizes both parts, builds a vocabulary, and serves them as input and target sequences for the model.

    Remember that you need to instantiate your model, loss function, and optimizer, as well as call the train_model_TXT function with these parameters. The modification assumes that the model expects the input sequence and predicts the output sequence (the summary) directly, without the use of an encoder-decoder architecture like in some sequence-to-sequence models.

    If you need to transform the inputs and outputs to a different representation (e.g., sequences of embeddings or encoded states), you would need to adapt the model definition and forward pass correspondingly.

    Also, don't forget to adjust paths, batch sizes, and any other parameters according to your specific requirements and the characteristics of the new dataset.