Skip to content
The Chat Archive
mistral-medium· Mistral AI Arena Human Preference 55K

this code -- import os import torch from transformers import GPT2Tokenizer, GPT2Model,…

  1. User

    this code -- import os import torch from transformers import GPT2Tokenizer, GPT2Model, GPT2Config, Trainer, TrainingArguments, AutoConfig, TrainerCallback, get_linear_schedule_with_warmup, AdamW from datasets import Dataset, load_dataset from transformers import TrainerCallback

    Cuda or Cpu

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    os.cpu_count() / 2 returns physical core amount

    convert this float into a int for use

    numOfProc = os.cpu_count() / 2 numOfProc = int(numOfProc) userName = os.getlogin()

    Define a class for trainer logging

    class LoggingCallback(TrainerCallback): def init(self, eval_steps): self.eval_steps = eval_steps

    def on_step_end(self, args, state, control, **kwargs):
        if state.global_step % self.eval_steps == 0:
            print(f"Step {state.global_step}:")
            print(f"  Evaluation Loss: {state.log_history['eval_loss'][-1]}")
    
    def on_epoch_end(self, args, state, control, **kwargs):
        print(f"Epoch {state.epoch}:")
        print(f"  Training Loss: {state.log_history['loss'][-1]}")
    
    

    Define a custom Trainer for knowledge distillation

    class KnowledgeDistillationTrainer(Trainer):

    def compute_loss(self, model, inputs, return_outputs=False):
        # print(self.args.teacher)
        # teacher_outputs = self.model.teacher(**inputs)  # Get teacher model outputs
        #teacher_outputs = self.args.teacher(**inputs)  # Get teacher model outputs
        #student_outputs = model(**inputs)  # Get student model outputs
    
        self.projection = torch.nn.Linear(self.model.config.hidden_size, self.args.teacher.config.hidden_size)
        self.projection.to(self.args.device)
    
    
        student_outputs = model(**inputs, return_dict=True)
        teacher_outputs = self.args.teacher(**inputs, return_dict=True)
        #print(student_outputs)
    
        # Get just the logits
        student_logits = student_outputs.last_hidden_state 
        teacher_logits = teacher_outputs.last_hidden_state
    
        # Apply linear projection to student logits
        student_logits = self.projection(student_logits)
    
        print(student_logits.shape)
        print(teacher_logits.shape)
    
        # Convert logits to probabilities by applying softmax
        probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1)
        probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1)
        
        # Calculate the knowledge distillation loss using Kullback-Leibler Divergence
        knowledge_distillation_loss = torch.nn.functional.kl_div(
            torch.log(probabilities_student), probabilities_teacher, reduction='batchmean'
        )
    
        # Standard CrossEntropy loss for the original task
        ce_loss = super().compute_loss(model, inputs, return_outputs=True)
        
        # Combine both losses with a weighted sum
        total_loss = (1 - self.args.alpha) * ce_loss.loss + self.args.alpha * knowledge_distillation_loss
        
        return total_loss if return_outputs else KnowledgeDistillationTrainer._remove_unused(tensor=total_loss)
    

    teacher_model_name = "distilgpt2" # gpt2 gpt2-medium gpt2-large gpt2-xl distilgpt2 displayTeacherConfig = False

    #load custom tokenizer tokenizer = GPT2Tokenizer.from_pretrained('terrycraddock/GPT2-PretrainV1-Tokenizer-en')

    Load the large GPT-2 Teacher model

    print(f"Loading Teacher Model {teacher_model_name}") teacher_model = GPT2Model.from_pretrained(teacher_model_name)

    Display teacher model config to help you decide on student parameters.

    if displayTeacherConfig: teacher_config = AutoConfig.from_pretrained(teacher_model_name) print(teacher_config)

    Show teacher parameter count

    numParams = sum(p.numel() for p in teacher_model.parameters() if p.requires_grad) print(f'Teacher model has {numParams:,} parameters')

    Define the smaller GPT-2 model configuration

    student_model_config = GPT2Config.from_pretrained(teacher_model_name) student_model_config.n_ctx = 768 student_model_config.n_embd = 128 student_model_config.n_head = 2 student_model_config.n_layer = 4

    student_model_config.save_pretrained("student_model") # Save the configuration

    Instantiate a smaller GPT-2 model

    print('Loading student model')

    student_model_config = GPT2Config.from_pretrained("./student_model/config.json")

    student_model = GPT2Model(student_model_config)

    Show teacher parameter count

    numParams = sum(p.numel() for p in student_model.parameters() if p.requires_grad) print(f'Student model has {numParams:,} parameters')

    create root save directory

    print('Creating save paths') isExist = os.path.exists('./models') if not isExist: os.mkdir('./models') isExist = os.path.exists('./models/distilledModel') if not isExist: os.mkdir('./models/distilledModel')

    load dataset

    print(f'Welcome {userName}. Loading the dataset') dsDict = load_dataset("terrycraddock/GPT2-PretrainV1-Tokenized-en", streaming=True)

    print(dsDict["val"][:1])

    Print/Log the content of datasets before passing to the trainer

    print("Train dataset content:")

    print(train_set[:1])

    print("Validation dataset content:")

    print(valid_set[:1])

    Knowledge distillation setup

    training_args = TrainingArguments( output_dir="./models/distilledModel",
    overwrite_output_dir=True, num_train_epochs=3,
    per_device_train_batch_size=2, save_steps=500, save_total_limit=1, )

    Set up the optimizer

    optimizer = torch.optim.AdamW( student_model.parameters(), # Pass the parameters of your model to the optimizer lr=5e-5, # Set the learning rate eps=1e-8 # Set epsilon value to prevent division by zero in AdamW )

    Define the number of training steps

    num_train_steps = training_args.per_device_train_batch_size * training_args.num_train_epochs # len(dsDict["train"])

    Create the learning rate scheduler

    scheduler = get_linear_schedule_with_warmup( optimizer=optimizer, num_warmup_steps=100, # You can adjust the number of warmup steps if needed num_training_steps=num_train_steps )

    Update training_args with the scheduler

    training_args.lr_scheduler_type = 'linear' # Set the scheduler type training_args.learning_rate = 1e-4 # with baby networks can afford to go a bit higher 5e-5 training_args.max_steps = num_train_steps training_args.scheduler = scheduler training_args.optimizer = optimizer

    Update training_args with teacher model

    training_args.teacher = teacher_model

    Update training_args with temp

    training_args.temperature = 1.0

    Update training_args with alpha

    training_args.alpha = 0.5

    Define the LoggingCallback

    logging_callback = LoggingCallback(eval_steps=500)

    Define trainer for knowledge distillation with the custom Trainer class

    trainer = KnowledgeDistillationTrainer( model=student_model, args=training_args, train_dataset=dsDict["train"], eval_dataset=dsDict["val"], tokenizer=tokenizer, callbacks=[logging_callback], )

    Load models to device

    student_model = student_model.to(device) teacher_model = teacher_model.to(device)

    Start knowledge distillation training

    trainer.train()

    Save the distilled smaller model

    student_model.save_pretrained("./models/distilledModel") this error -- Traceback (most recent call last): File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 191, in trainer.train() File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1537, in train return inner_training_loop( ^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1854, in _inner_training_loop tr_loss_step = self.training_step(model, inputs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 2735, in training_step loss = self.compute_loss(model, inputs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 69, in compute_loss ce_loss = super().compute_loss(model, inputs, return_outputs=True) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 2776, in compute_loss raise ValueError( ValueError: The model did not return a loss from the inputs, only the following keys: last_hidden_state,past_key_values. For reference, the inputs it received are input_ids,attention_mask.

  2. Assistant

    The error message indicates that the compute_loss method in the KnowledgeDistillationTrainer class is trying to call super().compute_loss(model, inputs, return_outputs=True), but the model is not returning a loss from the inputs. Instead, it is returning only the last_hidden_state and past_key_values. The error suggests that the Trainer class expects a loss to be returned from the model, but it is not receiving one.

    To fix this issue, you can modify the compute_loss method in the KnowledgeDistillationTrainer class to handle the case where the model does not return a loss. Here is an example of how you can modify the method:

    class KnowledgeDistillationTrainer(Trainer):
        def compute_loss(self, model, inputs, return_outputs=False):
            self.projection = torch.nn.Linear(self.model.config.hidden_size, self.args.teacher.config.hidden_size)
            self.projection.to(self.args.device)
    
            student_outputs = model(**inputs, return_dict=True)
            teacher_outputs = self.args.teacher(**inputs, return_dict=True)
    
            student_logits = student_outputs.last_hidden_state
            teacher_logits = teacher_outputs.last_hidden_state
    
            student_logits = self.projection(student_logits)
    
            probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1)
            probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1)
    
            knowledge_distillation_loss = torch.nn.functional.kl_div(
                torch.log(probabilities_student), probabilities_teacher, reduction='batchmean'
            )
    
            ce_loss = None
            if return_outputs and 'loss' in student_outputs:
                ce_loss = student_outputs['loss']
    
            total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss if ce_loss is not None else knowledge_distillation_loss
    
            return total_loss if return_outputs else KnowledgeDistillationTrainer._remove_unused(tensor=total_loss)
    

    In this modified version of the compute_loss method, we first check if the student_outputs dictionary contains a 'loss' key. If it does, we set ce_loss to the value of 'loss' in student_outputs. If it does not, we set ce_loss to None. Then, we modify the calculation of the total loss to use ce_loss only if it is not None. If ce_loss is None, we use only the knowledge distillation loss. This should ensure that the compute_loss method always returns a loss.

  3. User

    Epoch 1.0: Traceback (most recent call last): File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 226, in trainer.train() File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1537, in train return inner_training_loop( ^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1928, in _inner_training_loop self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer_callback.py", line 380, in on_epoch_end return self.call_event("on_epoch_end", args, state, control) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer_callback.py", line 414, in call_event result = getattr(callback, event)( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 29, in on_epoch_end print(f" Training Loss: {state.log_history['loss'][-1]}") ~~~~~~~~~~~~~~~~~^^^^^^^^ TypeError: list indices must be integers or slices, not str

  4. Assistant

    The error is occurring because state.log_history is a list of dictionaries, and you are trying to access the 'loss' key as if it were a dictionary.

    To fix this error, you can modify the on_epoch_end method in the LoggingCallback class to access the 'loss' key from the last dictionary in the state.log_history list. Here's an updated implementation:

    class LoggingCallback(TrainerCallback):
        def __init__(self, eval_steps):
            self.eval_steps = eval_steps
    
        def on_step_end(self, args, state, control, **kwargs):
            if state.global_step % self.eval_steps == 0:
                print(f"Step {state.global_step}:")
                print(f"  Evaluation Loss: {state.log_history[-1]['eval_loss']}")
    
        def on_epoch_end(self, args, state, control, **kwargs):
            print(f"Epoch {state.epoch}:")
            print(f"  Training Loss: {state.log_history[-1]['loss']}")
    

    Note that we're using state.log_history[-1] to access the last dictionary in the list, and then accessing the 'loss' key from that dictionary. This should fix the error.

  5. User

    I think the problem is that we are not logging -- class KnowledgeDistillationTrainer(Trainer): def compute_loss(self, model, inputs, return_outputs=False): # Teacher and student model outputs student_outputs = model(**inputs, return_dict=True) teacher_outputs = self.args.teacher(**inputs, return_dict=True)

        # Get just the logits
        student_logits = student_outputs.last_hidden_state 
        teacher_logits = teacher_outputs.last_hidden_state
    
        # Apply linear projection to student logits
        projection = torch.nn.Linear(student_logits.size(-1), teacher_logits.size(-1)).to(self.args.device)
        student_logits = projection(student_logits)
    
        # Convert logits to probabilities by applying softmax
        probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1)
        probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1)
        
        # Calculate the knowledge distillation loss using Kullback-Leibler Divergence
        knowledge_distillation_loss = torch.nn.functional.kl_div(
            torch.log(probabilities_student), probabilities_teacher, reduction='batchmean'
        )
    
        # Compute CrossEntropy loss manually
        if 'labels' in inputs:
            # Shift the labels to the right and add -100 to the padding tokens
            shift_logits = student_logits[..., :-1, :].contiguous()
            shift_labels = inputs['labels'][..., 1:].contiguous()
            loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100)
            ce_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
        else:
            ce_loss = 0
    
        # Combine both losses with a weighted sum
        total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss
        
        return (total_loss, student_outputs) if return_outputs else total_loss
    
  6. Assistant

    Yes, you're right that the compute_loss method in the KnowledgeDistillationTrainer class is not logging the loss. One way to fix this is to override the training_step method in the same class and manually log the losses.

    Here's an updated implementation of KnowledgeDistillationTrainer that includes logging:

    class KnowledgeDistillationTrainer(Trainer):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.train_loss_logger = []
    
        def compute_loss(self, model, inputs, return_outputs=False):
            # Teacher and student model outputs
            student_outputs = model(**inputs, return_dict=True)
            teacher_outputs = self.args.teacher(**inputs, return_dict=True)
    
            # Get just the logits
            student_logits = student_outputs.last_hidden_state 
            teacher_logits = teacher_outputs.last_hidden_state
    
            # Apply linear projection to student logits
            projection = torch.nn.Linear(student_logits.size(-1), teacher_logits.size(-1)).to(self.args.device)
            student_logits = projection(student_logits)
    
            # Convert logits to probabilities by applying softmax
            probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1)
            probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1)
            
            # Calculate the knowledge distillation loss using Kullback-Leibler Divergence
            knowledge_distillation_loss = torch.nn.functional.kl_div(
                torch.log(probabilities_student), probabilities_teacher, reduction='batchmean'
            )
    
            # Compute CrossEntropy loss manually
            if 'labels' in inputs:
                # Shift the labels to the right and add -100 to the padding tokens
                shift_logits = student_logits[..., :-1, :].contiguous()
                shift_labels = inputs['labels'][..., 1:].contiguous()
                loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100)
                ce_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
            else:
                ce_loss = 0
    
            # Combine both losses with a weighted sum
            total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss
            
            return (total_loss, student_outputs) if return_outputs else total_loss
    
        def training_step(self, model, inputs):
            loss, outputs = self.compute_loss(model, inputs, return_outputs=True)
            self.log('train_loss', loss, prog_bar=True)
            self.train_loss_logger.append(loss.item())
            return loss
    

    Here, we've overridden the training_step method to call compute_loss with return_outputs=True, and then log the returned loss using the log method. We also append the loss to a list train_loss_logger for later use.

    Note that you may want to modify the compute_loss method to return the CE loss and knowledge distillation loss separately, so that you can log them separately as well. For example:

    def compute_loss(self, model, inputs, return_outputs=False):
        # ...
        ce_loss = ...
        kd_loss = ...
        total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * kd_loss
        return total_loss, ce_loss, kd_loss
    
    def training_step(self, model, inputs):
        loss, ce_loss, kd_loss = self.compute_loss(model, inputs)
        self.log('train_loss', loss, prog_bar=True)
    

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.