this code -- import os import torch from transformers import GPT2Tokenizer, GPT2Model,…
- User
this code -- import os import torch from transformers import GPT2Tokenizer, GPT2Model, GPT2Config, Trainer, TrainingArguments, AutoConfig, TrainerCallback, get_linear_schedule_with_warmup, AdamW from datasets import Dataset, load_dataset from transformers import TrainerCallback
Cuda or Cpu
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
os.cpu_count() / 2 returns physical core amount
convert this float into a int for use
numOfProc = os.cpu_count() / 2 numOfProc = int(numOfProc) userName = os.getlogin()
Define a class for trainer logging
class LoggingCallback(TrainerCallback): def init(self, eval_steps): self.eval_steps = eval_steps
def on_step_end(self, args, state, control, **kwargs): if state.global_step % self.eval_steps == 0: print(f"Step {state.global_step}:") print(f" Evaluation Loss: {state.log_history['eval_loss'][-1]}") def on_epoch_end(self, args, state, control, **kwargs): print(f"Epoch {state.epoch}:") print(f" Training Loss: {state.log_history['loss'][-1]}")Define a custom Trainer for knowledge distillation
class KnowledgeDistillationTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False): # print(self.args.teacher) # teacher_outputs = self.model.teacher(**inputs) # Get teacher model outputs #teacher_outputs = self.args.teacher(**inputs) # Get teacher model outputs #student_outputs = model(**inputs) # Get student model outputs self.projection = torch.nn.Linear(self.model.config.hidden_size, self.args.teacher.config.hidden_size) self.projection.to(self.args.device) student_outputs = model(**inputs, return_dict=True) teacher_outputs = self.args.teacher(**inputs, return_dict=True) #print(student_outputs) # Get just the logits student_logits = student_outputs.last_hidden_state teacher_logits = teacher_outputs.last_hidden_state # Apply linear projection to student logits student_logits = self.projection(student_logits) print(student_logits.shape) print(teacher_logits.shape) # Convert logits to probabilities by applying softmax probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1) probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1) # Calculate the knowledge distillation loss using Kullback-Leibler Divergence knowledge_distillation_loss = torch.nn.functional.kl_div( torch.log(probabilities_student), probabilities_teacher, reduction='batchmean' ) # Standard CrossEntropy loss for the original task ce_loss = super().compute_loss(model, inputs, return_outputs=True) # Combine both losses with a weighted sum total_loss = (1 - self.args.alpha) * ce_loss.loss + self.args.alpha * knowledge_distillation_loss return total_loss if return_outputs else KnowledgeDistillationTrainer._remove_unused(tensor=total_loss)teacher_model_name = "distilgpt2" # gpt2 gpt2-medium gpt2-large gpt2-xl distilgpt2 displayTeacherConfig = False
#load custom tokenizer tokenizer = GPT2Tokenizer.from_pretrained('terrycraddock/GPT2-PretrainV1-Tokenizer-en')
Load the large GPT-2 Teacher model
print(f"Loading Teacher Model {teacher_model_name}") teacher_model = GPT2Model.from_pretrained(teacher_model_name)
Display teacher model config to help you decide on student parameters.
if displayTeacherConfig: teacher_config = AutoConfig.from_pretrained(teacher_model_name) print(teacher_config)
Show teacher parameter count
numParams = sum(p.numel() for p in teacher_model.parameters() if p.requires_grad) print(f'Teacher model has {numParams:,} parameters')
Define the smaller GPT-2 model configuration
student_model_config = GPT2Config.from_pretrained(teacher_model_name) student_model_config.n_ctx = 768 student_model_config.n_embd = 128 student_model_config.n_head = 2 student_model_config.n_layer = 4
student_model_config.save_pretrained("student_model") # Save the configuration
Instantiate a smaller GPT-2 model
print('Loading student model')
student_model_config = GPT2Config.from_pretrained("./student_model/config.json")
student_model = GPT2Model(student_model_config)
Show teacher parameter count
numParams = sum(p.numel() for p in student_model.parameters() if p.requires_grad) print(f'Student model has {numParams:,} parameters')
create root save directory
print('Creating save paths') isExist = os.path.exists('./models') if not isExist: os.mkdir('./models') isExist = os.path.exists('./models/distilledModel') if not isExist: os.mkdir('./models/distilledModel')
load dataset
print(f'Welcome {userName}. Loading the dataset') dsDict = load_dataset("terrycraddock/GPT2-PretrainV1-Tokenized-en", streaming=True)
print(dsDict["val"][:1])
Print/Log the content of datasets before passing to the trainer
print("Train dataset content:")
print(train_set[:1])
print("Validation dataset content:")
print(valid_set[:1])
Knowledge distillation setup
training_args = TrainingArguments( output_dir="./models/distilledModel",
overwrite_output_dir=True, num_train_epochs=3,
per_device_train_batch_size=2, save_steps=500, save_total_limit=1, )Set up the optimizer
optimizer = torch.optim.AdamW( student_model.parameters(), # Pass the parameters of your model to the optimizer lr=5e-5, # Set the learning rate eps=1e-8 # Set epsilon value to prevent division by zero in AdamW )
Define the number of training steps
num_train_steps = training_args.per_device_train_batch_size * training_args.num_train_epochs # len(dsDict["train"])
Create the learning rate scheduler
scheduler = get_linear_schedule_with_warmup( optimizer=optimizer, num_warmup_steps=100, # You can adjust the number of warmup steps if needed num_training_steps=num_train_steps )
Update training_args with the scheduler
training_args.lr_scheduler_type = 'linear' # Set the scheduler type training_args.learning_rate = 1e-4 # with baby networks can afford to go a bit higher 5e-5 training_args.max_steps = num_train_steps training_args.scheduler = scheduler training_args.optimizer = optimizer
Update training_args with teacher model
training_args.teacher = teacher_model
Update training_args with temp
training_args.temperature = 1.0
Update training_args with alpha
training_args.alpha = 0.5
Define the LoggingCallback
logging_callback = LoggingCallback(eval_steps=500)
Define trainer for knowledge distillation with the custom Trainer class
trainer = KnowledgeDistillationTrainer( model=student_model, args=training_args, train_dataset=dsDict["train"], eval_dataset=dsDict["val"], tokenizer=tokenizer, callbacks=[logging_callback], )
Load models to device
student_model = student_model.to(device) teacher_model = teacher_model.to(device)
Start knowledge distillation training
trainer.train()
Save the distilled smaller model
student_model.save_pretrained("./models/distilledModel") this error -- Traceback (most recent call last): File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 191, in trainer.train() File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1537, in train return inner_training_loop( ^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1854, in _inner_training_loop tr_loss_step = self.training_step(model, inputs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 2735, in training_step loss = self.compute_loss(model, inputs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 69, in compute_loss ce_loss = super().compute_loss(model, inputs, return_outputs=True) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 2776, in compute_loss raise ValueError( ValueError: The model did not return a loss from the inputs, only the following keys: last_hidden_state,past_key_values. For reference, the inputs it received are input_ids,attention_mask.
- Assistant
The error message indicates that the
compute_lossmethod in theKnowledgeDistillationTrainerclass is trying to callsuper().compute_loss(model, inputs, return_outputs=True), but the model is not returning a loss from the inputs. Instead, it is returning only thelast_hidden_stateandpast_key_values. The error suggests that theTrainerclass expects a loss to be returned from the model, but it is not receiving one.To fix this issue, you can modify the
compute_lossmethod in theKnowledgeDistillationTrainerclass to handle the case where the model does not return a loss. Here is an example of how you can modify the method:class KnowledgeDistillationTrainer(Trainer): def compute_loss(self, model, inputs, return_outputs=False): self.projection = torch.nn.Linear(self.model.config.hidden_size, self.args.teacher.config.hidden_size) self.projection.to(self.args.device) student_outputs = model(**inputs, return_dict=True) teacher_outputs = self.args.teacher(**inputs, return_dict=True) student_logits = student_outputs.last_hidden_state teacher_logits = teacher_outputs.last_hidden_state student_logits = self.projection(student_logits) probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1) probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1) knowledge_distillation_loss = torch.nn.functional.kl_div( torch.log(probabilities_student), probabilities_teacher, reduction='batchmean' ) ce_loss = None if return_outputs and 'loss' in student_outputs: ce_loss = student_outputs['loss'] total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss if ce_loss is not None else knowledge_distillation_loss return total_loss if return_outputs else KnowledgeDistillationTrainer._remove_unused(tensor=total_loss)In this modified version of the
compute_lossmethod, we first check if thestudent_outputsdictionary contains a'loss'key. If it does, we setce_lossto the value of'loss'instudent_outputs. If it does not, we setce_losstoNone. Then, we modify the calculation of the total loss to usece_lossonly if it is notNone. Ifce_lossisNone, we use only the knowledge distillation loss. This should ensure that thecompute_lossmethod always returns a loss. - User
Epoch 1.0: Traceback (most recent call last): File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 226, in trainer.train() File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1537, in train return inner_training_loop( ^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer.py", line 1928, in _inner_training_loop self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer_callback.py", line 380, in on_epoch_end return self.call_event("on_epoch_end", args, state, control) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/anaconda3/lib/python3.11/site-packages/transformers/trainer_callback.py", line 414, in call_event result = getattr(callback, event)( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/terry/WrkSpaces/LLM/CustomGPTHF/distillGPT2.py", line 29, in on_epoch_end print(f" Training Loss: {state.log_history['loss'][-1]}") ~~~~~~~~~~~~~~~~~^^^^^^^^ TypeError: list indices must be integers or slices, not str
- Assistant
The error is occurring because
state.log_historyis a list of dictionaries, and you are trying to access the'loss'key as if it were a dictionary.To fix this error, you can modify the
on_epoch_endmethod in theLoggingCallbackclass to access the'loss'key from the last dictionary in thestate.log_historylist. Here's an updated implementation:class LoggingCallback(TrainerCallback): def __init__(self, eval_steps): self.eval_steps = eval_steps def on_step_end(self, args, state, control, **kwargs): if state.global_step % self.eval_steps == 0: print(f"Step {state.global_step}:") print(f" Evaluation Loss: {state.log_history[-1]['eval_loss']}") def on_epoch_end(self, args, state, control, **kwargs): print(f"Epoch {state.epoch}:") print(f" Training Loss: {state.log_history[-1]['loss']}")Note that we're using
state.log_history[-1]to access the last dictionary in the list, and then accessing the'loss'key from that dictionary. This should fix the error. - User
I think the problem is that we are not logging -- class KnowledgeDistillationTrainer(Trainer): def compute_loss(self, model, inputs, return_outputs=False): # Teacher and student model outputs student_outputs = model(**inputs, return_dict=True) teacher_outputs = self.args.teacher(**inputs, return_dict=True)
# Get just the logits student_logits = student_outputs.last_hidden_state teacher_logits = teacher_outputs.last_hidden_state # Apply linear projection to student logits projection = torch.nn.Linear(student_logits.size(-1), teacher_logits.size(-1)).to(self.args.device) student_logits = projection(student_logits) # Convert logits to probabilities by applying softmax probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1) probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1) # Calculate the knowledge distillation loss using Kullback-Leibler Divergence knowledge_distillation_loss = torch.nn.functional.kl_div( torch.log(probabilities_student), probabilities_teacher, reduction='batchmean' ) # Compute CrossEntropy loss manually if 'labels' in inputs: # Shift the labels to the right and add -100 to the padding tokens shift_logits = student_logits[..., :-1, :].contiguous() shift_labels = inputs['labels'][..., 1:].contiguous() loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100) ce_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) else: ce_loss = 0 # Combine both losses with a weighted sum total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss return (total_loss, student_outputs) if return_outputs else total_loss - Assistant
Yes, you're right that the
compute_lossmethod in theKnowledgeDistillationTrainerclass is not logging the loss. One way to fix this is to override thetraining_stepmethod in the same class and manually log the losses.Here's an updated implementation of
KnowledgeDistillationTrainerthat includes logging:class KnowledgeDistillationTrainer(Trainer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.train_loss_logger = [] def compute_loss(self, model, inputs, return_outputs=False): # Teacher and student model outputs student_outputs = model(**inputs, return_dict=True) teacher_outputs = self.args.teacher(**inputs, return_dict=True) # Get just the logits student_logits = student_outputs.last_hidden_state teacher_logits = teacher_outputs.last_hidden_state # Apply linear projection to student logits projection = torch.nn.Linear(student_logits.size(-1), teacher_logits.size(-1)).to(self.args.device) student_logits = projection(student_logits) # Convert logits to probabilities by applying softmax probabilities_student = torch.nn.functional.softmax(student_logits / self.args.temperature, dim=-1) probabilities_teacher = torch.nn.functional.softmax(teacher_logits / self.args.temperature, dim=-1) # Calculate the knowledge distillation loss using Kullback-Leibler Divergence knowledge_distillation_loss = torch.nn.functional.kl_div( torch.log(probabilities_student), probabilities_teacher, reduction='batchmean' ) # Compute CrossEntropy loss manually if 'labels' in inputs: # Shift the labels to the right and add -100 to the padding tokens shift_logits = student_logits[..., :-1, :].contiguous() shift_labels = inputs['labels'][..., 1:].contiguous() loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100) ce_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) else: ce_loss = 0 # Combine both losses with a weighted sum total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * knowledge_distillation_loss return (total_loss, student_outputs) if return_outputs else total_loss def training_step(self, model, inputs): loss, outputs = self.compute_loss(model, inputs, return_outputs=True) self.log('train_loss', loss, prog_bar=True) self.train_loss_logger.append(loss.item()) return lossHere, we've overridden the
training_stepmethod to callcompute_losswithreturn_outputs=True, and then log the returned loss using thelogmethod. We also append the loss to a listtrain_loss_loggerfor later use.Note that you may want to modify the
compute_lossmethod to return the CE loss and knowledge distillation loss separately, so that you can log them separately as well. For example:def compute_loss(self, model, inputs, return_outputs=False): # ... ce_loss = ... kd_loss = ... total_loss = (1 - self.args.alpha) * ce_loss + self.args.alpha * kd_loss return total_loss, ce_loss, kd_loss def training_step(self, model, inputs): loss, ce_loss, kd_loss = self.compute_loss(model, inputs) self.log('train_loss', loss, prog_bar=True)
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.