Provide the validation loop needed in this code and fixes for any possible issues that…
- User
Provide the validation loop needed in this code and fixes for any possible issues that you see:
import os import yaml import torch import torch.nn as nn from torch.utils.data import DataLoader from transformers import AutoTokenizer, get_cosine_schedule_with_warmup, AdamW from transformers import AutoModelForCausalLM import torch.distributed as dist from torch.utils.data.distributed import DistributedSampler from tqdm import tqdm import wandb from datasets import load_from_disk
def main(): # Initialize distributed training dist.init_process_group('nccl') local_rank = int(os.environ["LOCAL_RANK"]) torch.cuda.set_device(local_rank) device = torch.device("cuda", local_rank)
# Load configuration with open('configs/text_training_config.yaml') as f: config = yaml.safe_load(f) # Initialize W&B if local_rank == 0: wandb.init(project='multi-modal-llm', name='llama3-2-1b-oasst1-finetune', config=config) # Load tokenizer and model tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-3.2-1B') model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-3.2-1B') if tokenizer.pad_token is None: tokenizer.add_special_tokens({'pad_token': '[PAD]'}) model.resize_token_embeddings(len(tokenizer)) model = model.to(device) model = nn.parallel.DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank) # Load dataset dataset = load_from_disk('data/processed/oasst1_tokenized') train_dataset = dataset['train'] val_dataset = dataset['validation'] # Create DataLoaders train_sampler = DistributedSampler(train_dataset) val_sampler = DistributedSampler(val_dataset) train_loader = DataLoader(train_dataset, batch_size=config['batch_size'], sampler=train_sampler) val_loader = DataLoader(val_dataset, batch_size=config['batch_size'], sampler=val_sampler) # Define optimizer and scheduler optimizer = AdamW(model.parameters(), lr=config['learning_rate']) total_steps = len(train_loader) * config['num_epochs'] scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=config['warmup_steps'], num_training_steps=total_steps) # Mixed precision scaler = torch.cuda.amp.GradScaler(enabled=config['fp16']) # Training loop for epoch in range(config['num_epochs']): model.train() train_sampler.set_epoch(epoch) pbar = tqdm(train_loader) if local_rank == 0 else train_loader for batch in pbar: optimizer.zero_grad() input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) with torch.cuda.amp.autocast(enabled=config['fp16']): outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels) loss = outputs.loss scaler.scale(loss).backward() torch.nn.utils.clip_grad_norm_(model.parameters(), config['max_grad_norm']) scaler.step(optimizer) scaler.update() scheduler.step() if local_rank == 0: pbar.set_postfix({'loss': loss.item()}) wandb.log({'loss': loss.item()}) # Validation loop can be added here # Save checkpoint if local_rank == 0: model.module.save_pretrained(f'models/checkpoints/epoch_{epoch}')if name == "main": main()
- Assistant
Certainly! I'll provide a comprehensive solution that includes:
-
Adding the Validation Loop: Implement a robust validation loop that evaluates the model's performance on the validation dataset after each training epoch.
-
Fixes and Improvements: Address potential issues and optimize the existing code for better performance and reliability.
Below is the updated code with detailed explanations of the changes made.
Updated Code with Validation Loop and Improvements
import os import yaml import torch import torch.nn as nn from torch.utils.data import DataLoader from transformers import ( AutoTokenizer, get_cosine_schedule_with_warmup, AdamW, AutoModelForCausalLM ) import torch.distributed as dist from torch.utils.data.distributed import DistributedSampler from tqdm import tqdm import wandb from datasets import load_from_disk def main(): # Initialize distributed training dist.init_process_group(backend='nccl', init_method='env://') local_rank = int(os.environ.get("LOCAL_RANK", 0)) torch.cuda.set_device(local_rank) device = torch.device("cuda", local_rank) # Get world size and rank for synchronization world_size = dist.get_world_size() global_rank = dist.get_rank() # Load configuration with open('configs/text_training_config.yaml') as f: config = yaml.safe_load(f) # Initialize W&B (only on master node) if global_rank == 0: wandb.init( project='multi-modal-llm', name='llama3-2-1b-oasst1-finetune', config=config ) # Load tokenizer and model tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-3.2-1B') model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-3.2-1B') # Add pad_token if not present if tokenizer.pad_token is None: tokenizer.add_special_tokens({'pad_token': '[PAD]'}) model.resize_token_embeddings(len(tokenizer)) model.to(device) # Wrap the model with DistributedDataParallel model = nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], output_device=local_rank, find_unused_parameters=False # Set to True if some parameters are not used ) # Load dataset dataset = load_from_disk('data/processed/oasst1_tokenized') train_dataset = dataset['train'] val_dataset = dataset['validation'] # Create DataLoaders with DistributedSampler train_sampler = DistributedSampler(train_dataset, shuffle=True) val_sampler = DistributedSampler(val_dataset, shuffle=False) train_loader = DataLoader( train_dataset, batch_size=config['batch_size'], sampler=train_sampler, num_workers=config.get('num_workers', 4), pin_memory=True ) val_loader = DataLoader( val_dataset, batch_size=config['batch_size'], sampler=val_sampler, num_workers=config.get('num_workers', 4), pin_memory=True ) # Define optimizer and scheduler optimizer = AdamW(model.parameters(), lr=config['learning_rate']) total_steps = len(train_loader) * config['num_epochs'] scheduler = get_cosine_schedule_with_warmup( optimizer, num_warmup_steps=config['warmup_steps'], num_training_steps=total_steps ) # Mixed precision scaler = torch.cuda.amp.GradScaler(enabled=config.get('fp16', True)) # Training loop for epoch in range(config['num_epochs']): model.train() train_sampler.set_epoch(epoch) # Ensure shuffling is epoch-dependent if global_rank == 0: pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{config['num_epochs']}") else: pbar = train_loader for batch in pbar: optimizer.zero_grad() input_ids = batch['input_ids'].to(device, non_blocking=True) attention_mask = batch['attention_mask'].to(device, non_blocking=True) labels = batch['labels'].to(device, non_blocking=True) with torch.cuda.amp.autocast(enabled=config.get('fp16', True)): outputs = model( input_ids=input_ids, attention_mask=attention_mask, labels=labels ) loss = outputs.loss # Backpropagation scaler.scale(loss).backward() # Gradient clipping scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), config.get('max_grad_norm', 1.0)) # Optimizer step scaler.step(optimizer) scaler.update() # Scheduler step scheduler.step() if global_rank == 0: pbar.set_postfix({'loss': loss.item()}) wandb.log({'loss': loss.item(), 'epoch': epoch + 1}) # Validation loop model.eval() val_loss_total = 0.0 val_steps = 0 with torch.no_grad(): if global_rank == 0: val_pbar = tqdm(val_loader, desc=f"Validation Epoch {epoch+1}") else: val_pbar = val_loader for batch in val_pbar: input_ids = batch['input_ids'].to(device, non_blocking=True) attention_mask = batch['attention_mask'].to(device, non_blocking=True) labels = batch['labels'].to(device, non_blocking=True) with torch.cuda.amp.autocast(enabled=config.get('fp16', True)): outputs = model( input_ids=input_ids, attention_mask=attention_mask, labels=labels ) loss = outputs.loss loss_value = loss.item() val_loss_total += loss_value val_steps += 1 if global_rank == 0: val_pbar.set_postfix({'val_loss': loss_value}) # Aggregate validation loss across all processes val_loss_tensor = torch.tensor(val_loss_total, device=device) val_steps_tensor = torch.tensor(val_steps, device=device) dist.all_reduce(val_loss_tensor, op=dist.ReduceOp.SUM) dist.all_reduce(val_steps_tensor, op=dist.ReduceOp.SUM) # Compute average validation loss avg_val_loss = val_loss_tensor.item() / val_steps_tensor.item() if global_rank == 0: print(f"Epoch {epoch+1} Validation Loss: {avg_val_loss:.4f}") wandb.log({'val_loss': avg_val_loss, 'epoch': epoch + 1}) # Save checkpoint (only on master node) if global_rank == 0: checkpoint_dir = f'models/checkpoints/epoch_{epoch + 1}' os.makedirs(checkpoint_dir, exist_ok=True) model.module.save_pretrained(checkpoint_dir) tokenizer.save_pretrained(checkpoint_dir) # Cleanup dist.destroy_process_group() if __name__ == "__main__": main()
Detailed Explanation of Changes and Fixes
-
Initialization of Distributed Process Group:
dist.init_process_group(backend='nccl', init_method='env://')init_method='env://: Ensures that the process group is initialized using environment variables, which is a common practice for distributed training (e.g., when usingtorch.distributed.launchortorchrun).
-
Retrieving World Size and Rank:
world_size = dist.get_world_size() global_rank = dist.get_rank()world_size: Total number of processes participating in the training.global_rank: Unique identifier for each process. This replaceslocal_rankwhere appropriate to handle multi-node setups.
-
DataLoader Improvements:
shuffle=Truefor Training Sampler: Ensures that data is shuffled differently at each epoch.num_workersandpin_memory: Added for potential performance improvements. These can be adjusted based on the system's capabilities and the dataset size.
train_loader = DataLoader( train_dataset, batch_size=config['batch_size'], sampler=train_sampler, num_workers=config.get('num_workers', 4), pin_memory=True ) val_loader = DataLoader( val_dataset, batch_size=config['batch_size'], sampler=val_sampler, num_workers=config.get('num_workers', 4), pin_memory=True ) -
Optimizer and Scheduler Configuration:
- Graceful Handling of Missing Configs: Using
config.get('param', default)to provide default values if certain configuration parameters are missing. - Ensuring Correct Total Steps for Scheduler: Calculated based on the length of the
train_loaderand the number of epochs.
- Graceful Handling of Missing Configs: Using
-
Training Loop Enhancements:
- Non-blocking Data Transfers: Using
non_blocking=Truecan improve data transfer performance between CPU and GPU. - Gradient Clipping: Now includes
scaler.unscale_(optimizer)before clipping to ensure gradients are properly scaled. - Logging: Added
epochinformation to the logs for better tracking in W&B.
input_ids = batch['input_ids'].to(device, non_blocking=True) attention_mask = batch['attention_mask'].to(device, non_blocking=True) labels = batch['labels'].to(device, non_blocking=True) - Non-blocking Data Transfers: Using
-
Validation Loop Implementation:
- Switch to Evaluation Mode:
model.eval() - No Gradient Computation:
with torch.no_grad(): ... - Progress Bar for Validation: Only shows the progress bar on the master node.
- Loss Aggregation Across All Processes:
dist.all_reduce(val_loss_tensor, op=dist.ReduceOp.SUM) dist.all_reduce(val_steps_tensor, op=dist.ReduceOp.SUM)- Aggregates the total validation loss and the number of validation steps from all processes to compute the average validation loss accurately.
- Logging Validation Metrics: Only the master node logs the validation loss to W&B to avoid duplicate logs.
- Example:
if global_rank == 0: print(f"Epoch {epoch+1} Validation Loss: {avg_val_loss:.4f}") wandb.log({'val_loss': avg_val_loss, 'epoch': epoch + 1})
- Switch to Evaluation Mode:
-
Checkpoint Saving Enhancements:
- Creating Checkpoint Directory:
os.makedirs(checkpoint_dir, exist_ok=True) - Saving Tokenizer: Ensures that the tokenizer configuration is also saved alongside the model.
tokenizer.save_pretrained(checkpoint_dir)
- Creating Checkpoint Directory:
-
Cleanup of Distributed Process Group:
dist.destroy_process_group()- Ensures that the distributed environment is properly cleaned up once training is complete.
-
Additional Recommendations:
- Handling Exceptions: It might be beneficial to wrap the training loop in a try-except block to handle potential interruptions gracefully.
- Saving Additional Metrics: Depending on the use case, you might want to log more metrics (e.g., perplexity, accuracy) during validation.
- Learning Rate Monitoring: Logging the learning rate can help in diagnosing training behavior.
if global_rank == 0: current_lr = scheduler.get_last_lr()[0] wandb.log({'learning_rate': current_lr, 'epoch': epoch + 1})
-
Environment Variables for Distributed Training:
- Ensure that when you launch the training script, the necessary environment variables (
LOCAL_RANK,WORLD_SIZE,RANK, etc.) are correctly set. This is typically handled by launch utilities liketorch.distributed.launchortorchrun.
- Ensure that when you launch the training script, the necessary environment variables (
Example Configuration (
configs/text_training_config.yaml)Ensure that your configuration file includes all the necessary parameters. Here's an example:
batch_size: 8 learning_rate: 5e-5 num_epochs: 3 warmup_steps: 500 max_grad_norm: 1.0 fp16: true num_workers: 4Adjust these values based on your dataset size, available computational resources, and specific training requirements.
Launching Distributed Training
To run the training script in a distributed manner, use PyTorch's
torchrunortorch.distributed.launch. Here's an example usingtorchrun:torchrun --nproc_per_node=4 your_training_script.py--nproc_per_node: Specifies the number of processes to launch on each node, typically equal to the number of GPUs per node.
Ensure that the environment variables are correctly propagated, especially
LOCAL_RANK, which is automatically handled bytorchrun.
Conclusion
The updated code now includes a robust validation loop that evaluates the model after each training epoch, aggregates the validation loss across all distributed processes, and logs the results using Weights & Biases (W&B). Additionally, several improvements have been made to optimize performance and ensure reliability in a distributed training environment. Make sure to adjust the configuration and training parameters based on your specific requirements and computational resources.
-