Fine-tune a BERT model for custom text classification using PyTorch. Implement a data…
- User
Fine-tune a BERT model for custom text classification using PyTorch. Implement a data loader that tokenizes input on the fly and save model checkpoints only when validation loss improves for early stopping.
Assistant
This is a comprehensive request to create a complete pipeline for fine-tuning BERT for text classification with:
- Custom data loader with on-the-fly tokenization
- Early stopping based on validation loss
- Model checkpoints saving only when validation improves
- Using PyTorch
I'll create a complete, production-ready implementation with:
- Dataset and DataLoader classes
- Training and evaluation functions
- Early stopping mechanism
- Checkpoint management
- Configuration
- Example usage
- Proper error handling
- Best practices
Let me structure this comprehensively:
Fine-tune BERT for Custom Text Classification
Here's a complete, production-ready implementation:
1. Core Implementation
import os import json import torch import numpy as np from pathlib import Path from typing import Dict, List, Tuple, Optional from dataclasses import dataclass import logging from datetime import datetime import torch.nn as nn from torch.utils.data import Dataset, DataLoader from torch.optim import AdamW from transformers import ( BertTokenizer, BertForSequenceClassification, get_linear_schedule_with_warmup ) from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix import matplotlib.pyplot as plt import seaborn as sns # Setup logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) @dataclass class Config: """Training configuration""" model_name: str = "bert-base-uncased" num_classes: int = 2 max_length: int = 128 batch_size: int = 32 learning_rate: float = 2e-5 num_epochs: int = 10 warmup_steps: int = 500 weight_decay: float = 0.01 early_stopping_patience: int = 3 seed: int = 42 device: str = "cuda" if torch.cuda.is_available() else "cpu" checkpoint_dir: str = "./checkpoints" log_dir: str = "./logs" class CustomDataset(Dataset): """Custom dataset with on-the-fly tokenization""" def __init__( self, texts: List[str], labels: List[int], tokenizer: BertTokenizer, max_length: int = 128, label_mapping: Optional[Dict[str, int]] = None ): self.texts = texts self.labels = labels self.tokenizer = tokenizer self.max_length = max_length self.label_mapping = label_mapping if len(texts) != len(labels): raise ValueError( f"Mismatch: {len(texts)} texts but {len(labels)} labels" ) def __len__(self) -> int: return len(self.texts) def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: """ Get item with on-the-fly tokenization Args: idx: Sample index Returns: Dictionary with tokenized inputs and label """ text = self.texts[idx] label = self.labels[idx] # Tokenize on the fly encoding = self.tokenizer( text, max_length=self.max_length, padding='max_length', truncation=True, return_tensors='pt' ) return { 'input_ids': encoding['input_ids'].squeeze(), 'attention_mask': encoding['attention_mask'].squeeze(), 'token_type_ids': encoding['token_type_ids'].squeeze(), 'label': torch.tensor(label, dtype=torch.long) } class TextClassifier(nn.Module): """BERT-based text classifier""" def __init__(self, model_name: str, num_classes: int): super().__init__() self.bert = BertForSequenceClassification.from_pretrained( model_name, num_labels=num_classes, output_hidden_states=False ) self.num_classes = num_classes def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, token_type_ids: torch.Tensor, labels: Optional[torch.Tensor] = None ): """ Forward pass Args: input_ids: Token IDs attention_mask: Attention mask token_type_ids: Token type IDs labels: Target labels (optional) Returns: Model output with loss and logits """ outputs = self.bert( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels ) return outputs class EarlyStopping: """Early stopping with checkpoint management""" def __init__( self, patience: int = 3, min_delta: float = 0.0, checkpoint_dir: str = "./checkpoints", mode: str = "min" ): self.patience = patience self.min_delta = min_delta self.checkpoint_dir = Path(checkpoint_dir) self.checkpoint_dir.mkdir(parents=True, exist_ok=True) self.mode = mode self.counter = 0 self.best_value = None self.best_epoch = 0 if mode == "min": self.is_better = lambda x, y: x < y - min_delta else: # max self.is_better = lambda x, y: x > y + min_delta def __call__( self, current_value: float, model: nn.Module, epoch: int, optimizer: Optional[torch.optim.Optimizer] = None, additional_info: Optional[Dict] = None ) -> bool: """ Check if training should stop Args: current_value: Current metric value model: Model to save epoch: Current epoch optimizer: Optional optimizer to save additional_info: Additional data to save Returns: True if training should stop, False otherwise """ if self.best_value is None: self.best_value = current_value self._save_checkpoint(model, epoch, optimizer, additional_info) elif self.is_better(current_value, self.best_value): self.best_value = current_value self.counter = 0 self.best_epoch = epoch self._save_checkpoint(model, epoch, optimizer, additional_info) logger.info( f"✓ Validation metric improved to {current_value:.4f}. " f"Checkpoint saved." ) else: self.counter += 1 logger.info( f"✗ No improvement for {self.counter}/{self.patience} epochs" ) if self.counter >= self.patience: logger.info(f"Early stopping at epoch {epoch}") return True return False def _save_checkpoint( self, model: nn.Module, epoch: int, optimizer: Optional[torch.optim.Optimizer] = None, additional_info: Optional[Dict] = None ): """Save model checkpoint""" checkpoint = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'best_value': self.best_value, 'timestamp': datetime.now().isoformat() } if optimizer is not None: checkpoint['optimizer_state_dict'] = optimizer.state_dict() if additional_info: checkpoint.update(additional_info) checkpoint_path = self.checkpoint_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) # Also save as best model best_checkpoint_path = self.checkpoint_dir / "best_model.pt" torch.save(checkpoint, best_checkpoint_path) logger.info(f"Checkpoint saved: {checkpoint_path}") class TextClassificationTrainer: """Main trainer class for text classification""" def __init__(self, config: Config): self.config = config self.device = torch.device(config.device) self._set_seed(config.seed) # Create directories Path(config.checkpoint_dir).mkdir(parents=True, exist_ok=True) Path(config.log_dir).mkdir(parents=True, exist_ok=True) # Load tokenizer and model self.tokenizer = BertTokenizer.from_pretrained(config.model_name) self.model = TextClassifier( config.model_name, config.num_classes ).to(self.device) # Initialize early stopping self.early_stopping = EarlyStopping( patience=config.early_stopping_patience, checkpoint_dir=config.checkpoint_dir, mode="min" # Minimize validation loss ) self.train_losses = [] self.val_losses = [] self.val_metrics = [] logger.info(f"Model loaded: {config.model_name}") logger.info(f"Device: {self.device}") @staticmethod def _set_seed(seed: int): """Set random seed for reproducibility""" np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) def create_dataloaders( self, train_texts: List[str], train_labels: List[int], val_texts: List[str], val_labels: List[int], test_texts: Optional[List[str]] = None, test_labels: Optional[List[int]] = None, ) -> Tuple[DataLoader, DataLoader, Optional[DataLoader]]: """ Create data loaders with on-the-fly tokenization Args: train_texts: Training texts train_labels: Training labels val_texts: Validation texts val_labels: Validation labels test_texts: Test texts (optional) test_labels: Test labels (optional) Returns: Tuple of (train_loader, val_loader, test_loader) """ train_dataset = CustomDataset( train_texts, train_labels, self.tokenizer, self.config.max_length ) val_dataset = CustomDataset( val_texts, val_labels, self.tokenizer, self.config.max_length ) train_loader = DataLoader( train_dataset, batch_size=self.config.batch_size, shuffle=True, num_workers=4, pin_memory=True ) val_loader = DataLoader( val_dataset, batch_size=self.config.batch_size, shuffle=False, num_workers=4, pin_memory=True ) test_loader = None if test_texts is not None: test_dataset = CustomDataset( test_texts, test_labels, self.tokenizer, self.config.max_length ) test_loader = DataLoader( test_dataset, batch_size=self.config.batch_size, shuffle=False, num_workers=4, pin_memory=True ) logger.info( f"Data loaders created:\n" f" Train: {len(train_loader)} batches\n" f" Val: {len(val_loader)} batches" ) if test_loader: logger.info(f" Test: {len(test_loader)} batches") return train_loader, val_loader, test_loader def train_epoch( self, train_loader: DataLoader, optimizer: torch.optim.Optimizer, scheduler: torch.optim.lr_scheduler.LambdaLR ) -> float: """ Train for one epoch Args: train_loader: Training data loader optimizer: Optimizer scheduler: Learning rate scheduler Returns: Average training loss """ self.model.train() total_loss = 0 total_samples = 0 for batch_idx, batch in enumerate(train_loader): input_ids = batch['input_ids'].to(self.device) attention_mask = batch['attention_mask'].to(self.device) token_type_ids = batch['token_type_ids'].to(self.device) labels = batch['label'].to(self.device) # Forward pass outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels ) loss = outputs.loss # Backward pass optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) optimizer.step() scheduler.step() total_loss += loss.item() * labels.size(0) total_samples += labels.size(0) if (batch_idx + 1) % 100 == 0: avg_loss = total_loss / total_samples logger.info( f"Batch {batch_idx + 1}/{len(train_loader)} - " f"Loss: {avg_loss:.4f}" ) avg_epoch_loss = total_loss / total_samples return avg_epoch_loss @torch.no_grad() def evaluate( self, data_loader: DataLoader, return_predictions: bool = False ) -> Tuple[float, Dict, Optional[np.ndarray]]: """ Evaluate model Args: data_loader: Evaluation data loader return_predictions: Whether to return predictions Returns: Tuple of (loss, metrics_dict, predictions) """ self.model.eval() total_loss = 0 total_samples = 0 all_logits = [] all_labels = [] for batch in data_loader: input_ids = batch['input_ids'].to(self.device) attention_mask = batch['attention_mask'].to(self.device) token_type_ids = batch['token_type_ids'].to(self.device) labels = batch['label'].to(self.device) outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels ) loss = outputs.loss logits = outputs.logits total_loss += loss.item() * labels.size(0) total_samples += labels.size(0) all_logits.append(logits.cpu().numpy()) all_labels.append(labels.cpu().numpy()) avg_loss = total_loss / total_samples # Calculate metrics all_logits = np.concatenate(all_logits, axis=0) all_labels = np.concatenate(all_labels, axis=0) predictions = np.argmax(all_logits, axis=1) accuracy = accuracy_score(all_labels, predictions) precision, recall, f1, _ = precision_recall_fscore_support( all_labels, predictions, average='weighted' ) metrics = { 'accuracy': accuracy, 'precision': precision, 'recall': recall, 'f1': f1 } predictions_output = predictions if return_predictions else None return avg_loss, metrics, predictions_output def train( self, train_loader: DataLoader, val_loader: DataLoader, test_loader: Optional[DataLoader] = None ) -> Dict: """ Main training loop Args: train_loader: Training data loader val_loader: Validation data loader test_loader: Test data loader (optional) Returns: Training history """ num_training_steps = len(train_loader) * self.config.num_epochs optimizer = AdamW( self.model.parameters(), lr=self.config.learning_rate, weight_decay=self.config.weight_decay ) scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=self.config.warmup_steps, num_training_steps=num_training_steps ) logger.info("Starting training...") logger.info(f"Total steps: {num_training_steps}") for epoch in range(self.config.num_epochs): logger.info(f"\n{'='*50}") logger.info(f"Epoch {epoch + 1}/{self.config.num_epochs}") logger.info(f"{'='*50}") # Train epoch train_loss = self.train_epoch(train_loader, optimizer, scheduler) self.train_losses.append(train_loss) logger.info(f"Training Loss: {train_loss:.4f}") # Validate val_loss, val_metrics, _ = self.evaluate(val_loader) self.val_losses.append(val_loss) self.val_metrics.append(val_metrics) logger.info(f"Validation Loss: {val_loss:.4f}") logger.info( f"Validation Metrics: Accuracy={val_metrics['accuracy']:.4f}, " f"F1={val_metrics['f1']:.4f}" ) # Early stopping should_stop = self.early_stopping( val_loss, self.model, epoch, optimizer, additional_info={ 'config': self.config.__dict__, 'metrics': val_metrics, 'train_loss': train_loss } ) if should_stop: break logger.info(f"\nTraining completed!") logger.info( f"Best epoch: {self.early_stopping.best_epoch} " f"(Validation Loss: {self.early_stopping.best_value:.4f})" ) # Test on test set if provided test_metrics = None if test_loader is not None: logger.info("\nEvaluating on test set...") test_loss, test_metrics, _ = self.evaluate(test_loader) logger.info(f"Test Loss: {test_loss:.4f}") logger.info( f"Test Metrics: Accuracy={test_metrics['accuracy']:.4f}, " f"F1={test_metrics['f1']:.4f}" ) return { 'train_losses': self.train_losses, 'val_losses': self.val_losses, 'val_metrics': self.val_metrics, 'test_metrics': test_metrics, 'best_epoch': self.early_stopping.best_epoch } def load_checkpoint(self, checkpoint_path: str, load_optimizer: bool = False): """ Load model checkpoint Args: checkpoint_path: Path to checkpoint load_optimizer: Whether to load optimizer state """ checkpoint = torch.load(checkpoint_path, map_location=self.device) self.model.load_state_dict(checkpoint['model_state_dict']) logger.info(f"Checkpoint loaded from {checkpoint_path}") if load_optimizer and 'optimizer_state_dict' in checkpoint: return checkpoint['optimizer_state_dict'] return checkpoint def predict( self, texts: List[str], return_confidence: bool = True ) -> Tuple[np.ndarray, Optional[np.ndarray]]: """ Predict on new texts Args: texts: List of input texts return_confidence: Whether to return confidence scores Returns: Tuple of (predictions, confidence_scores) """ dataset = CustomDataset( texts, [0] * len(texts), # Dummy labels self.tokenizer, self.config.max_length ) loader = DataLoader( dataset, batch_size=self.config.batch_size, shuffle=False ) self.model.eval() all_logits = [] with torch.no_grad(): for batch in loader: input_ids = batch['input_ids'].to(self.device) attention_mask = batch['attention_mask'].to(self.device) token_type_ids = batch['token_type_ids'].to(self.device) outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids ) all_logits.append(outputs.logits.cpu().numpy()) all_logits = np.concatenate(all_logits, axis=0) predictions = np.argmax(all_logits, axis=1) confidence = np.max( np.exp(all_logits) / np.exp(all_logits).sum(axis=1, keepdims=True), axis=1 ) return predictions, confidence if return_confidence else None def plot_training_history(self, save_path: Optional[str] = None): """ Plot training history Args: save_path: Path to save figure """ fig, axes = plt.subplots(1, 2, figsize=(12, 4)) # Loss plot axes[0].plot(self.train_losses, label='Train Loss') axes[0].plot(self.val_losses, label='Val Loss') axes[0].set_xlabel('Epoch') axes[0].set_ylabel('Loss') axes[0].set_title('Training and Validation Loss') axes[0].legend() axes[0].grid(True, alpha=0.3) # Metrics plot val_accuracies = [m['accuracy'] for m in self.val_metrics] val_f1s = [m['f1'] for m in self.val_metrics] axes[1].plot(val_accuracies, label='Accuracy') axes[1].plot(val_f1s, label='F1 Score') axes[1].set_xlabel('Epoch') axes[1].set_ylabel('Score') axes[1].set_title('Validation Metrics') axes[1].legend() axes[1].grid(True, alpha=0.3) plt.tight_layout() if save_path: plt.savefig(save_path, dpi=300, bbox_inches='tight') logger.info(f"Training history plot saved: {save_path}") plt.show() def plot_confusion_matrix( y_true: np.ndarray, y_pred: np.ndarray, labels: Optional[List[str]] = None, save_path: Optional[str] = None ): """Plot confusion matrix""" cm = confusion_matrix(y_true, y_pred) plt.figure(figsize=(8, 6)) sns.heatmap( cm, annot=True, fmt='d', cmap='Blues', xticklabels=labels, yticklabels=labels ) plt.title('Confusion Matrix') plt.ylabel('True Label') plt.xlabel('Predicted Label') if save_path: plt.savefig(save_path, dpi=300, bbox_inches='tight') logger.info(f"Confusion matrix saved: {save_path}") plt.show()2. Example Usage
def main(): """Example: Movie reviews sentiment classification""" # Sample data train_texts = [ "This movie is absolutely fantastic! I loved every minute of it.", "Terrible film, a complete waste of time.", "Amazing acting and great plot. Highly recommended!", "Boring and predictable. I fell asleep halfway through.", "One of the best movies I've ever seen.", "Unwatchable garbage. The worst film ever made.", "Entertaining and well-made. Worth watching.", "Disappointing. Below expectations.", ] train_labels = [1, 0, 1, 0, 1, 0, 1, 0] # 1: positive, 0: negative val_texts = [ "Really good movie. Enjoyed it thoroughly.", "Not worth the hype. Disappointing.", ] val_labels = [1, 0] test_texts = [ "Spectacular performance and excellent direction!", "Dull and uninspiring film.", "Perfect! Absolutely loved it!", ] test_labels = [1, 0, 1] # Configuration config = Config( model_name="bert-base-uncased", num_classes=2, max_length=128, batch_size=8, learning_rate=2e-5, num_epochs=5, warmup_steps=100, early_stopping_patience=2, checkpoint_dir="./checkpoints", log_dir="./logs" ) # Initialize trainer trainer = TextClassificationTrainer(config) # Create data loaders train_loader, val_loader, test_loader = trainer.create_dataloaders( train_texts, train_labels, val_texts, val_labels, test_texts, test_labels ) # Train history = trainer.train(train_loader, val_loader, test_loader) # Plot results trainer.plot_training_history(save_path="./logs/training_history.png") # Make predictions new_texts = [ "This is the best movie ever!", "Absolutely horrible film.", "Pretty good, not bad at all.", ] predictions, confidence = trainer.predict(new_texts) print("\n" + "="*60) print("PREDICTIONS ON NEW TEXTS") print("="*60) for text, pred, conf in zip(new_texts, predictions, confidence): label = "Positive" if pred == 1 else "Negative" print(f"Text: {text}") print(f"Prediction: {label} (Confidence: {conf:.4f})\n") # Load best model for inference checkpoint_path = Path(config.checkpoint_dir) / "best_model.pt" if checkpoint_path.exists(): trainer.load_checkpoint(str(checkpoint_path)) print(f"✓ Best model loaded from {checkpoint_path}") if __name__ == "__main__": main()3. Advanced Usage with Real Dataset
from sklearn.model_selection import train_test_split def train_with_csv_data(csv_path: str, text_column: str, label_column: str): """ Train with CSV data Args: csv_path: Path to CSV file text_column: Column name for texts label_column: Column name for labels """ import pandas as pd # Load data df = pd.read_csv(csv_path) texts = df[text_column].tolist() labels = df[label_column].tolist() # Create label mapping if labels are strings unique_labels = list(set(labels)) label_mapping = {label: idx for idx, label in enumerate(unique_labels)} labels = [label_mapping[label] for label in labels] # Split data train_texts, temp_texts, train_labels, temp_labels = train_test_split( texts, labels, test_size=0.3, random_state=42, stratify=labels ) val_texts, test_texts, val_labels, test_labels = train_test_split( temp_texts, temp_labels, test_size=0.5, random_state=42, stratify=temp_labels ) logger.info( f"Data split:\n" f" Train: {len(train_texts)}\n" f" Val: {len(val_texts)}\n" f" Test: {len(test_texts)}" ) # Setup config config = Config( num_classes=len(label_mapping), batch_size=32, num_epochs=10, early_stopping_patience=3 ) # Train trainer = TextClassificationTrainer(config) train_loader, val_loader, test_loader = trainer.create_dataloaders( train_texts, train_labels, val_texts, val_labels, test_texts, test_labels ) history = trainer.train(train_loader, val_loader, test_loader) return trainer, history, label_mapping def train_multi_class_example(): """Example: Multi-class classification""" train_texts = [ "The battery life is excellent. Best phone ever!", "Camera quality is poor. Major disappointment.", "Good design, comfortable to hold.", "The camera quality is outstanding!", "Battery drains too quickly.", "Sleek design and perfect fit.", "Terrible phone. Many issues.", "Amazing camera performance.", ] # 0: Battery, 1: Camera, 2: Design train_labels = [0, 0, 2, 1, 0, 2, 1, 1] val_texts = [ "Excellent battery performance.", "Beautiful design.", "Great camera quality.", ] val_labels = [0, 2, 1] config = Config( num_classes=3, num_epochs=5, batch_size=4 ) trainer = TextClassificationTrainer(config) train_loader, val_loader, _ = trainer.create_dataloaders( train_texts, train_labels, val_texts, val_labels ) history = trainer.train(train_loader, val_loader) class_labels = ["Battery", "Camera", "Design"] # Predictions new_texts = [ "The battery performance is excellent.", "Amazing photograph quality!", "Very elegant design.", ] predictions, confidence = trainer.predict(new_texts) print("\n" + "="*60) print("MULTI-CLASS PREDICTIONS") print("="*60) for text, pred, conf in zip(new_texts, predictions, confidence): print(f"Text: {text}") print(f"Predicted Class: {class_labels[pred]} (Confidence: {conf:.4f})\n")4. Key Features
Feature Description On-the-fly Tokenization Tokenization happens during iteration, saves memory Early Stopping Stops training when validation loss stops improving Smart Checkpointing Saves only when validation improves Gradient Clipping Prevents gradient explosion Learning Rate Scheduling Warm-up and linear decay schedule Multi-class Support Works with any number of classes Prediction Pipeline Built-in inference with confidence scores Visualization Training history plots Logging Comprehensive training logs This implementation is production-ready and handles all major aspects of fine-tuning BERT!
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.