The following code works great , but we need to adapt it to work on mps on a M2-Ultra…
- User
The following code works great , but we need to adapt it to work on mps on a M2-Ultra Mac Studio
import torch import torch.nn as nn import logging from datasets import load_dataset, Dataset from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling from torch.cuda.amp import autocast import warnings from tqdm import tqdm
warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=UserWarning)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(name)
MODEL_NAME = "nisten/Biggie-SmoLlm-0.15B-Base" MAX_LENGTH = 2048 BATCH_SIZE = 8 LEARNING_RATE = 2e-4 MAX_STEPS = 3000 GRADIENT_ACCUMULATION_STEPS = 2 NUM_WARMUP_STEPS = 30 OUTPUT_DIR = "./capybara_finetuned_results"
torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True
class GrokAdamW(torch.optim.Optimizer): def init(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-2, alpha_init=0.98, lamb=2.0, gamma=0.1, grokking_signal_fns=None, grokking_signal_decay_rate=0.1, gradient_clipping=1.0): defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, alpha_init=alpha_init, lamb=lamb, gamma=gamma, grokking_signal_fns=grokking_signal_fns, grokking_signal_decay_rate=grokking_signal_decay_rate, gradient_clipping=gradient_clipping) super(GrokAdamW, self).init(params, defaults)
def step(self, closure=None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: grokking_signal = self._compute_grokking_signal(group) for i, p in enumerate(group['params']): if p.grad is None: continue grad = p.grad if group['gradient_clipping'] > 0: grad = torch.clamp(grad, -group['gradient_clipping'], group['gradient_clipping']) state = self.state[p] if len(state) == 0: state['step'] = 0 state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format) state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format) state['grok_ema'] = torch.zeros_like(p, memory_format=torch.preserve_format) exp_avg, exp_avg_sq, grok_ema = state['exp_avg'], state['exp_avg_sq'], state['grok_ema'] beta1, beta2 = group['betas'] state['step'] += 1 layer_beta1 = beta1 * (1 - group['gamma'])**i alpha = group['alpha_init'] * torch.exp(torch.tensor(-group['grokking_signal_decay_rate'] * grokking_signal)) grok_ema.mul_(alpha).add_(grad, alpha=1 - alpha) grok_grad = grad + group['lamb'] * grok_ema exp_avg.mul_(layer_beta1).add_(grok_grad, alpha=1 - layer_beta1) exp_avg_sq.mul_(beta2).addcmul_(grok_grad, grok_grad, value=1 - beta2) denom = exp_avg_sq.sqrt().add_(group['eps']) step_size = group['lr'] if group['weight_decay'] != 0: p.data.mul_(1 - group['lr'] * group['weight_decay']) p.addcdiv_(exp_avg, denom, value=-step_size) return loss def _compute_grokking_signal(self, group): if group['grokking_signal_fns'] is None: return 0.0 signals = [] for fn in group['grokking_signal_fns']: try: signal = fn() if signal is not None: signals.append(signal) except Exception as e: logger.warning(f"Error in grokking_signal_fn: {e}. Ignoring this function.") if not signals: return 0.0 return sum(signals) / len(signals)def format_capybara_prompts(examples): texts = [] for conversation in examples['conversation']: formatted_text = "" for turn in conversation: if 'input' in turn: formatted_text += f"Human: {turn['input']}\n\n" if 'output' in turn: formatted_text += f"Assistant: {turn['output']}\n\n" texts.append(formatted_text.strip()) return {"text": texts}
class CustomTrainer(Trainer): def init(self, *args, **kwargs): super().init(*args, **kwargs) self.grokking_signal = 0.0
def compute_loss(self, model, inputs, return_outputs=False): labels = inputs.pop("labels") outputs = model(**inputs) logits = outputs.logits shift_logits = logits[..., :-1, :].contiguous() shift_labels = labels[..., 1:].contiguous() loss_fct = nn.CrossEntropyLoss() loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) return (loss, outputs) if return_outputs else loss def training_step(self, model, inputs): model.train() inputs = self._prepare_inputs(inputs) with autocast(dtype=torch.bfloat16): loss = self.compute_loss(model, inputs) if self.args.gradient_accumulation_steps > 1: loss = loss / self.args.gradient_accumulation_steps loss.backward() self.grokking_signal = loss.item() return loss.detach()def grokking_signal_fn(): return trainer.grokking_signal
def main(): logger.info(f"🚀 Initializing {MODEL_NAME} finetuning with GrokAdamW")
try: config = AutoConfig.from_pretrained(MODEL_NAME) tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.bfloat16) except Exception as e: logger.error(f"❌ Failed to load model or tokenizer: {str(e)}") return if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token model.config.pad_token_id = model.config.eos_token_id logger.info("📚 Loading Capybara dataset") try: capybara_dataset = load_dataset("LDJnr/Capybara", split="train") capybara_dataset = capybara_dataset.map(format_capybara_prompts, batched=True, remove_columns=capybara_dataset.column_names) except Exception as e: logger.error(f"❌ Failed to load Capybara dataset: {str(e)}") return logger.info(f"📊 Capybara dataset size: {len(capybara_dataset)}") def tokenize_function(examples): return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=MAX_LENGTH) logger.info("🔢 Tokenizing dataset") tokenized_dataset = capybara_dataset.map(tokenize_function, batched=True, remove_columns=capybara_dataset.column_names) logger.info("🏋️ Setting up the training arguments") training_args = TrainingArguments( output_dir=OUTPUT_DIR, num_train_epochs=3, per_device_train_batch_size=BATCH_SIZE, gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS, learning_rate=LEARNING_RATE, weight_decay=0.01, bf16=True, logging_steps=10, save_steps=300, save_total_limit=10, dataloader_num_workers=4, warmup_steps=NUM_WARMUP_STEPS, gradient_checkpointing=True, evaluation_strategy="steps", eval_steps=300, max_steps=MAX_STEPS, fp16=False, optim="adamw_hf", lr_scheduler_type="cosine", load_best_model_at_end=True, metric_for_best_model="loss", greater_is_better=False, ) data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False) optimizer = GrokAdamW( model.parameters(), lr=LEARNING_RATE, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01, alpha_init=0.98, lamb=2.0, gamma=0.1, grokking_signal_fns=[grokking_signal_fn], grokking_signal_decay_rate=0.1, gradient_clipping=1.0 ) logger.info("🏃♂️ Initializing Trainer with GrokAdamW") global trainer trainer = CustomTrainer( model=model, args=training_args, train_dataset=tokenized_dataset, eval_dataset=tokenized_dataset.select(range(min(1000, len(tokenized_dataset)))), data_collator=data_collator, optimizers=(optimizer, None), ) logger.info("🔥 Starting the training with GrokAdamW") try: trainer.train() except Exception as e: logger.error(f"❌ Training failed: {str(e)}") return logger.info("💾 Saving the model") try: trainer.save_model(OUTPUT_DIR) except Exception as e: logger.error(f"❌ Failed to save model: {str(e)}") logger.info("🎉 Finetuning with GrokAdamW completed!")if name == "main": main()
Here are some docs as well
Docs > MPS backend MPS backend mps device enables high-performance training on GPU for MacOS devices with Metal programming framework. It introduces a new device to map Machine Learning computational graphs and primitives on highly efficient Metal Performance Shaders Graph framework and tuned kernels provided by Metal Performance Shaders framework respectively.
The new MPS backend extends the PyTorch ecosystem and provides existing scripts capabilities to setup and run operations on GPU.
To get started, simply move your Tensor and Module to the mps device:
Check that MPS is available
if not torch.backends.mps.is_available(): if not torch.backends.mps.is_built(): print("MPS not available because the current PyTorch install was not " "built with MPS enabled.") else: print("MPS not available because the current MacOS version is not 12.3+ " "and/or you do not have an MPS-enabled device on this machine.")
else: mps_device = torch.device("mps")
# Create a Tensor directly on the mps device x = torch.ones(5, device=mps_device) # Or x = torch.ones(5, device="mps") # Any operation happens on the GPU y = x * 2 # Move your model to mps just like any other device model = YourFavoriteNet() model.to(mps_device) # Now every call runs on the GPU pred = model(x)The dataset in sharegpt format should follow the below format:
[ { "conversations": [ { "from": "human", "value": "user instruction" }, { "from": "gpt", "value": "model response" } ], "system": "system prompt (optional)", "tools": "tool description (optional)" } ]
but i think we have the system column preceding the conversations column in our case but that should not be an issue
also instead of using the capybara dataset we wanna use our own local one saved at /Users/n/dllama/sharegpt_cleaned_dataset.jsonl so adapt the code to do that too please,
so again rewrite the entire training script so it works using ONLY our own dataset and actually our own local file model too from /Users/n/qwen2-14B/
- Assistant
Certainly! Below is the fully adapted training script tailored to run on the MPS backend of your M2 Ultra Mac Studio. This script:
- Uses the MPS device for training.
- Loads a local dataset from
/Users/n/dllama/sharegpt_cleaned_dataset.jsonl. - Utilizes a local model from
/Users/n/qwen2-14B/.
Complete Training Script for MPS on M2 Ultra Mac Studio
import torch import torch.nn as nn import logging from datasets import load_dataset, Dataset from transformers import ( AutoConfig, AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling, ) from torch.cuda.amp import autocast import warnings from tqdm import tqdm import os # Suppress unnecessary warnings warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=UserWarning) # Configure logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) logger = logging.getLogger(__name__) # Define constants MODEL_PATH = "/Users/n/qwen2-14B/" DATASET_PATH = "/Users/n/dllama/sharegpt_cleaned_dataset.jsonl" MAX_LENGTH = 2048 BATCH_SIZE = 8 LEARNING_RATE = 2e-4 MAX_STEPS = 3000 GRADIENT_ACCUMULATION_STEPS = 2 NUM_WARMUP_STEPS = 30 OUTPUT_DIR = "./finetuned_results" # Check for MPS availability if not torch.backends.mps.is_available(): if not torch.backends.mps.is_built(): logger.error( "MPS is not available because PyTorch was not built with MPS enabled." ) else: logger.error( "MPS is not available because the current macOS version is not 12.3+, " "and/or you do not have an MPS-enabled device on this machine." ) exit(1) else: device = torch.device("mps") logger.info("✅ MPS backend is available. Using MPS device.") # Custom Optimizer: GrokAdamW class GrokAdamW(torch.optim.Optimizer): def __init__( self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-2, alpha_init=0.98, lamb=2.0, gamma=0.1, grokking_signal_fns=None, grokking_signal_decay_rate=0.1, gradient_clipping=1.0, ): defaults = { "lr": lr, "betas": betas, "eps": eps, "weight_decay": weight_decay, "alpha_init": alpha_init, "lamb": lamb, "gamma": gamma, "grokking_signal_fns": grokking_signal_fns, "grokking_signal_decay_rate": grokking_signal_decay_rate, "gradient_clipping": gradient_clipping, } super(GrokAdamW, self).__init__(params, defaults) def step(self, closure=None): loss = None if closure is not None: with torch.enable_grad(): loss = closure() for group in self.param_groups: grokking_signal = self._compute_grokking_signal(group) for i, p in enumerate(group["params"]): if p.grad is None: continue grad = p.grad if group["gradient_clipping"] > 0: grad = torch.clamp( grad, -group["gradient_clipping"], group["gradient_clipping"] ) state = self.state[p] if len(state) == 0: state["step"] = 0 state["exp_avg"] = torch.zeros_like( p, memory_format=torch.preserve_format ) state["exp_avg_sq"] = torch.zeros_like( p, memory_format=torch.preserve_format ) state["grok_ema"] = torch.zeros_like( p, memory_format=torch.preserve_format ) exp_avg, exp_avg_sq, grok_ema = ( state["exp_avg"], state["exp_avg_sq"], state["grok_ema"], ) beta1, beta2 = group["betas"] state["step"] += 1 layer_beta1 = beta1 * (1 - group["gamma"]) ** i alpha = group["alpha_init"] * torch.exp( torch.tensor( -group["grokking_signal_decay_rate"] * grokking_signal ) ) grok_ema.mul_(alpha).add_(grad, alpha=1 - alpha) grok_grad = grad + group["lamb"] * grok_ema exp_avg.mul_(layer_beta1).add_(grok_grad, alpha=1 - layer_beta1) exp_avg_sq.mul_(beta2).addcmul_( grok_grad, grok_grad, value=1 - beta2 ) denom = exp_avg_sq.sqrt().add_(group["eps"]) step_size = group["lr"] if group["weight_decay"] != 0: p.data.mul_(1 - group["lr"] * group["weight_decay"]) p.addcdiv_(exp_avg, denom, value=-step_size) return loss def _compute_grokking_signal(self, group): if group["grokking_signal_fns"] is None: return 0.0 signals = [] for fn in group["grokking_signal_fns"]: try: signal = fn() if signal is not None: signals.append(signal) except Exception as e: logger.warning( f"Error in grokking_signal_fn: {e}. Ignoring this function." ) if not signals: return 0.0 return sum(signals) / len(signals) # Data Formatting Function def format_my_prompts(examples): texts = [] for example in examples: formatted_text = "" if "system" in example and example["system"]: formatted_text += f"{example['system']}\n\n" for turn in example["conversations"]: if turn["from"] == "human": formatted_text += f"Human: {turn['value']}\n\n" elif turn["from"] == "gpt": formatted_text += f"Assistant: {turn['value']}\n\n" texts.append(formatted_text.strip()) return {"text": texts} # Custom Trainer Class class CustomTrainer(Trainer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.grokking_signal = 0.0 def compute_loss(self, model, inputs, return_outputs=False): labels = inputs.pop("labels") outputs = model(**inputs) logits = outputs.logits shift_logits = logits[..., :-1, :].contiguous() shift_labels = labels[..., 1:].contiguous() loss_fct = nn.CrossEntropyLoss() loss = loss_fct( shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1), ) return (loss, outputs) if return_outputs else loss def training_step(self, model, inputs): model.train() inputs = self._prepare_inputs(inputs) with autocast(dtype=torch.bfloat16): loss = self.compute_loss(model, inputs) if self.args.gradient_accumulation_steps > 1: loss = loss / self.args.gradient_accumulation_steps loss.backward() self.grokking_signal = loss.item() return loss.detach() def grokking_signal_fn(): return trainer.grokking_signal def main(): logger.info(f"🚀 Initializing finetuning with GrokAdamW using model from {MODEL_PATH}") # Load model and tokenizer from local directory try: config = AutoConfig.from_pretrained(MODEL_PATH) tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH) model = AutoModelForCausalLM.from_pretrained( MODEL_PATH, torch_dtype=torch.bfloat16 ) model.to(device) except Exception as e: logger.error(f"❌ Failed to load model or tokenizer: {str(e)}") return # Ensure pad token is set if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token model.config.pad_token_id = model.config.eos_token_id logger.info("📚 Loading local dataset") # Load the local JSONL dataset try: data_files = {"train": DATASET_PATH} dataset = load_dataset("json", data_files=data_files, split="train") dataset = dataset.map( format_my_prompts, batched=True, remove_columns=dataset.column_names, ) except Exception as e: logger.error(f"❌ Failed to load local dataset: {str(e)}") return logger.info(f"📊 Dataset size: {len(dataset)}") # Tokenization Function def tokenize_function(examples): return tokenizer( examples["text"], truncation=True, padding="max_length", max_length=MAX_LENGTH, ) logger.info("🔢 Tokenizing dataset") tokenized_dataset = dataset.map( tokenize_function, batched=True, remove_columns=["text"], ) # Define Training Arguments logger.info("🏋️ Setting up the training arguments") training_args = TrainingArguments( output_dir=OUTPUT_DIR, num_train_epochs=3, per_device_train_batch_size=BATCH_SIZE, gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS, learning_rate=LEARNING_RATE, weight_decay=0.01, bf16=True, # Use bfloat16 for MPS logging_steps=10, save_steps=300, save_total_limit=10, dataloader_num_workers=4, warmup_steps=NUM_WARMUP_STEPS, gradient_checkpointing=True, evaluation_strategy="steps", eval_steps=300, max_steps=MAX_STEPS, fp16=False, optim="adamw_hf", lr_scheduler_type="cosine", load_best_model_at_end=True, metric_for_best_model="loss", greater_is_better=False, ) # Data Collator data_collator = DataCollatorForLanguageModeling( tokenizer=tokenizer, mlm=False ) # Initialize Optimizer optimizer = GrokAdamW( model.parameters(), lr=LEARNING_RATE, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01, alpha_init=0.98, lamb=2.0, gamma=0.1, grokking_signal_fns=[grokking_signal_fn], grokking_signal_decay_rate=0.1, gradient_clipping=1.0, ) # Initialize Trainer logger.info("🏃♂️ Initializing Trainer with GrokAdamW") global trainer trainer = CustomTrainer( model=model, args=training_args, train_dataset=tokenized_dataset, eval_dataset=tokenized_dataset.select( range(min(1000, len(tokenized_dataset))) ), data_collator=data_collator, optimizers=(optimizer, None), ) # Start Training logger.info("🔥 Starting the training with GrokAdamW") try: trainer.train() except Exception as e: logger.error(f"❌ Training failed: {str(e)}") return # Save the Model logger.info("💾 Saving the model") try: trainer.save_model(OUTPUT_DIR) except Exception as e: logger.error(f"❌ Failed to save model: {str(e)}") logger.info("🎉 Finetuning with GrokAdamW completed!") if __name__ == "__main__": main()Key Adaptations and Explanations
-
MPS Device Configuration:
- Check for MPS Availability: The script first verifies if the MPS backend is available. If not, it logs an error and exits.
- Set Device: If available, it sets the device to MPS (
torch.device("mps")) and moves the model to this device.
-
Loading Local Model and Tokenizer:
- Model Path: The model is loaded from the local directory
/Users/n/qwen2-14B/usingAutoConfig,AutoTokenizer, andAutoModelForCausalLM. - Data Type: The model is loaded with
torch.bfloat16precision, which is supported by MPS on Apple Silicon. - Device Transfer: After loading, the model is moved to the MPS device with
model.to(device).
- Model Path: The model is loaded from the local directory
-
Loading and Formatting Local Dataset:
- Dataset Path: The script loads the local JSONL dataset from
/Users/n/dllama/sharegpt_cleaned_dataset.jsonl. - Data Formatting: It defines a
format_my_promptsfunction to convert the dataset into the required format, handling "system" prompts and conversations between "human" and "gpt". - Tokenization: The dataset is tokenized with padding and truncation to a
MAX_LENGTHof 2048 tokens.
- Dataset Path: The script loads the local JSONL dataset from
-
Custom Optimizer (
GrokAdamW):- Remains unchanged but ensures compatibility with the MPS device.
-
Training Arguments:
- Mixed Precision: Sets
bf16=Trueto utilize bfloat16 precision with MPS. - Other Hyperparameters: Retains the original training hyperparameters but tailored for the local environment.
- Mixed Precision: Sets
-
Trainer Initialization:
- Custom Trainer: Uses the
CustomTrainerclass to integrate the custom loss computation and grokking signal. - Optimizer: Initializes
GrokAdamWwith the required parameters and links it to the trainer. - Evaluation Dataset: Selects a subset of the training dataset for evaluation (
min(1000, len(tokenized_dataset))).
- Custom Trainer: Uses the
-
Error Handling and Logging:
- Comprehensive logging is included to track the progress and catch any errors during loading, training, and saving.
Additional Notes
-
Environment Requirements:
- Ensure that you have a version of PyTorch built with MPS support. You can install it via:
Make sure to follow the official PyTorch instructions for Macos with MPS support.pip install torch torchvision torchaudio
- Ensure that you have a version of PyTorch built with MPS support. You can install it via:
-
Dataset Format:
- The script assumes that each line in your
sharegpt_cleaned_dataset.jsonlfollows the ShareGPT format:
Adjust the{ "conversations": [ { "from": "human", "value": "user instruction" }, { "from": "gpt", "value": "model response" } ], "system": "optional system prompt", "tools": "optional tool description" }format_my_promptsfunction if your dataset deviates from this structure.
- The script assumes that each line in your
-
Model Compatibility:
- Ensure that the model located at
/Users/n/qwen2-14B/contains all necessary files (config.json,pytorch_model.bin, tokenizer files, etc.) required by HuggingFace'sAutoModelForCausalLM.
- Ensure that the model located at
-
Performance Considerations:
- The batch size (
BATCH_SIZE = 8) andMAX_LENGTH = 2048are set based on typical GPU memory constraints. You might need to adjust these values depending on your Mac Studio's RAM and performance during training.
- The batch size (
-
Saving Checkpoints:
- The script saves checkpoints every 300 steps and keeps a maximum of 10 checkpoints (
save_steps=300,save_total_limit=10). Modify these parameters inTrainingArgumentsas needed.
- The script saves checkpoints every 300 steps and keeps a maximum of 10 checkpoints (
-
Evaluation Strategy:
- The evaluation dataset is a subset of the training data. For more robust evaluation, consider using a separate validation set.
Running the Script
-
Ensure Dependencies Are Installed:
pip install torch torchvision torchaudio transformers datasets -
Execute the Script:
python path_to_your_script.pyReplace
path_to_your_script.pywith the actual path where you saved the script. -
Monitoring Training:
- The script logs progress and any potential issues. Monitor the console output to track training status.
-
Post-Training:
- After successful training, the fine-tuned model is saved in the directory specified by
OUTPUT_DIR(./finetuned_results).
- After successful training, the fine-tuned model is saved in the directory specified by
Troubleshooting
-
MPS Not Available:
- Ensure you're running on macOS 12.3 or later.
- Verify that your PyTorch installation supports MPS.
- Confirm that your Mac Studio has an MPS-capable GPU.
-
Memory Issues:
- If you encounter out-of-memory errors, consider reducing the
BATCH_SIZEorMAX_LENGTH.
- If you encounter out-of-memory errors, consider reducing the
-
Dataset Loading Errors:
- Double-check the path to your dataset and ensure it's in the correct JSONL format.
Feel free to adjust the script further based on your specific requirements and system capabilities. Happy training!
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.