Is there any way to speed up inference here? Should I be using vllm or something? Can't…
- User
Is there any way to speed up inference here? Should I be using vllm or something? Can't i do some stuff in parallel
from unsloth import FastLanguageModel import torch from datasets import Dataset import pandas as pd import json from tqdm import tqdm import argparse import wandb
from transformers import TrainingArguments from unsloth import is_bfloat16_supported from unsloth import UnslothTrainer, UnslothTrainingArguments
Add argument parsing
parser = argparse.ArgumentParser(description="Pretrain or evaluate Unsloth models") parser.add_argument("--model", type=str, default="unsloth/mistral-7b-v0.3", help="Model name or path to use") parser.add_argument("--save_model_name", type=str, help="Name to save the trained model as") parser.add_argument("--max_seq_length", type=int, default=2048, help="Maximum sequence length") parser.add_argument("--load_in_4bit", action="store_true", default=True, help="Use 4-bit quantization") parser.add_argument("--eval_only", action="store_true", help="Only run evaluation without training") parser.add_argument("--num_eval_samples", type=int, default=100, help="Number of samples to evaluate on")
args = parser.parse_args()
wandb.init(project="unsloth-pretraining-movies", name=args.save_model_name) wandb.config.update(args)
save_model_name = args.save_model_name if args.save_model_name else args.model.replace("/", "_") max_seq_length = args.max_seq_length dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit = args.load_in_4bit eval_only = args.eval_only
4bit pre quantized models we support for 4x faster downloading + no OOMs.
fourbit_models = [ "unsloth/mistral-7b-v0.3-bnb-4bit", # New Mistral v3 2x faster! "unsloth/mistral-7b-instruct-v0.3-bnb-4bit", "unsloth/llama-3-8b-bnb-4bit", # Llama-3 15 trillion tokens model 2x faster! "unsloth/llama-3-8b-Instruct-bnb-4bit", "unsloth/llama-3-70b-bnb-4bit", "unsloth/Phi-3-mini-4k-instruct", # Phi-3 2x faster! "unsloth/Phi-3-medium-4k-instruct", "unsloth/mistral-7b-bnb-4bit", "unsloth/gemma-7b-bnb-4bit", "unsloth/gemma-3-27b-it-unsloth-bnb-4bit" ] # More models at https://huggingface.co/unsloth
model, tokenizer = FastLanguageModel.from_pretrained( model_name = args.model, # Now using the model from command line args max_seq_length = max_seq_length, dtype = dtype, load_in_4bit = load_in_4bit, )
model = FastLanguageModel.get_peft_model( model, r = 128, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "embed_tokens", "lm_head",], # Add for continual pretraining lora_alpha = 32, lora_dropout = 0, # Supports any, but = 0 is optimized bias = "none", # Supports any, but = "none" is optimized use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context random_state = 3407, use_rslora = True, # We support rank stabilized LoRA loftq_config = None, # And LoftQ )
EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN db_prompt = """Database entry:
Row:
{}"""
def format_json(row): return "\n".join([f"{k}: {v}" for k, v in row.items()])
def formatting_prompts_func(input): text = db_prompt.format(format_json(input)) + EOS_TOKEN return { "text" : text }
read jsonl file of pretraining_data.jsonl
with open("pretraining_data.jsonl", "r") as f: records = [json.loads(line) for line in f]
dataset = [formatting_prompts_func(record) for record in records]
turn into a datasets
dataset = Dataset.from_pandas(pd.DataFrame(dataset))
if not eval_only: trainer = UnslothTrainer( model = model, tokenizer = tokenizer, train_dataset = dataset, dataset_text_field = "text", max_seq_length = max_seq_length, dataset_num_proc = 2,
args = UnslothTrainingArguments( per_device_train_batch_size = 16, gradient_accumulation_steps = 8, # Use warmup_ratio and num_train_epochs for longer runs! max_steps = 120, warmup_steps = 10, warmup_ratio = 0.1, num_train_epochs = 1, # Select a 2 to 10x smaller learning rate for the embedding matrices! learning_rate = 5e-5, embedding_learning_rate = 1e-5, fp16 = not is_bfloat16_supported(), bf16 = is_bfloat16_supported(), logging_steps = 1, optim = "adamw_8bit", weight_decay = 0.01, lr_scheduler_type = "linear", seed = 3407, output_dir = "outputs", report_to = "wandb", # Use this for WandB etc ),) trainer.train()
print("{"Title":"Deep Blue Sea"\n\n") print("Saving model...") model.save_pretrained(save_model_name) tokenizer.save_pretrained(save_model_name) print("Model saved to", save_model_name) else: print("Loading model...") model, tokenizer = FastLanguageModel.from_pretrained( model_name = save_model_name, # Load from locally saved model max_seq_length = max_seq_length, dtype = dtype, load_in_4bit = load_in_4bit, ) print("Model loaded from", save_model_name)
FastLanguageModel.for_inference(model) # Enable native 2x faster inference with open("pretraining_data_no_genre_gt.jsonl", "r") as f: no_genre_data = [json.loads(line) for line in f]
def convert(entry): format = """ Database entry:
Row:
Title: {title} """ return format.format(title = entry["Title"])
def text_to_dict(formatted_text): """ Convert the formatted database entry text back into a dictionary.
Example input: ''' Database entry: ### Row: Title: Deep Blue Sea Genre: Action, Sci-Fi Year: 1999 ''' """ # Extract just the data part if "### Row:" in formatted_text: data_part = formatted_text.split("### Row:")[1].strip() else: data_part = formatted_text lines = [line.strip() for line in data_part.split("\n") if line.strip()] result = {} for line in lines: if ":" in line: if line.startswith('"') and '": ' in line: key = line.split('": ')[0].strip('"') value = line.split('": ')[1].strip() else: parts = line.split(":", 1) key = parts[0].strip() value = parts[1].strip() if len(parts) > 1 else "" result[key] = value return resultimport unicodedata def super_normalize_text(text): """Ultra-aggressive text normalization handling quotes and edge cases""" if text is None: return "" text = str(text) for quote in ['\u2018','\u2019','\u201C','\u201D','\u2032','\u2033','\u2039','\u203A','\u00AB','\u00BB']: text = text.replace(quote, "'") text = text.replace('"', "'") text = ''.join(ch for ch in text if unicodedata.category(ch)[0] != 'C' or ch.isspace()) return unicodedata.normalize('NFKD', text).strip().lower()
def genre_correct(gt_value, pred_value): try: gt_genre = gt_value.split(", ") pred_genre = pred_value.split(", ") return set(gt_genre) == set(pred_genre) except: print(f"Error: {gt_value} {pred_value}") return False
def evaluate(gt_entry, pred_entry): gt_keys = gt_entry.keys() pred_keys = pred_entry.keys() extra_keys = [key for key in pred_keys if key not in gt_keys] results = {} for key in gt_keys: if key in pred_keys: gt_value = super_normalize_text(gt_entry[key]) if type(gt_entry[key]) == str else gt_entry[key] pred_value = super_normalize_text(pred_entry[key]) if type(pred_entry[key]) == str else pred_entry[key] if key == "Genre": if genre_correct(gt_value, pred_value): results[key] = "correct" else: results[key] = "incorrect" elif gt_value == pred_value: results[key] = "correct" else: results[key] = "incorrect" else: results[key] = "missing" num_correct = sum(value == "correct" for value in results.values()) num_incorrect = sum(value == "incorrect" for value in results.values()) num_missing = sum(value == "missing" for value in results.values()) return { "ground_truth": gt_entry, "predicted": pred_entry, "num_correct": num_correct, "num_incorrect": num_incorrect, "num_missing_keys": num_missing, "num_extra_keys": len(extra_keys), "results": results, "genre_correct": results["Genre"] == "correct", "accuracy": num_correct / len(gt_keys) }
results = [] for i in tqdm(range(len(no_genre_data)), desc="Generating outputs"): inputs = tokenizer( [ convert(no_genre_data[i]) ], return_tensors = "pt").to("cuda") outputs = model.generate(**inputs, max_new_tokens = 512, use_cache = True) full_output = tokenizer.batch_decode(outputs)[0] dict_output = text_to_dict(full_output) results.append(evaluate(no_genre_data[i], dict_output))
results_df = pd.DataFrame(results) results_df.to_json("results.jsonl", orient="records", lines=True)
wandb.log({ "eval_results": wandb.Table(dataframe=results_df), }) wandb.summary["eval_accuracy"] = results_df["accuracy"].mean() wandb.summary["eval_genre_correct"] = results_df["genre_correct"].mean() wandb.summary["eval_num_correct"] = results_df["num_correct"].mean() wandb.summary["eval_num_incorrect"] = results_df["num_incorrect"].mean() wandb.summary["eval_num_missing_keys"] = results_df["num_missing_keys"].mean() wandb.summary["eval_num_extra_keys"] = results_df["num_extra_keys"].mean()
Assistant
To speed up inference in your current setup, there are several strategies and tools you can consider. Here are the most relevant options:
1. Use vLLM for Faster Inference
vLLM is an open-source library designed to optimize large language model (LLM) inference. It achieves significant speedups through techniques like:
- PagedAttention: Efficient memory management that reduces GPU memory overhead and enables larger context windows.
- Continuous Batching: Dynamically merges incoming requests into ongoing batches, maximizing GPU utilization and reducing latency.
- Optimized CUDA Kernels: Enhances performance on high-end GPUs.
vLLM has demonstrated up to 24x throughput improvements compared to traditional frameworks like Hugging Face Transformers, making it a strong candidate for your use case[2][5][9].
2. Explore Parallelization Techniques
Inference parallelism can distribute the computational workload across multiple GPUs or processing units. Key methods include:
- Model Parallelism: Splits the model across GPUs to handle larger models that exceed single-GPU memory capacity.
- Pipeline Parallelism: Divides the model into stages, with each stage running on a different GPU.
- Data Parallelism: Processes multiple inputs simultaneously by replicating the model across GPUs[3].
3. Implement Speculative Decoding
Speculative decoding uses a smaller, faster draft model to generate token candidates in parallel, which are then verified by the larger model. This approach can significantly reduce the sequential nature of autoregressive token generation, speeding up inference without sacrificing accuracy[1][4][6].
4. Use Quantization and Pruning
- Quantization: Reduces the precision of model weights (e.g., from FP32 to INT8 or 4-bit), which decreases memory usage and speeds up computations.
- Pruning: Removes less significant connections in the model, reducing its size and computational requirements[4].
5. Consider Medusa for Multi-Token Prediction
Medusa introduces extra decoding heads to predict multiple tokens simultaneously, reducing the number of sequential steps required for token generation. This method can achieve a 2-3x speedup depending on the model and dataset[1].
6. Optimize Runtime Settings
- Batching: Increase batch sizes to maximize GPU utilization.
- Gradient Checkpointing: Reduces memory usage during training and inference by recomputing intermediate results as needed.
- Flash Attention: Optimizes attention mechanisms to improve speed and memory efficiency for large context windows[4].
7. Hardware Considerations
Ensure that your hardware is optimized for LLM inference:
- Use GPUs with high memory capacity (e.g., NVIDIA A100, H100).
- Leverage hardware-specific optimizations like Tensor Cores for mixed-precision calculations.
Recommendations for Your Code
- Replace the current inference engine with vLLM to benefit from its memory and throughput optimizations.
- Experiment with speculative decoding or Medusa if your application can tolerate slight modifications to the model's architecture.
- Enable continuous batching in vLLM to handle requests more efficiently.
- Quantize your model further if memory constraints are an issue.
By integrating these techniques, you can achieve significant speedups in inference while maintaining model quality.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.