What is the best way to batch some code like this? from transformers import…
- User
What is the best way to batch some code like this? from transformers import AutoModelForSequenceClassification, AutoTokenizer from tabulate import tabulate import torch import torch.nn.functional as F
Load the model and tokenizer from the Hugging Face Hub
model_name = "qingy2024/ActionBERT-109M" model = AutoModelForSequenceClassification.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)
text = "A mushroom exploding in a forest."
inputs = tokenizer(text, truncation=True, max_length=512, padding=True, return_tensors="pt")
with torch.no_grad(): outputs = model(**inputs) action_probability, no_action_probability = F.softmax(outputs.logits[0], dim = 0).tolist()
table = [ ['Probability of Action', action_probability], ['Probability of No Action', no_action_probability] ] print(tabulate(table, tablefmt='rounded_grid'))For example if a HF dataset had a column called "name" and I wanted to run the model on all of the "name"s and add a new column called "classification"?
Assistant
Here's how you can modify your code to batch process a HuggingFace dataset:
from transformers import AutoModelForSequenceClassification, AutoTokenizer from datasets import load_dataset import torch import torch.nn.functional as F from tqdm import tqdm # Load the model and tokenizer model_name = "qingy2024/ActionBERT-109M" model = AutoModelForSequenceClassification.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) # Load your dataset (replace with your actual dataset) dataset = load_dataset("your_dataset_name")["train"] # or whatever split you need def classify_batch(batch, batch_size=16): """Process a batch of texts and return classifications""" texts = batch["name"] # assuming "name" is your text column # Tokenize the batch inputs = tokenizer( texts, truncation=True, max_length=512, padding=True, return_tensors="pt" ) with torch.no_grad(): outputs = model(**inputs) # Get probabilities for each text in the batch probabilities = F.softmax(outputs.logits, dim=1) # Extract action probabilities (assuming index 0 is action) action_probs = probabilities[:, 0].tolist() # You can return either the probability or a binary classification # For binary classification based on threshold (e.g., 0.5): classifications = ["action" if prob > 0.5 else "no_action" for prob in action_probs] # Or return the actual probabilities: # classifications = action_probs return {"classification": classifications} # Apply the function to your dataset in batches dataset_with_classifications = dataset.map( classify_batch, batched=True, batch_size=16, # Adjust based on your GPU memory desc="Classifying texts" ) # Now your dataset has a new "classification" column print(dataset_with_classifications[0]) # See the first exampleIf you want to be more memory-efficient and handle larger datasets, you can also process in smaller chunks:
def classify_texts_chunked(dataset, chunk_size=1000): """Process dataset in chunks to manage memory""" results = [] for i in tqdm(range(0, len(dataset), chunk_size)): chunk = dataset.select(range(i, min(i + chunk_size, len(dataset)))) classified_chunk = chunk.map( classify_batch, batched=True, batch_size=16 ) results.append(classified_chunk) # Concatenate all chunks from datasets import concatenate_datasets return concatenate_datasets(results) # Use the chunked approach for very large datasets dataset_with_classifications = classify_texts_chunked(dataset)For even more detailed results, you could store both probabilities:
def classify_batch_detailed(batch): texts = batch["name"] inputs = tokenizer(texts, truncation=True, max_length=512, padding=True, return_tensors="pt") with torch.no_grad(): outputs = model(**inputs) probabilities = F.softmax(outputs.logits, dim=1) action_probs = probabilities[:, 0].tolist() no_action_probs = probabilities[:, 1].tolist() # Return detailed classification info classifications = ["action" if ap > nap else "no_action" for ap, nap in zip(action_probs, no_action_probs)] return { "classification": classifications, "action_probability": action_probs, "no_action_probability": no_action_probs }Key benefits of this batched approach:
- Much faster than processing one text at a time
- More memory efficient than loading everything at once
- Progress tracking with tqdm
- GPU utilization is optimized with proper batch sizes
Adjust the
batch_sizeparameter based on your available GPU memory - start with 16 and increase if you have more memory available.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.