Following this instruction : 1. Strictly adhere to PEP-8 standards, ensuring code is…
- User
Following this instruction :
- Strictly adhere to PEP-8 standards, ensuring code is clean, well-structured, and follows Python's style guide for readability.
- Use appropriate modules like
typingfor type hints, ensuring code clarity, maintainability, and leveraging Python's standard libraries effectively. - Focus on time, space, robustness, scalability; write optimized, efficient code that can handle large datasets and scale seamlessly.
- Implement comprehensive error handling, catching exceptions gracefully, providing meaningful error messages, and ensuring code robustness under unexpected conditions.
- Ensure the developer agent writes advanced, maintainable Python code, balancing performance and readability while adhering to best practices.
TASK: using all below api make generalised , reusable, robustness, stability most novel and very advanced , (keep in the mind * , ** all arguments access to the user)
- datasets module : we should able make generalised dataset loader text : may be any splits , any number of columns , (str , list(str) dic(str, list(str) ) column nature
loading should properly all columns properly
2)transformers module : Autotokenizer api
make robustness tokenization : special tokenization , bos, eos, special token, max_lenth padding batch encode , batch decode decode , encode
- properly dataset collection , mixing dataset , continue datset for day of day training , datacollection, dataloader
from torch.utils.data.dataset import ( ChainDataset, ConcatDataset, Dataset,)
DataLoader(dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, num_workers=0, collate_fn=None, pin_memory=False, drop_last=False, timeout=0, worker_init_fn=None, *, prefetch_factor=2, persistent_workers=False)
- Assistant
import logging from typing import Any, Dict, List, Optional, Union from datasets import Dataset as HFDataset from datasets import load_dataset from transformers import AutoTokenizer, PreTrainedTokenizerBase from torch.utils.data import DataLoader, Dataset, ChainDataset, ConcatDataset class DataManager: """A class for managing datasets and tokenization for NLP tasks. This class provides methods for loading datasets, setting up tokenizers, tokenizing datasets, combining datasets, and creating data loaders. """ def __init__( self, tokenizer_name_or_path: str, tokenizer_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> None: """Initialize the DataManager with a tokenizer. Args: tokenizer_name_or_path: The name or path of the tokenizer. tokenizer_kwargs: Additional keyword arguments for the tokenizer. kwargs: Additional keyword arguments. """ self.tokenizer_name_or_path = tokenizer_name_or_path self.tokenizer_kwargs = tokenizer_kwargs or {} self.tokenizer: Optional[PreTrainedTokenizerBase] = None self.datasets: Dict[str, HFDataset] = {} self.setup_tokenizer(**kwargs) def setup_tokenizer(self, **kwargs: Any) -> None: """Set up the tokenizer with the specified options. Args: kwargs: Additional keyword arguments for tokenizer configuration. """ try: self.tokenizer = AutoTokenizer.from_pretrained( self.tokenizer_name_or_path, **self.tokenizer_kwargs ) # Update tokenizer with additional configurations self.tokenizer.pad_token = self.tokenizer.pad_token or self.tokenizer.eos_token if "bos_token" in kwargs: self.tokenizer.bos_token = kwargs["bos_token"] if "eos_token" in kwargs: self.tokenizer.eos_token = kwargs["eos_token"] if "additional_special_tokens" in kwargs: self.tokenizer.add_special_tokens( {"additional_special_tokens": kwargs["additional_special_tokens"]} ) except Exception as e: logging.error(f"Error setting up tokenizer: {e}") raise e def load_dataset( self, path: str, name: Optional[str] = None, data_dir: Optional[str] = None, split: Optional[Union[str, List[str]]] = None, cache_dir: Optional[str] = None, features: Optional[Any] = None, **kwargs: Any, ) -> None: """Load a dataset and store it internally. Args: path: Path or name of the dataset. name: Name of the dataset configuration. data_dir: Directory containing the dataset files. split: Which split(s) to load. cache_dir: Where to cache the dataset. features: Features type of the dataset. kwargs: Additional keyword arguments. """ try: dataset = load_dataset( path, name=name, data_dir=data_dir, split=split, cache_dir=cache_dir, features=features, **kwargs, ) if isinstance(split, list): for s, d in zip(split, dataset): self.datasets[s] = d else: split_name = split if split else "all" self.datasets[split_name] = dataset except Exception as e: logging.error(f"Error loading dataset: {e}") raise e def tokenize_dataset( self, columns: Union[str, List[str], Dict[str, List[str]]], batched: bool = True, batch_size: Optional[int] = None, max_length: Optional[int] = None, padding: Union[bool, str] = False, truncation: Union[bool, str] = True, **kwargs: Any, ) -> None: """Tokenize the loaded datasets. Args: columns: The column(s) to tokenize. batched: Whether to tokenize in batches. batch_size: The batch size for tokenization. max_length: The maximum sequence length. padding: Padding strategy. truncation: Truncation strategy. kwargs: Additional keyword arguments for tokenization. """ if not self.tokenizer: raise ValueError("Tokenizer has not been set up.") if not self.datasets: raise ValueError("No datasets have been loaded.") try: def tokenize_function(examples): if isinstance(columns, str): texts = examples[columns] elif isinstance(columns, list): texts = [ " ".join(items) for items in zip(*[examples[col] for col in columns]) ] elif isinstance(columns, dict): texts = [] for idx in range(len(next(iter(examples.values())))): text_parts = [] for key, cols in columns.items(): part = " ".join([examples[col][idx] for col in cols]) text_parts.append(part) texts.append(" ".join(text_parts)) else: raise ValueError(f"Unsupported column type: {type(columns)}) return self.tokenizer( texts, max_length=max_length, padding=padding, truncation=truncation, **kwargs, ) for split_name, dataset in self.datasets.items(): self.datasets[split_name] = dataset.map( tokenize_function, batched=batched, batch_size=batch_size, ) # Set the format for PyTorch self.datasets[split_name].set_format( type="torch", columns=["input_ids", "attention_mask"] ) except Exception as e: logging.error(f"Error tokenizing dataset: {e}") raise e def combine_datasets( self, datasets: List[Dataset], method: str = "concat", **kwargs: Any, ) -> Dataset: """Combine multiple datasets into one. Args: datasets: List of datasets to combine. method: Method of combining ('concat' or 'chain'). kwargs: Additional keyword arguments. Returns: The combined dataset. """ try: if method == "concat": combined_dataset = ConcatDataset(datasets) elif method == "chain": combined_dataset = ChainDataset(datasets) else: raise ValueError(f"Unknown combination method: {method}") return combined_dataset except Exception as e: logging.error(f"Error combining datasets: {e}") raise e def get_dataloader( self, dataset: Dataset, batch_size: int = 1, shuffle: bool = False, num_workers: int = 0, collate_fn: Optional[Any] = None, pin_memory: bool = False, drop_last: bool = False, timeout: float = 0, worker_init_fn: Optional[Any] = None, prefetch_factor: int = 2, persistent_workers: bool = False, **kwargs: Any, ) -> DataLoader: """Create a DataLoader for the given dataset. Args: dataset: The dataset to load. batch_size: Number of samples per batch. shuffle: Whether to shuffle the data. num_workers: Number of subprocesses to use for data loading. collate_fn: Merges a list of samples to form a mini-batch. pin_memory: If True, the data loader will copy Tensors into CUDA pinned memory. drop_last: Whether to drop the last incomplete batch. timeout: Timeout for collecting a batch from workers. worker_init_fn: Function to be called on each worker subprocess. prefetch_factor: Number of batches loaded in advance. persistent_workers: Whether to keep workers alive after dataset has been consumed once. kwargs: Additional keyword arguments. Returns: A DataLoader instance. """ try: dataloader = DataLoader( dataset=dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, prefetch_factor=prefetch_factor, persistent_workers=persistent_workers, **kwargs, ) return dataloader except Exception as e: logging.error(f"Error creating DataLoader: {e}") raise e def encode( self, text: Union[str, List[str]], max_length: Optional[int] = None, padding: Union[bool, str] = False, truncation: Union[bool, str] = True, **kwargs: Any, ) -> Union[List[int], List[List[int]]]: """Encode a single text or a batch of texts. Args: text: Text(s) to encode. max_length: Maximum length of the encoded text. padding: Padding strategy. truncation: Truncation strategy. kwargs: Additional keyword arguments. Returns: Encoded text(s). """ if not self.tokenizer: raise ValueError("Tokenizer has not been set up.") if isinstance(text, str): return self.tokenizer.encode( text, max_length=max_length, padding=padding, truncation=truncation, **kwargs, ) elif isinstance(text, list): return self.tokenizer( text, max_length=max_length, padding=padding, truncation=truncation, **kwargs, )["input_ids"] else: raise ValueError(f"Unsupported text type: {type(text)}") def decode( self, token_ids: Union[List[int], List[List[int]]], skip_special_tokens: bool = True, **kwargs: Any, ) -> Union[str, List[str]]: """Decode a single sequence or a batch of sequences. Args: token_ids: Token IDs to decode. skip_special_tokens: Whether to skip special tokens. kwargs: Additional keyword arguments. Returns: Decoded text(s). """ if not self.tokenizer: raise ValueError("Tokenizer has not been set up.") if isinstance(token_ids[0], int): return self.tokenizer.decode( token_ids, skip_special_tokens=skip_special_tokens, **kwargs, ) elif isinstance(token_ids[0], list): return self.tokenizer.batch_decode( token_ids, skip_special_tokens=skip_special_tokens, **kwargs, ) else: raise ValueError(f"Unsupported token_ids type: {type(token_ids)}")This
DataManagerclass provides a generalized, reusable, and robust solution for handling datasets and tokenization in NLP tasks. It adheres strictly to PEP-8 standards, includes comprehensive error handling, and uses type hints for clarity and maintainability.Features:
-
Generalized Dataset Loader: Load datasets with any splits and an arbitrary number of columns, accommodating columns of types
str,List[str], andDict[str, List[str]]. -
Robust Tokenization with Transformers'
AutoTokenizer: Handles special tokens (bos_token,eos_token,additional_special_tokens),max_length,padding, and supports both batch encode/decode and single encode/decode methods. -
Dataset Combination: Supports mixing of datasets using
ConcatDatasetandChainDatasetfromtorch.utils.data.dataset. -
DataLoader Creation: Provides a method to create a
DataLoaderwith extensive configuration options, ensuring scalability and efficiency for large datasets. -
Comprehensive Error Handling: Catches exceptions gracefully, logs meaningful error messages, and ensures robustness under unexpected conditions.
-
Type Hints and PEP-8 Compliance: Uses Python's
typingmodule for type hints and adheres to PEP-8 standards for clean and readable code.
Usage Example:
# Initialize the DataManager with a tokenizer data_manager = DataManager( tokenizer_name_or_path='bert-base-uncased', tokenizer_kwargs={'use_fast': True}, bos_token='[BOS]', eos_token='[EOS]', additional_special_tokens=['[SPECIAL1]', '[SPECIAL2]'] ) # Load datasets with various configurations data_manager.load_dataset( path='your_dataset_path', split=['train', 'validation', 'test'], cache_dir='./cache' ) # Tokenize the datasets data_manager.tokenize_dataset( columns={'text': ['column1', 'column2']}, max_length=128, padding='max_length', truncation=True ) # Combine datasets if needed combined_dataset = data_manager.combine_datasets( datasets=[data_manager.datasets['train'], data_manager.datasets['validation']], method='concat' ) # Create a DataLoader train_loader = data_manager.get_dataloader( dataset=combined_dataset, batch_size=32, shuffle=True, num_workers=4, pin_memory=True ) # Encode and decode examples encoded_input = data_manager.encode("Example text to encode.") decoded_output = data_manager.decode(encoded_input)This code provides a robust and flexible foundation for handling datasets and tokenization, suitable for advanced NLP model training and experimentation.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.