Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0301· openaiHugging Face Suzhen/CodeChat

import torch from transformers import AutoTokenizer,…

  1. User

    import torch from transformers import AutoTokenizer, AutoModelForSequenceClassification,AutoModel, AutoTokenizer

    Define the model repo

    model_name = "cointegrated/rubert-tiny" #"blanchefort/rubert-base-cased-sentiment"

    Download pytorch model

    model = AutoModel.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)

    Загрузка модели и токенизатора

    #model_name = 'DeepPavlov/rubert-base-cased-sentiment' #tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForSequenceClassification.from_pretrained(model_name) как нужно прокачать голову,чтобы можно было провести многоклассовую классификацию с помощью данной модели вот такого dataseta oid category text 0 365271984 winter_sport Волшебные фото Виктория Поплавская ЕвгенияМедв... 1 503385563 extreme Возвращение в подземелье Треша 33 Эйфория тупо... 2 146016084 football Лучшие чешские вратари – Доминик Доминатор Гаш...

  2. Assistant

    To perform multi-class classification using the given model on the dataset with columns oid, category and text, we can follow these steps:

    1. Load the dataset using pandas
    2. Preprocess the text data by tokenizing it and encoding it with the tokenizer
    3. Define the labels and encode them using a LabelEncoder from scikit-learn
    4. Split the dataset into train and test set
    5. Define the training parameters and optimizer
    6. Train the model on the training set
    7. Evaluate the model on the test set using accuracy and other metrics.

    Here is a sample code for performing multi-class classification using the given model:

    import pandas as pd
    import numpy as np
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import LabelEncoder
    import torch
    from torch.utils.data import DataLoader, Dataset
    from transformers import AutoTokenizer, AutoModelForSequenceClassification, AdamW
    
    # define the model repo
    model_name = "cointegrated/rubert-tiny"
    
    # load the dataset
    df = pd.read_csv('dataset.csv')
    
    # preprocess the text data
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    
    class CustomDataset(Dataset):
        def __init__(self, texts, labels, tokenizer, max_length):
            self.texts = texts
            self.labels = labels
            self.tokenizer = tokenizer
            self.max_length = max_length
        
        def __len__(self):
            return len(self.texts)
        
        def __getitem__(self, idx):
            text = self.texts[idx]
            label = self.labels[idx]
            
            # tokenize the text using the tokenizer
            encoding = self.tokenizer.encode_plus(
                text,
                add_special_tokens=True,
                max_length=self.max_length,
                pad_to_max_length=True,
                return_tensors='pt'
            )
            
            return {
                'input_ids': encoding['input_ids'].squeeze(),
                'attention_mask': encoding['attention_mask'].squeeze(),
                'label': torch.tensor(label, dtype=torch.long)
            }
    
    # encode the labels
    le = LabelEncoder()
    df['label'] = le.fit_transform(df['category'])
    
    # split the dataset into train and test set
    X_train, X_test, y_train, y_test = train_test_split(df['text'], df['label'], test_size=0.2, random_state=42)
    
    # create the dataloaders
    train_dataset = CustomDataset(X_train.values, y_train.values, tokenizer, max_length=512)
    test_dataset = CustomDataset(X_test.values, y_test.values, tokenizer, max_length=512)
    
    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
    
    # define the model and its optimizer
    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=len(le.classes_))
    optimizer = AdamW(model.parameters(), lr=2e-5)
    
    # training loop
    num_epochs = 5
    for epoch in range(num_epochs):
        model.train()
        for batch in train_loader:
            inputs = {
                'input_ids': batch['input_ids'].to(device),
                'attention_mask': batch['attention_mask'].to(device),
                'labels': batch['label'].to(device)
            }
            loss, _ = model(**inputs)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
        
        model.eval()
        with torch.no_grad():
            total_correct = 0
            total_count = 0
            for batch in test_loader:
                inputs = {
                    'input_ids': batch['input_ids'].to(device),
                    'attention_mask': batch['attention_mask'].to(device),
                    'labels': batch['label'].to(device)
                }
                _, outputs = model(**inputs)
                _, predicted = torch.max(outputs, 1)
                total_correct += (predicted == batch['label'].to(device)).sum().item()
                total_count += len(batch)
            
            accuracy = total_correct / total_count
            
            print(f'Epoch {epoch}, Accuracy: {accuracy}')
    

    Note that the code above assumes that the text data is in a column named 'text', and the labels are in a column named 'category', both stored in a CSV file named 'dataset.csv'. You may need to adjust the code accordingly to match your data format.

    Also, note that the code above is just a sample and may need further modifications and tuning to work with your specific use case.

  3. User

    create the dataloaders

    train_dataset = CustomDataset(X_train.values, y_train.values, tokenizer, max_length=512) TypeError: CustomDataset() takes no arguments

  4. Assistant

    The CustomDataset class you defined does not take any arguments in its constructor function. However, when creating the train_dataset object, you are passing 4 arguments: X_train.values, y_train.values, tokenizer, and max_length.

    To fix this, you need to modify the CustomDataset class to accept these additional arguments as follows:

    class CustomDataset(Dataset):
        def __init__(self, texts, labels, tokenizer, max_length):
            self.texts = texts
            self.labels = labels
            self.tokenizer = tokenizer
            self.max_length = max_length
        
        def __len__(self):
            return len(self.texts)
        
        def __getitem__(self, idx):
            text = self.texts[idx]
            label = self.labels[idx]
            
            # tokenize the text using the tokenizer
            encoding = self.tokenizer.encode_plus(
                text,
                add_special_tokens=True,
                max_length=self.max_length,
                pad_to_max_length=True,
                return_tensors='pt'
            )
            
            return {
                'input_ids': encoding['input_ids'].squeeze(),
                'attention_mask': encoding['attention_mask'].squeeze(),
                'label': torch.tensor(label, dtype=torch.long)
            }
    

    This modified CustomDataset class now takes 4 arguments in its constructor function: texts, labels, tokenizer, and max_length. The rest of the class implementation remains the same as before.

    After modifying the CustomDataset class, you can create the dataloaders as follows:

    # create the dataloaders
    train_dataset = CustomDataset(X_train.values, y_train.values, tokenizer, max_length=512)
    test_dataset = CustomDataset(X_test.values, y_test.values, tokenizer, max_length=512)
    
    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
    

    This should resolve the TypeError you encountered earlier.