Hi, I've implemented a AlexNet Model and a trainmodel() function to train my alexNet.…
- User
Hi, I've implemented a AlexNet Model and a train_model() function to train my alexNet. But I'm getting the following error : --------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) in <cell line: 89>() 87 88 alexnet = AlexNet() ---> 89 train_losses, train_accuracies, test_losses, test_accuracies = train_model( 90 alexnet, train_loader, test_loader, epochs=5, learning_rate=0.01, optimization_technique='early_stopping', patience=10, num_batches=10)
4 frames /usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py in forward(self, input) 112 113 def forward(self, input: Tensor) -> Tensor: --> 114 return F.linear(input, self.weight, self.bias) 115 116 def extra_repr(self) -> str:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (32x3 and 9216x4096). Here is my entire code:
Import the necessary libraries
import zipfile import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns import os, cv2 import itertools import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader, Dataset from torchvision import transforms from PIL import Image from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split
Folder names in the extracted zip file
folder_names = ['dogs', 'food', 'vehicles']
Define AlexNet and ImageDataset classes
from torch.optim.lr_scheduler import ReduceLROnPlateau
class AlexNet(nn.Module): def init(self): super(AlexNet, self).init()
self.features = nn.Sequential( nn.Conv2d(3, 96, kernel_size=11, stride=4), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2), nn.Conv2d(96, 256, kernel_size=5, padding=2), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2), nn.Conv2d(256, 384, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv2d(384, 384, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv2d(384, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2), ) self.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Linear(256 * 6 * 6, 4096), nn.ReLU(inplace=True), nn.Dropout(p=0.5), nn.Linear(4096, 4096), nn.ReLU(inplace=True), nn.Linear(4096, len(folder_names)), ) def forward(self, x): x = self.features(x) x = x.view(x.size(0), 256 * 6 * 6) x = self.classifier(x) return xclass ImageDataset(Dataset): def init(self, folder_names, transform=None): self.folder_names = folder_names self.transform = transform self.y = self.get_labels()
def get_labels(self): labels = [] for i, folder in enumerate(self.folder_names): labels.extend([i] * len(os.listdir(folder))) return labels def __len__(self): return sum(len(os.listdir(folder)) for folder in self.folder_names) def __getitem__(self, idx): for i, folder_name in enumerate(self.folder_names): if idx < len(os.listdir(folder_name)): file = os.listdir(folder_name)[idx] img = Image.open(os.path.join(folder_name, file)) if self.transform: img = self.transform(img) label = i return img, label idx -= len(os.listdir(folder_name))from torchvision import transforms
Preprocessing transform
transform = transforms.Compose([ transforms.Resize((227, 227)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])
Initialize dataset and dataloader
batch_size = 32 image_dataset = ImageDataset(folder_names, transform) data_loader = DataLoader(image_dataset, batch_size=batch_size, shuffle=True)
Split indices into train and test
all_indices = np.arange(len(image_dataset))
train_indices, test_indices = train_test_split(all_indices, test_size=0.2, random_state=42, stratify=image_dataset.y)
print(len(train_indices.tolist())) print(len(test_indices.tolist()))
def find_features_output_size(): alexnet = AlexNet() test_input = torch.randn(32, 3, 227, 227) output = alexnet.features(test_input) print('Output Size after Features:', output.size())
find_features_output_size()
def train_model(model, train_loader, test_loader, epochs = None, learning_rate = None, optimization_technique = None, patience=None, scheduler_patience=None,num_batches = None, **kwargs): criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=learning_rate)
if optimization_technique == 'learning_rate_scheduler' and scheduler_patience: scheduler = ReduceLROnPlateau(optimizer, 'min', patience=scheduler_patience, verbose=True) train_losses = [] train_accuracies = [] test_losses = [] test_accuracies = [] best_loss = float('inf') stopping_counter = 0 with_dropout = model.classifier[6] without_dropout = nn.Sequential(*[layer for layer in model.classifier if layer != with_dropout]) for epoch in range(epochs): running_train_loss = 0.0 running_train_acc = 0 num_batches_train = 0 for X_batch, y_batch in itertools.islice(train_loader, 0, num_batches): # for X_batch, y_batch in train_loader: optimizer.zero_grad() y_pred = model(X_batch) loss = criterion(y_pred, y_batch) loss.backward() optimizer.step() running_train_loss += loss.item() running_train_acc += accuracy_score(y_batch.numpy(), y_pred.argmax(dim=1).numpy()) num_batches_train += 1 train_losses.append(running_train_loss / num_batches_train) train_accuracies.append(running_train_acc / num_batches_train) # Testing segment running_test_loss = 0.0 running_test_acc = 0 num_batches_test = 0 with torch.no_grad(): for X_batch, y_batch in itertools.islice(test_loader, 0, num_batches): # with torch.no_grad(): # for X_batch, y_batch in test_loader: y_pred = model(X_batch) #y_pred_without_dropout = y_pred.view(y_pred.size(0), 256 * 6 * 6) y_pred_without_dropout = without_dropout(y_pred) # Use the classifier without dropout for testing # y_pred = y_pred.view(y_pred.size(0), 256 * 6 * 6) # y_pred = without_dropout(y_pred) # Use the classifier without dropout for testing loss = criterion(y_pred_without_dropout, y_batch) running_test_loss += loss.item() running_test_acc += accuracy_score(y_batch.numpy(), y_pred_without_dropout.argmax(dim=1).numpy()) num_batches_test += 1 test_losses.append(running_test_loss / num_batches_test) test_accuracies.append(running_test_acc / num_batches_test) # Early stopping if optimization_technique == 'early_stopping' and patience: if test_losses[-1] < best_loss: best_loss = test_losses[-1] stopping_counter = 0 else: stopping_counter += 1 if stopping_counter > patience: print(f"Early stopping at epoch {epoch+1}/{epochs}") break # Learning rate scheduler if optimization_technique == 'learning_rate_scheduler' and scheduler_patience and scheduler: scheduler.step(test_losses[-1]) if epoch % 10 == 0: print(f"Epoch: {epoch+1}/{epochs}, Training Loss: {train_losses[-1]}, Test Loss: {test_losses[-1]}, Training Accuracy: {train_accuracies[-1]}, Test Accuracy: {test_accuracies[-1]}") return train_losses, train_accuracies, test_losses, test_accuraciesTrain the model
train_loader = DataLoader(image_dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(train_indices)) test_loader = DataLoader(image_dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(test_indices))
alexnet = AlexNet() train_losses, train_accuracies, test_losses, test_accuracies = train_model( alexnet, train_loader, test_loader, epochs=5, learning_rate=0.01, optimization_technique='early_stopping', patience=10, num_batches=10)
- Assistant
The main issue here is that the reshaping of the output tensor before passing it to the
classifierin theforward()function ofAlexNetis not done correctly. Instead of reshaping tensor to (32, 256 * 6 * 6), reshaping it to the correct shape which is (32, 9216) should be done.To fix this issue, change the line in
forward()function:x = x.view(x.size(0), 256 * 6 * 6)to
x = x.view(x.size(0), -1)The
-1in theviewfunction will automatically calculate the appropriate size to reshape the tensor in that dimension.Here's the modified
forward()function:def forward(self, x): x = self.features(x) x = x.view(x.size(0), -1) x = self.classifier(x) return xAfter making this change, you should not receive the
RuntimeErrorabout the mismatched tensor shapes. - User
I commented all the following lines :
#y_pred_without_dropout = y_pred.view(y_pred.size(0), 256 * 6 * 6) #y_pred_without_dropout = without_dropout(y_pred) # Use the classifier without dropout for testing # y_pred = y_pred.view(y_pred.size(0), 256 * 6 * 6) # y_pred = without_dropout(y_pred) # Use the classifier without dropout for testingand only used these lines y_pred = model(X_batch); loss = criterion(y_pred, y_batch) inside torch.no_grad() within train_model() function. After this, The code is now working just fine. So, Is this a good approach or not? if not, provide me the best alternative.
- Assistant
It's a good approach if you simply want to test the model's performance on the test dataset without worrying about dropout. In this case, you're using the dropout layers for training to avoid overfitting, but during the test phase, you're allowing the estimated test performance to include dropout.
Generally, during the test phase, it's common to turn off dropout layers by setting the model to evaluation mode, so that the estimated performance is a better reflection of the model's true performance.
To do this, you can set the model to evaluation mode inside the
torch.no_grad()block within thetrain_model()function:with torch.no_grad(): model.eval() # Sets the model to evaluation mode # Testing segment ... model.train() # Sets the model back to train modeBy doing this, you can still use
y_pred = model(X_batch)inside thetorch.no_grad()block without worrying about dropout layers affecting test performance.Here's the modified
train_model()function with a model evaluation mode:def train_model(model, train_loader, test_loader, epochs = None, learning_rate = None, optimization_technique = None, patience=None, scheduler_patience=None,num_batches = None, **kwargs): ... for epoch in range(epochs): ... # Testing segment with torch.no_grad(): model.eval() # Sets the model to evaluation mode for X_batch, y_batch in itertools.islice(test_loader, 0, num_batches): y_pred = model(X_batch) loss = criterion(y_pred, y_batch) running_test_loss += loss.item() running_test_acc += accuracy_score(y_batch.numpy(), y_pred.argmax(dim=1).numpy()) num_batches_test += 1 model.train() # Sets the model back to train mode ... return train_losses, train_accuracies, test_losses, test_accuraciesThis should let you evaluate the model on the test dataset without taking dropout into account, and it's a cleaner approach compared to manually removing dropout layers from the classifier.