AttributeError: 'NeuralNetClassifier' object has no attribute 'decision_function'

[Centrattic]

I run into this error when using the auc_roc skorch callback instead of accuracy although auc_roc should be implemented (it's an available score on the sklearn scoring page). When I try other scoring callbacks, specifically precision or recall, I receive the following error message:
AttributeError: 'NeuralNetClassifier' object has no attribute 'classes_inferred_'

Here's my full code, just in case. The relevant part is under the comment "Setting Model Parameters With Skorch"

import numpy as np
import pandas as pd
import cv2
from PIL import Image
import os
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, models
import torchvision.transforms.functional as tf
from torch.utils.data import random_split, ConcatDataset,DataLoader

from skorch import NeuralNetClassifier
from skorch.callbacks import LRScheduler, Checkpoint, EpochScoring, EarlyStopping
from skorch.dataset import Dataset
from skorch.helper import predefined_split

import warnings
warnings.filterwarnings('ignore')

os.chdir("/cluster/qtim/users/riya/race_ukbb")

class PretrainedModel(nn.Module):
    def __init__(self, output_features):
        super().__init__()
        model = models.resnet18(pretrained=True)
        num_ftrs = model.fc.in_features
        model.fc = nn.Linear(num_ftrs, output_features)
        self.model = model

    def forward(self, x):
        return self.model(x)
    
def train(data_dir, experiment_name,
          num_classes=2, batch_size=64, num_epochs=50, lr=0.001, image_size = (224, 224)):
    
    # USING GPU
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    if device == 'cuda:0':
        torch.cuda.empty_cache()
        
    # TRANSFORMING TRAIN/TEST DATA
    def train_loader(path):
        # transforms
        return tensor

    def test_loader(path):
         # transforms
        return tensor
    
    # APPENDING DATA TO DATA FOLDERS.
    train_folder = os.path.join(data_dir, 'train')
    test_folder = os.path.join(data_dir, 'test')
    
    train_dataset = datasets.ImageFolder(train_folder, loader = train_loader)
    test_dataset = datasets.ImageFolder(test_folder, loader = test_loader)
    
    # DEFINING VALIDATION SPLIT FOR 5-FOLD CROSS VALIDATION
    tl = np.round(len(train_dataset)/5) # manually done 5 fold
    print(len(train_dataset))
    print(tl)
        
    generator1 = torch.Generator().manual_seed(86)
    subset_arr = random_split(train_dataset, np.array([tl, tl, tl, tl+1, tl+1]).astype(int), generator1)
    for i in range(len(subset_arr)):
        idxs = set(np.arange(0, len(train_dataset)))
        val_subset = subset_arr[i]
        
        train_idxs = []
        train_arr = subset_arr[:i] + subset_arr[i+1:] # indexing lolzzz
        for j in train_arr:
            train_idxs.append(j.indices) # neat
        train_idxs = np.concatenate(train_idxs) # idxs preserved from train_dataset
        train_subset = torch.utils.data.Subset(train_dataset, train_idxs)

        f_params = f'./outputs/checkpoints/{experiment_name}/model_fold{i+1}.pt'
        f_history = f'./outputs/histories/{experiment_name}/model_fold{i+1}.json'
        csv_name = f'./outputs/probabilities/{experiment_name}/model_fold{i+1}.csv'
        
        print(len(train_subset), len(val_subset)) 
        print("Train/Test/Val datasets have been created.")
                
        # WEIGHTED SAMPLING
        val_indxs = val_subset.indices
        for k in val_indxs:
            idxs.discard(k) # this is now train_idxs
        print(f"indxs:{len(idxs)}")
        train_label_indxs = list(idxs)
        train_labels = np.array(train_dataset.samples)[:,1][train_label_indxs]
        labels = train_labels.astype(int) 
        
        black_weight = 1 / len(labels[labels == 0]) 
        white_weight = 1 / len(labels[labels == 1])
        sample_weights = np.array([black_weight, white_weight])
        weights = sample_weights[labels]
        sampler = torch.utils.data.WeightedRandomSampler(weights, len(train_subset), replacement=True)

        # SETTING MODEL PARAMETERS WITH SKORCH
        checkpoint = Checkpoint(monitor='valid_loss_best',
                                f_params=f_params,
                                f_history=f_history)
        
        train_acc = EpochScoring(scoring='accuracy',
                                 on_train=True,
                                 name='train_acc',
                                 lower_is_better=False)
        
        val_roc = EpochScoring(scoring='roc_auc',
                                 on_train=False,
                                 name='valid-auc',
                                 lower_is_better=False)
        
        val_pr = EpochScoring(scoring='precision',
                                 on_train=False,
                                 name='valid-precision',
                                 lower_is_better=False)
        
        val_re = EpochScoring(scoring='recall',
                                 on_train=False,
                                 name='valid-recall',
                                 lower_is_better=False)

        early_stopping = EarlyStopping()
        callbacks = [checkpoint, train_acc, early_stopping, val_roc, val_pr, val_re]

        net = NeuralNetClassifier(PretrainedModel,
                                  criterion=nn.CrossEntropyLoss,
                                  lr=lr,
                                  batch_size=batch_size,
                                  max_epochs=num_epochs,
                                  module__output_features=num_classes,
                                  optimizer=optim.SGD,
                                  optimizer__momentum=0.9,
                                  iterator_train__num_workers=1,
                                  iterator_train__sampler=sampler,
                                  iterator_valid__shuffle=False,
                                  iterator_valid__num_workers=1,
                                  train_split=predefined_split(val_subset),
                                  callbacks=callbacks,
                                  device=device)
        # SKORCH MODEL FITTING
        print ("Model is fitting. Thank you for your patience.")
        net.fit(train_subset, y=None)

        print ("Model is performing inference. Results saved in probabilities folder.")
        # SAVING RESULTS
        img_locs = [loc for loc, _ in test_dataset.samples]
        test_probs = net.predict_proba(test_dataset)
        test_probs = [prob[0] for prob in test_probs]
        data = {'img_loc' : img_locs, 'probability' : test_probs}
        pd.DataFrame(data=data).to_csv(csv_name, index=False)
        
        print (f"Model {i+1} is done and saved.")

    print ("The code is done.") ```

[BenjaminBossan]

Hi, I converted your issue to a discussion, I hope you don't mind.

There is a fix for your problem, which is to pass the classes argument to your net. So e.g. for binary classification:

net = NeuralNetClassifier(..., classes=np.array([0, 1]))

For your specific problem, replace np.array([0, 1]) with the (label-encoded) classes of your task.

To explain the error: AUC needs to know all existing classes. So it asks the net for the classes_ attribute. Since the net was fitted with y=None, it does not know the classes, so it tries to infer it with classes_inferred_. This in turn tries to extract the y from the dataset, but that only works for some types of dataset. In your case, it didn't work, thus the attribute is not set, thus you get the error.

In the future, it would probably be nice to have a better error message, but I think it's not trivial to know what exactly went wrong. Also, we don't want to always raise an error if classes_ cannot be determined, because most stuff will still work, but some stuff (like AUC) will not.

[Centrattic]

Wow. Thank you for the detailed traceback. I was quite confused, but this makes a lot of sense.