Torch accuracy and sklearn accuracy is v different

[talhaanwarch]

This is the code

    def test_step(self,batch,batch_idx):
        image,label=batch
        pred = self(image)
        loss=self.criterion(pred.flatten(),label.float()) #calculate loss
        acc=self.metrics(pred.flatten(),label)#calculate accuracy
        pred=torch.sigmoid(pred)
        return {'loss':loss,'acc':acc,'label':label,'pred':pred}

    def test_epoch_end(self, outputs):
        loss=torch.stack([x["loss"] for x in outputs]).mean().detach().cpu().numpy().round(2)
        acc=torch.stack([x["acc"] for x in outputs]).mean().detach().cpu().numpy().round(2)
        label=torch.cat([x["label"] for x in outputs]).detach().cpu().numpy().ravel()
        pred=torch.cat([x["pred"] for x in outputs]).detach().cpu().numpy().ravel()
        pred=pred.astype(int)
        print('torch acc',acc)
        print(classification_report(label,pred))
        print('sklearn',accuracy_score(label,pred))

There is difference of 10-15% between accuracies obtained by torchmetrics and sklearn

[akihironitta]

Hi @talhaanwarch, could you keep the description as it was and share how you've fixed it? Someone in the community might have the same issue and this thread could be useful for them :)

[talhaanwarch]

This is the solution

def test_step(self,batch,batch_idx):
        image,label=batch
        pred = self(image)
        
        return {'label':label,'pred':pred}

def test_epoch_end(self, outputs):

    label=torch.cat([x["label"] for x in outputs])
    pred=torch.cat([x["pred"] for x in outputs])
    acc=self.metrics(pred.flatten(),label)
    pred=pred.detach().cpu().numpy().ravel()
    label=label.detach().cpu().numpy().ravel()
    
    pred=np.where(pred>0.5,1,0).astype(int)
    print('torch acc',acc)
    print(classification_report(label,pred))
    print('sklearn',accuracy_score(label,pred))