My AUC is coming for less than 0.5 for a lot of dataset sets what code be the potential issue

[bis1999]

So I am training GCN on a corpus of data, the problem I am getting is that in some of the datasets, the AUC is coming very low like 0.2 , 0.1 in some of the dataset, I am trying to fix it but I could not

import argparse

import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torch_geometric.utils import negative_sampling

import torch_geometric.transforms as T
from torch_geometric.nn import GCNConv, SAGEConv

from ogb.linkproppred import PygLinkPropPredDataset, Evaluator
from torch_sparse import SparseTensor
import os 
import pickle
import random
import numpy as np 


def load_process_data(idx):
    infile_ = open('All_dataset/data_{}.pkl'.format(idx), 'rb')
    df = pickle.load(infile_)
    
    infile = open('OLP_updated.pickle'.format(idx), 'rb')
    new= pickle.load(infile)
    
    df_ho_top = df["df_ho_top"]
    df_tr_top = df["df_tr_top"]
    positive_train= df["Gho_edges"]
    
    neg_train = df_tr_top[df_tr_top["Link_present"] != 1]["non_links"].tolist()
    
    ## Make the adjancey matrix 
    
    num_nodes = int(new[new["network_index"] == idx]["number_nodes"])

    row_ = [i[0] for i in positive_train]
    cols_ =[i[1] for i in positive_train]
    
    ## Training adj matrix 
    adj_training = SparseTensor(row=torch.tensor(row_),
                             col=torch.tensor(cols_), sparse_sizes=(num_nodes,num_nodes))
    
    negative_train_tensor =  torch.tensor(df_tr_top[df_tr_top["Link_present"] ==0]["non_links"].tolist()).T
    
    test_postive = df_ho_top[df_ho_top["Link_present"] ==1]["non_links"].tolist()
    test_negative  =  df_ho_top[df_ho_top["Link_present"] ==0]["non_links"].tolist()
    
    test_negative_tensor = torch.tensor(test_negative).T
    test_positive_tensor = torch.tensor(test_postive).T
    
    test_edge_label_index = torch.cat(
        [test_positive_tensor,test_negative_tensor],
        dim=-1)
    test_edge_label = torch.cat([
            torch.ones(test_positive_tensor.size(1)),
            torch.zeros(test_negative_tensor.size(1))
        ], dim=0)
    

    
    
    return adj_training, negative_train_tensor,test_edge_label_index,test_edge_label
    
    
class GCN(torch.nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels, num_layers,
                 dropout):
        super(GCN, self).__init__()

        self.convs = torch.nn.ModuleList()
        self.convs.append(GCNConv(in_channels, hidden_channels, cached=True))
        for _ in range(num_layers - 2):
            self.convs.append(
                GCNConv(hidden_channels, hidden_channels, cached=True))
        self.convs.append(GCNConv(hidden_channels, out_channels, cached=True))

        self.dropout = dropout

    def reset_parameters(self):
        for conv in self.convs:
            conv.reset_parameters()

    def forward(self, x, adj_t):
        for conv in self.convs[:-1]:
            x = conv(x, adj_t)
            x = F.relu(x)
            x = F.dropout(x, p=self.dropout, training=self.training)
        x = self.convs[-1](x, adj_t)
        return x
    
    
class LinkPredictor(torch.nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels, num_layers,
                 dropout):
        super(LinkPredictor, self).__init__()

        self.lins = torch.nn.ModuleList()
        self.lins.append(torch.nn.Linear(in_channels, hidden_channels))
        for _ in range(num_layers - 2):
            self.lins.append(torch.nn.Linear(hidden_channels, hidden_channels))
        self.lins.append(torch.nn.Linear(hidden_channels, out_channels))

        self.dropout = dropout

    def reset_parameters(self):
        for lin in self.lins:
            lin.reset_parameters()

    def forward(self, x_i, x_j):
        x = x_i * x_j
        for lin in self.lins[:-1]:
            x = lin(x)
            x = F.relu(x)
            x = F.dropout(x, p=self.dropout, training=self.training)
        x = self.lins[-1](x)
        return torch.sigmoid(x)

    
def train(model, predictor, x, adj_t,optimizer,negative_ex):

    row, col, _ = adj_t.coo()
    edge_index = torch.stack([col, row], dim=0)

    model.train()
    predictor.train()


    total_loss = total_examples = 0
    
    optimizer.zero_grad()
    
    h = model(x, adj_t)
    

    

    pos_out = predictor(h[edge_index[0]], h[edge_index[1]])
    pos_loss = -torch.log(pos_out + 1e-15).mean()


    neg_out = predictor(h[negative_ex[0]], h[negative_ex[1]])
    neg_loss = -torch.log(1 - neg_out + 1e-15).mean()

    loss = pos_loss + neg_loss
    loss.backward()

    torch.nn.utils.clip_grad_norm_(x, 1.0)
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
    torch.nn.utils.clip_grad_norm_(predictor.parameters(), 1.0)

    optimizer.step()

   


    return loss.item()


def train(model, predictor, x, adj_t,optimizer,negative_ex):

    row, col, _ = adj_t.coo()
    edge_index = torch.stack([col, row], dim=0)

    model.train()
    predictor.train()


    total_loss = total_examples = 0
    
    optimizer.zero_grad()
    
    h = model(x, adj_t)
    

    

    pos_out = predictor(h[edge_index[0]], h[edge_index[1]])
    pos_loss = -torch.log(pos_out + 1e-15).mean()


    neg_out = predictor(h[negative_ex[0]], h[negative_ex[1]])
    neg_loss = -torch.log(1 - neg_out + 1e-15).mean()

    loss = pos_loss + neg_loss
    loss.backward()

    torch.nn.utils.clip_grad_norm_(x, 1.0)
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
    torch.nn.utils.clip_grad_norm_(predictor.parameters(), 1.0)

    optimizer.step()

   


    return loss.item()


def get_results(idx):
    adj_training, negative_train_tensor,test_edge_label_index,test_edge_label = load_process_data(idx)
    num_layers = 2
    hidden_channels = 128
    dropout = 0.01
    lr = 0.001
    device ="cpu"
    model = GCN(hidden_channels, hidden_channels,hidden_channels, num_layers,dropout).to(device)
    emb = torch.nn.Embedding(adj_training.size(0),hidden_channels).to(device)
    
    predictor = LinkPredictor(hidden_channels, hidden_channels, 1,num_layers, dropout).to(device)
    optimizer = torch.optim.Adam(
                list(model.parameters()) +list(predictor.parameters())+list(emb.parameters()), lr=lr)
    x = emb.weight
    
    for i in range(100):
        ll = train(model, predictor, x, adj_training,optimizer,negative_train_tensor)
    
    model.eval()
    predictor.eval()
    h = model(x, adj_training)
    pred = predictor(h[test_edge_label_index[0]],h[test_edge_label_index[1]])
    roc_ = roc_auc_score(test_edge_label.numpy(),pred.view(-1).detach().numpy())
    
    return roc_,pred.view(-1).detach().numpy()
    
    

[rusty1s]

On a first glance, this looks all good. Does your loss actually decrease? What is the train AUROC (just to confirm that there is not a bug in the code)?

[bis1999]

Yes so the loss actually decreases and train AUC is 0.98 , what I observe was I am having small networks like 100 nodes and 140 edges or 50 nodes 120 edges, so for networks like this aka where the average degree is small , I had to hidden channels to 16 than 128 or 256

[rusty1s]

Mh, interesting. That would mean there is some distribution shift between training and validation examples/or you are just heavily overfitting on training examples.

[bis1999]

So I am getting this problem when the average degree is small when I reduce the number of hidden layers to like 32 it works well