Why my accuracy go up that much when use GATConv layers?

[WindWalker69]

Hi, I'm not sure if somebody can help me here but I try. I'm trying to reproduce the unsupervised scenario of this project https://github.com/fanyun-sun/InfoGraph but using GATConv layers instead of GIN layers.
My project run without errors after a few tries but the accuracies I get are far away from the paper's authors. The problem is not much this however, rather that my accuracy has an oscillating behavior and always remains in a certain range without growing as it should even if only slightly. I don't understand how to get the accuracy trend of the authors of the article, no matter if with lower values. Can someone help me?

Here is my code:

• main.py

import torch
from torch.nn import Linear
from torch_geometric.datasets import TUDataset
from torch_geometric.loader import DataLoader

from InfoGraphUtilities import GATEncoder, FF, local_global_loss_, logistic_classify, svc_classify

Why my accuracy go up that much when use GATConv layers?

# Our Dataset
# dataset = TUDataset(root='/tmp/TUDataset', name='MUTAG')
# dataset = TUDataset(root='/tmp/TUDataset', name='IMDB-BINARY')
# dataset = TUDataset(root='/tmp/TUDataset', name='IMDB-MULTI')
# dataset = TUDataset(root='/tmp/TUDataset', name='REDDIT-BINARY')
# dataset = TUDataset(root='/tmp/TUDataset', name='REDDIT-MULTI-5K')
# dataset = TUDataset(root='/tmp/TUDataset', name='ENZYMES')
dataset = TUDataset(root='/tmp/TUDataset', name='PTC_MR')

# Dataset Info
print('\n')
print(f'Dataset: {dataset}')
print('==========================')
print(f'Number of graphs: {len(dataset)}')
print(f'Number of features: {dataset.num_features}')
print(f'Number of classes: {dataset.num_classes}')

data = dataset[0]  # Get first graph object of the dataset to analyze

# Data object info
print(f'\n{data}')
print('=================================================')

print(f'Number of nodes: {data.num_nodes}')
print(f'Number of edges: {data.num_edges}')
print(f'Average node degree: {data.num_edges / data.num_nodes:.2f}')
print(f'Has isolated nodes: {data.has_isolated_nodes()}')
print(f'Has self-loops: {data.has_self_loops()}')
print(f'Is undirect: {data.is_undirected()}\n')

def check_balance(dataset, name):
    total_classes = dataset.num_classes
    total_per_class = [0] * total_classes

    for _, label in enumerate(dataset.data.y):
        total_per_class[label] += 1

    print(f'{name} {sum(total_per_class)}')

    for label, class_sum in enumerate(total_per_class):
        print(f'{class_sum} of class: {label} - {class_sum/sum(total_per_class) * 100:0.1f}%')
    print('\n')

check_balance(dataset, dataset.name)

seed = 98765
torch.manual_seed(seed)
dataset = dataset.shuffle()

class GAT(torch.nn.Module):
    def __init__(self, hidden_channels, num_gat_layers, heads):
        super(GAT, self).__init__()

        self.embedding_dim = hidden_channels * heads * (num_gat_layers - 1) + dataset.num_classes
        self.encoder = GATEncoder(dataset.num_features, dataset.num_classes, hidden_channels, num_gat_layers, heads)

        self.local_d = FF(self.embedding_dim)
        self.global_d = FF(self.embedding_dim)

        self.init_emb()

    def init_emb(self):
        for m in self.modules():
            if isinstance(m, Linear):
                torch.nn.init.xavier_uniform_(m.weight.data)
                if m.bias is not None:
                    m.bias.data.fill_(0.0)

    def forward(self, x, edge_index, batch):
        if x is None:
            x = torch.ones(batch.shape[0]).to(device)

        y, M = self.encoder(x, edge_index, batch)

        g_enc = self.global_d(y)
        l_enc = self.local_d(M)

        measure = 'JSD'
        local_global_loss = local_global_loss_(l_enc, g_enc, batch, measure)

        return local_global_loss

# Settings
batch_size = 128
lr = 0.01
weight_decay = 5e-4

# Setup utilities
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = GAT(hidden_channels=32, num_gat_layers=3, heads=4).to(device)
# model = GAT(hidden_channels=64, in_heads=8, out_heads=1).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

data_loader = DataLoader(dataset, batch_size=batch_size)

for step, data in enumerate(data_loader):  # to see the mini-batching we makes
    print(f'Step: {step + 1}')
    print('=========')
    print(f'NUmber of graphs in the current batch: {data.num_graphs}')
    print(f'{data}\n')

print(model)

print('\n=============== Before Training ==================')
emb, y = model.encoder.get_embeddings(data_loader)
log_reg_acc = logistic_classify(emb, y)
svc_acc = svc_classify(emb, y)
print(f'LogReg Acc: {log_reg_acc:.4f}')
print(f'SVC Acc: {svc_acc:.4f}\n')


# Training function
def train():
    loss_all = 0

    model.train()
    for data in data_loader:  # Iterate in batches over the training dataset
        data = data.to(device)
        optimizer.zero_grad()
        loss = model(data.x, data.edge_index, data.batch)
        loss_all += loss.item() * data.num_graphs
        loss.backward()
        optimizer.step()
    return loss_all

def test(loader):
    model.eval()

    emb, y = model.encoder.get_embeddings(loader)
    svc_acc = svc_classify(emb, y)
    log_reg_acc = logistic_classify(emb, y)
    return log_reg_acc, svc_acc

for epoch in range(100):
    tot_loss = train()
    log_reg_acc, svc_acc = test(data_loader)
    print(f'Epoch: {epoch + 1:03d}, '
          f'Loss: {tot_loss / len(data_loader):.3f}, '
          f'LogReg Acc: {log_reg_acc:.4f}, '
          f'SVC Acc: {svc_acc:.4f}')

• InfoGraphUtilities.py

import math
import torch
import numpy as np
import torch.nn.functional as F
from sklearn import preprocessing
from sklearn.svm import SVC
from sklearn.metrics import balanced_accuracy_score
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from torch.nn import Sequential, Linear, ReLU
from torch_geometric.nn import GATConv, global_add_pool, global_mean_pool

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')


class GATEncoder(torch.nn.Module):
    def __init__(self, num_features, num_classes, hid_dim, num_gat_layers, heads):
        super(GATEncoder, self).__init__()

        self.num_gat_layers = num_gat_layers

        self.convs = torch.nn.ModuleList()  # List of convolutions
        self.bns = torch.nn.ModuleList()  # List of batch-normalizations

        self.convs.append(GATConv(in_channels=num_features,
                                  out_channels=hid_dim,
                                  heads=heads,
                                  dropout=0.5))
        self.bns.append(torch.nn.BatchNorm1d(hid_dim * heads))
        for _ in range(num_gat_layers - 2):
            self.convs.append(GATConv(in_channels=hid_dim * heads,
                                      out_channels=hid_dim,
                                      heads=heads,
                                      dropout=0.5))
            self.bns.append(torch.nn.BatchNorm1d(hid_dim * heads))
        self.convs.append(GATConv(in_channels=hid_dim * heads,
                                  out_channels=num_classes,
                                  concat=False,
                                  dropout=0.5))
        self.bns.append(torch.nn.BatchNorm1d(num_classes))

        # self.convs.append(Linear(in_features=hid_dim * heads, out_features=num_classes))

    def forward(self, x, edge_index, batch):
        if x is None:
            x = torch.ones((batch.shape[0], 1)).to(device)

        xs = []
        for i in range(self.num_gat_layers):
            x = F.dropout(x, p=0.5, training=self.training)
            x = F.elu(self.convs[i](x, edge_index))
            x = self.bns[i](x)
            xs.append(x)

        x_pool = [global_mean_pool(x, batch) for x in xs]
        # x_pool = [F.log_softmax(x, dim=1) for x in x_pool]
        x = torch.cat(x_pool, 1)
        # return F.log_softmax(x, dim=1), F.log_softmax(torch.cat(xs, 1), dim=1)
        return x, torch.cat(xs, 1)

    def get_embeddings(self, loader):

        ret = []
        y = []
        with torch.no_grad():
            for data in loader:
                data.to(device)
                x, edge_index, batch = data.x, data.edge_index, data.batch
                if x is None:
                    x = torch.ones((batch.shape[0], 1)).to(device)
                x, _ = self.forward(x, edge_index, batch)
                ret.append(x.cpu().numpy())
                y.append(data.y.cpu().numpy())
        ret = np.concatenate(ret, 0)
        y = np.concatenate(y, 0)
        return ret, y


class LogReg(torch.nn.Module):
    def __init__(self, ft_in, nb_classes):
        super(LogReg, self).__init__()
        self.fc = Linear(ft_in, nb_classes)

        for m in self.modules():
            self.weights_init(m)

    def weights_init(self, m):
        if isinstance(m, Linear):
            torch.nn.init.xavier_uniform_(m.weight.data)
            if m.bias is not None:
                m.bias.data.fill_(0.0)

    def forward(self, seq):
        ret = self.fc(seq)
        return ret


def logistic_classify(x, y):
    y = preprocessing.LabelEncoder().fit_transform(y)
    x, y = np.array(x), np.array(y)

    nb_classes = np.unique(y).shape[0]
    xent = torch.nn.CrossEntropyLoss()
    hid_units = x.shape[1]

    accs = []
    kf = StratifiedKFold(n_splits=10, shuffle=True, random_state=None)
    for train_index, test_index in kf.split(x, y):
        train_embs, test_embs = x[train_index], x[test_index]
        train_lbls, test_lbls = y[train_index], y[test_index]

        train_embs, train_lbls = torch.from_numpy(train_embs).cuda(), torch.from_numpy(train_lbls).cuda()
        test_embs, test_lbls = torch.from_numpy(test_embs).cuda(), torch.from_numpy(test_lbls).cuda()

        log = LogReg(hid_units, nb_classes)
        log.cuda()
        opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0)

        for it in range(100):
            log.train()
            opt.zero_grad()

            logits = log(train_embs)
            loss = xent(logits, train_lbls)

            loss.backward()
            opt.step()

        logits = log(test_embs)
        preds = torch.argmax(logits, dim=1)
        acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0]
        accs.append(acc.item())
    return np.mean(accs)


def svc_classify(x, y, search=True):
    y = preprocessing.LabelEncoder().fit_transform(y)
    x, y = np.array(x), np.array(y)

    kf = StratifiedKFold(n_splits=10, shuffle=True, random_state=None)
    accuracies = []
    for train_index, test_index in kf.split(x, y):

        x_train, x_test = x[train_index], x[test_index]
        y_train, y_test = y[train_index], y[test_index]
        if search:
            params = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}
            classifier = GridSearchCV(SVC(), params, cv=5, scoring='accuracy', verbose=0)
        else:
            classifier = SVC(C=10)
        classifier.fit(x_train, y_train)
        accuracies.append(balanced_accuracy_score(y_test, classifier.predict(x_test)))
    return np.mean(accuracies)


class FF(torch.nn.Module):
    def __init__(self, input_dim):
        super().__init__()
        self.block = Sequential(
            Linear(input_dim, input_dim),
            ReLU(),
            Linear(input_dim, input_dim),
            ReLU(),
            Linear(input_dim, input_dim),
            ReLU()
        )
        self.linear_shortcut = Linear(input_dim, input_dim)

    def forward(self, x):
        return self.block(x) + self.linear_shortcut(x)


def local_global_loss_(l_enc, g_enc, batch, measure):
    num_graphs = g_enc.shape[0]  # numero di grafi nei vari batch
    num_nodes = l_enc.shape[0]  # numero di nodi totali di tutti i grafi nei vari batch

    pos_mask = torch.zeros((num_nodes, num_graphs)).cuda()
    neg_mask = torch.ones((num_nodes, num_graphs)).cuda()
    for node_idx, graph_idx in enumerate(batch):
        pos_mask[node_idx][graph_idx] = 1.
        neg_mask[node_idx][graph_idx] = 0.

    res = torch.mm(l_enc, g_enc.t())

    E_pos = get_positive_expectation(res * pos_mask, measure, average=False).sum()
    E_pos = E_pos / num_nodes
    E_neg = get_negative_expectation(res * neg_mask, measure, average=False).sum()
    E_neg = E_neg / (num_nodes * (num_graphs - 1))

    return E_neg - E_pos


def get_positive_expectation(p_samples, measure, average=True):
    log_2 = math.log(2.)

    if measure == 'JSD':
        Ep = log_2 - F.softplus(- p_samples)

    if average:
        return Ep.mean()
    else:
        return Ep


def get_negative_expectation(q_samples, measure, average=True):
    log_2 = math.log(2.)

    if measure == 'JSD':
        Eq = F.softplus(-q_samples) + q_samples - log_2

    if average:
        return Eq.mean()
    else:
        return Eq

This is my output:

Dataset: PTC_MR(344)
==========================
Number of graphs: 344
Number of features: 18
Number of classes: 2

Data(edge_index=[2, 2], x=[2, 18], edge_attr=[2, 4], y=[1])
=================================================
Number of nodes: 2
Number of edges: 2
Average node degree: 1.00
Has isolated nodes: False
Has self-loops: False
Is undirect: True

PTC_MR 344
192 of class: 0 - 55.8%
152 of class: 1 - 44.2%

Step: 1
=========
NUmber of graphs in the current batch: 128
DataBatch(edge_index=[2, 3928], x=[1898, 18], edge_attr=[3928, 4], y=[128], batch=[1898], ptr=[129])

Step: 2
=========
NUmber of graphs in the current batch: 128
DataBatch(edge_index=[2, 3562], x=[1756, 18], edge_attr=[3562, 4], y=[128], batch=[1756], ptr=[129])

Step: 3
=========
NUmber of graphs in the current batch: 88
DataBatch(edge_index=[2, 2618], x=[1261, 18], edge_attr=[2618, 4], y=[88], batch=[1261], ptr=[89])

GAT(
(encoder): GATEncoder(
(convs): ModuleList(
(0): GATConv(18, 32, heads=4)
(1): GATConv(128, 32, heads=4)
(2): GATConv(128, 2, heads=1)
)
(bns): ModuleList(
(0): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): BatchNorm1d(2, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(local_d): FF(
(block): Sequential(
(0): Linear(in_features=258, out_features=258, bias=True)
(1): ReLU()
(2): Linear(in_features=258, out_features=258, bias=True)
(3): ReLU()
(4): Linear(in_features=258, out_features=258, bias=True)
(5): ReLU()
)
(linear_shortcut): Linear(in_features=258, out_features=258, bias=True)
)
(global_d): FF(
(block): Sequential(
(0): Linear(in_features=258, out_features=258, bias=True)
(1): ReLU()
(2): Linear(in_features=258, out_features=258, bias=True)
(3): ReLU()
(4): Linear(in_features=258, out_features=258, bias=True)
(5): ReLU()
)
(linear_shortcut): Linear(in_features=258, out_features=258, bias=True)
)
)

=============== Before Training ==================
LogReg Acc: 0.5024
SVC Acc: 0.4991

Epoch: 001, Loss: 883.944, LogReg Acc: 0.5841, SVC Acc: 0.5444
Epoch: 002, Loss: 912.209, LogReg Acc: 0.5790, SVC Acc: 0.5202
Epoch: 003, Loss: 417.263, LogReg Acc: 0.5785, SVC Acc: 0.4993
Epoch: 004, Loss: 196.676, LogReg Acc: 0.5847, SVC Acc: 0.5302
Epoch: 005, Loss: 161.735, LogReg Acc: 0.5666, SVC Acc: 0.5341
Epoch: 006, Loss: 116.916, LogReg Acc: 0.5727, SVC Acc: 0.5825
Epoch: 007, Loss: 74.265, LogReg Acc: 0.5702, SVC Acc: 0.5993
Epoch: 008, Loss: 47.079, LogReg Acc: 0.5903, SVC Acc: 0.5791
Epoch: 009, Loss: 27.744, LogReg Acc: 0.5752, SVC Acc: 0.6009
Epoch: 010, Loss: 3.859, LogReg Acc: 0.5840, SVC Acc: 0.6030
Epoch: 011, Loss: -0.692, LogReg Acc: 0.5953, SVC Acc: 0.6144
Epoch: 012, Loss: 2.159, LogReg Acc: 0.5841, SVC Acc: 0.5875
Epoch: 013, Loss: -7.436, LogReg Acc: 0.5906, SVC Acc: 0.6121
Epoch: 014, Loss: -20.164, LogReg Acc: 0.5840, SVC Acc: 0.5979
Epoch: 015, Loss: -27.074, LogReg Acc: 0.5982, SVC Acc: 0.6037
Epoch: 016, Loss: -25.175, LogReg Acc: 0.5582, SVC Acc: 0.6164
Epoch: 017, Loss: -39.406, LogReg Acc: 0.5988, SVC Acc: 0.6157
Epoch: 018, Loss: -44.971, LogReg Acc: 0.5994, SVC Acc: 0.6018
Epoch: 019, Loss: -47.925, LogReg Acc: 0.5754, SVC Acc: 0.5943
Epoch: 020, Loss: -52.633, LogReg Acc: 0.6015, SVC Acc: 0.6142
Epoch: 021, Loss: -51.680, LogReg Acc: 0.5818, SVC Acc: 0.6125
Epoch: 022, Loss: -54.450, LogReg Acc: 0.5931, SVC Acc: 0.6332
Epoch: 023, Loss: -54.265, LogReg Acc: 0.5962, SVC Acc: 0.6188
Epoch: 024, Loss: -60.407, LogReg Acc: 0.6189, SVC Acc: 0.6270
Epoch: 025, Loss: -60.543, LogReg Acc: 0.5985, SVC Acc: 0.6519
Epoch: 026, Loss: -62.516, LogReg Acc: 0.5785, SVC Acc: 0.6017
Epoch: 027, Loss: -63.800, LogReg Acc: 0.5875, SVC Acc: 0.6192
Epoch: 028, Loss: -68.019, LogReg Acc: 0.6076, SVC Acc: 0.6324
Epoch: 029, Loss: -70.275, LogReg Acc: 0.5782, SVC Acc: 0.6016
Epoch: 030, Loss: -72.166, LogReg Acc: 0.5899, SVC Acc: 0.6146
Epoch: 031, Loss: -72.008, LogReg Acc: 0.5843, SVC Acc: 0.6335
Epoch: 032, Loss: -72.859, LogReg Acc: 0.6051, SVC Acc: 0.5978
Epoch: 033, Loss: -73.864, LogReg Acc: 0.5873, SVC Acc: 0.6184
Epoch: 034, Loss: -76.908, LogReg Acc: 0.5668, SVC Acc: 0.5827
Epoch: 035, Loss: -76.664, LogReg Acc: 0.5810, SVC Acc: 0.5929
Epoch: 036, Loss: -78.479, LogReg Acc: 0.6107, SVC Acc: 0.5890
Epoch: 037, Loss: -80.435, LogReg Acc: 0.5958, SVC Acc: 0.5913
Epoch: 038, Loss: -79.538, LogReg Acc: 0.5929, SVC Acc: 0.6373
Epoch: 039, Loss: -79.569, LogReg Acc: 0.5930, SVC Acc: 0.5935
Epoch: 040, Loss: -82.212, LogReg Acc: 0.6249, SVC Acc: 0.6169
Epoch: 041, Loss: -82.966, LogReg Acc: 0.6024, SVC Acc: 0.5936
Epoch: 042, Loss: -81.875, LogReg Acc: 0.6016, SVC Acc: 0.6085
Epoch: 043, Loss: -82.066, LogReg Acc: 0.5637, SVC Acc: 0.6258
Epoch: 044, Loss: -83.603, LogReg Acc: 0.6023, SVC Acc: 0.6207
Epoch: 045, Loss: -81.957, LogReg Acc: 0.5933, SVC Acc: 0.5957
Epoch: 046, Loss: -85.494, LogReg Acc: 0.6018, SVC Acc: 0.5835
Epoch: 047, Loss: -87.232, LogReg Acc: 0.5758, SVC Acc: 0.5861
Epoch: 048, Loss: -86.071, LogReg Acc: 0.5760, SVC Acc: 0.6168
Epoch: 049, Loss: -87.953, LogReg Acc: 0.5840, SVC Acc: 0.6304
Epoch: 050, Loss: -88.232, LogReg Acc: 0.5640, SVC Acc: 0.6194
Epoch: 051, Loss: -86.492, LogReg Acc: 0.5811, SVC Acc: 0.6107
Epoch: 052, Loss: -88.245, LogReg Acc: 0.5932, SVC Acc: 0.6155
Epoch: 053, Loss: -88.694, LogReg Acc: 0.5786, SVC Acc: 0.6573
Epoch: 054, Loss: -91.695, LogReg Acc: 0.5873, SVC Acc: 0.6089
Epoch: 055, Loss: -92.950, LogReg Acc: 0.5615, SVC Acc: 0.5870
Epoch: 056, Loss: -93.706, LogReg Acc: 0.5844, SVC Acc: 0.6171
Epoch: 057, Loss: -92.296, LogReg Acc: 0.5871, SVC Acc: 0.6115
Epoch: 058, Loss: -94.807, LogReg Acc: 0.5873, SVC Acc: 0.6148
Epoch: 059, Loss: -94.914, LogReg Acc: 0.5961, SVC Acc: 0.6264
Epoch: 060, Loss: -95.641, LogReg Acc: 0.5955, SVC Acc: 0.6171
Epoch: 061, Loss: -96.781, LogReg Acc: 0.5640, SVC Acc: 0.6127
Epoch: 062, Loss: -97.895, LogReg Acc: 0.6044, SVC Acc: 0.6364
Epoch: 063, Loss: -98.666, LogReg Acc: 0.5761, SVC Acc: 0.6144
Epoch: 064, Loss: -98.320, LogReg Acc: 0.6021, SVC Acc: 0.6070
Epoch: 065, Loss: -98.395, LogReg Acc: 0.5839, SVC Acc: 0.5951
Epoch: 066, Loss: -98.911, LogReg Acc: 0.5814, SVC Acc: 0.5917
Epoch: 067, Loss: -98.078, LogReg Acc: 0.5957, SVC Acc: 0.6222
Epoch: 068, Loss: -98.040, LogReg Acc: 0.5780, SVC Acc: 0.5832
Epoch: 069, Loss: -97.847, LogReg Acc: 0.5996, SVC Acc: 0.6329
Epoch: 070, Loss: -95.562, LogReg Acc: 0.5675, SVC Acc: 0.6183
Epoch: 071, Loss: -97.879, LogReg Acc: 0.5962, SVC Acc: 0.6036
Epoch: 072, Loss: -98.564, LogReg Acc: 0.5933, SVC Acc: 0.5966
Epoch: 073, Loss: -99.928, LogReg Acc: 0.5493, SVC Acc: 0.5909
Epoch: 074, Loss: -98.096, LogReg Acc: 0.5815, SVC Acc: 0.5944
Epoch: 075, Loss: -101.493, LogReg Acc: 0.5786, SVC Acc: 0.5922
Epoch: 076, Loss: -100.052, LogReg Acc: 0.6045, SVC Acc: 0.5790
Epoch: 077, Loss: -100.212, LogReg Acc: 0.5696, SVC Acc: 0.5855
Epoch: 078, Loss: -101.539, LogReg Acc: 0.5957, SVC Acc: 0.6033
Epoch: 079, Loss: -100.924, LogReg Acc: 0.5874, SVC Acc: 0.5696
Epoch: 080, Loss: -102.149, LogReg Acc: 0.5727, SVC Acc: 0.5773
Epoch: 081, Loss: -103.037, LogReg Acc: 0.5640, SVC Acc: 0.6003
Epoch: 082, Loss: -104.126, LogReg Acc: 0.6078, SVC Acc: 0.5970
Epoch: 083, Loss: -102.732, LogReg Acc: 0.6020, SVC Acc: 0.5748
Epoch: 084, Loss: -102.467, LogReg Acc: 0.5814, SVC Acc: 0.5993
Epoch: 085, Loss: -102.834, LogReg Acc: 0.5639, SVC Acc: 0.5639
Epoch: 086, Loss: -103.259, LogReg Acc: 0.5670, SVC Acc: 0.5718
Epoch: 087, Loss: -106.174, LogReg Acc: 0.5840, SVC Acc: 0.5567
Epoch: 088, Loss: -102.991, LogReg Acc: 0.5782, SVC Acc: 0.5955
Epoch: 089, Loss: -105.488, LogReg Acc: 0.5582, SVC Acc: 0.5648
Epoch: 090, Loss: -105.466, LogReg Acc: 0.5553, SVC Acc: 0.5875
Epoch: 091, Loss: -105.982, LogReg Acc: 0.5492, SVC Acc: 0.5561
Epoch: 092, Loss: -104.486, LogReg Acc: 0.5497, SVC Acc: 0.5772
Epoch: 093, Loss: -106.702, LogReg Acc: 0.5697, SVC Acc: 0.5271
Epoch: 094, Loss: -105.203, LogReg Acc: 0.5409, SVC Acc: 0.5936
Epoch: 095, Loss: -106.335, LogReg Acc: 0.5672, SVC Acc: 0.5824
Epoch: 096, Loss: -105.710, LogReg Acc: 0.5813, SVC Acc: 0.5625
Epoch: 097, Loss: -106.281, LogReg Acc: 0.5692, SVC Acc: 0.5584
Epoch: 098, Loss: -108.486, LogReg Acc: 0.5756, SVC Acc: 0.5503
Epoch: 099, Loss: -106.482, LogReg Acc: 0.5552, SVC Acc: 0.5667
Epoch: 100, Loss: -108.091, LogReg Acc: 0.5700, SVC Acc: 0.5557

[rusty1s]

Are you sure your loss is set up correctly since it currently allows negative values?

[rusty1s]

Can you post the stacktrace of the IndexError?

[WindWalker69]

I repeat, this is the output given by the original InfoGraph code by changing only the dataset to IMDB-BINARY:

Traceback (most recent call last):
File "C:/Users/mR_95/OneDrive/Documenti/GitHub/InfoGraph/unsupervised/main.py", line 128, in
loss = model(data.x, data.edge_index, data.batch, data.num_graphs)
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch\nn\modules\module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "C:/Users/mR_95/OneDrive/Documenti/GitHub/InfoGraph/unsupervised/main.py", line 65, in forward
y, M = self.encoder(x, edge_index, batch)
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch\nn\modules\module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\unsupervised\gin.py", line 52, in forward
x = F.relu(self.convs[i](x, edge_index))
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch\nn\modules\module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch_geometric\nn\conv\gin_conv.py", line 74, in forward
out = self.propagate(edge_index, x=x, size=size)
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch_geometric\nn\conv\message_passing.py", line 366, in propagate
coll_dict = self.collect(self.user_args, edge_index,
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch_geometric\nn\conv\message_passing.py", line 255, in collect
self.set_size(size, 1 - dim, data[1 - dim])
File "C:\Users\mR_95\OneDrive\Documenti\GitHub\InfoGraph\venv\lib\site-packages\torch_geometric\nn\conv\message_passing.py", line 221, in set_size
size[dim] = src.size(self.node_dim)
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got -2)

With other dataset such as MUTAG this doesn't happen. This is an error that only appears when use the GIN layers, GAT layers doesn't support static database such as IMDB but with a small change to the batch I managed to get it working, although the (LogReg) accuracy remains fixed at 50% in every epoch.

[rusty1s]

Looks like there are no node features. Can you confirm? What is the output of print(dataset.data)?

[WindWalker69]

Yes, the nodes in the IMDB dataset have no features, in fact in the InfoGraph code the authors inserted the line of code dataset_num_features = max(dataset.num_features, 1), and so did I

[WindWalker69]

The output you requested:

Data(edge_index=[2, 193062], y=[1000], num_nodes=19773)

[ZeynepP]

Hi,
The authors reported accuracy 61.65 for the PTC_MR dataset with GIN. Did you run with GIN and get the accuracy similar or not? Just to verify the env.
Bests,

[WindWalker69]

Yes, for non-static dataset, with GIN layers I obtained similar results to those described by them. Maybe slightly lower