Questions about trying to reproduce FAGCN results.

[jiaruHithub]

I tried to replicate the results of FAGCN using PYG's basics. However, the accuracy rate on chameleon dataset is only 0.038! I think there must be something wrong with my code. The code is as follows, I sincerely ask for your help:
So first of all, this is the first file, which contains the model:

import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch_geometric.nn import GCNConv, FAConv

from fa_train_eval import random_planetoid_splits, run
from torch_geometric.datasets import Actor, WikipediaNetwork
import os.path as osp


parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default='Chameleon')
parser.add_argument('--random_splits', type=bool, default=True)
parser.add_argument('--runs', type=int, default=1)
parser.add_argument('--epochs', type=int, default=500)
parser.add_argument('--lr', type=float, default=0.01)
parser.add_argument('--weight_decay', type=float, default=0.0005)
parser.add_argument('--early_stopping', type=int, default=10)
parser.add_argument('--hidden', type=int, default=32)
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--eps', type=float, default=0.3, help='Fixed scalar or learnable weight.')
parser.add_argument('--normalize_features', type=bool, default=True)
args = parser.parse_args()
path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', args.dataset)

class Net(torch.nn.Module):
    def __init__(self, dataset):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(dataset.num_features, args.hidden)
        self.conv1 = FAConv(args.hidden, args.eps, args.dropout)
        self.conv2 = FAConv(args.hidden, args.eps, args.dropout)
        self.fc2 = nn.Linear(args.hidden, dataset.num_classes)
    def reset_parameters(self):
        self.conv1.reset_parameters()
        self.conv2.reset_parameters()

    def forward(self, data):
        x, edge_index = data.x, data.edge_index
        x = F.dropout(x, p=args.dropout, training=self.training)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, p=args.dropout, training=self.training)
        x0 = x
        x = self.conv1(x, x0, edge_index)
        x = self.conv2(x, x0, edge_index)
        return F.log_softmax(x, dim=1)

dataset = WikipediaNetwork(path, args.dataset, transform=None, pre_transform=None)
permute_masks = random_planetoid_splits if args.random_splits else None
run(dataset, Net(dataset), args.runs, args.epochs, args.lr, args.weight_decay, args.early_stopping, permute_masks)

This is the second file, which is about running:

import time

import torch
import torch.nn.functional as F
from torch import tensor
from torch.optim import Adam
from torch_geometric.utils import dropout_adj

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')


def index_to_mask(index, size):
    mask = torch.zeros(size, dtype=torch.bool, device=index.device)
    mask[index] = 1
    return mask


def random_planetoid_splits(data, num_classes, train_ratio):
    # Set new random planetoid splits:
    # * 20 * num_classes labels for training
    # * 500 labels for validation
    # * 1000 labels for testing
    N = data.num_nodes
    indices = []
    for i in range(num_classes):
        index = (data.y == i).nonzero().view(-1)
        index = index[torch.randperm(index.size(0))]
        indices.append(index)

    train_index = torch.cat([i[:train_ratio * N] for i in indices], dim=0)

    rest_index = torch.cat([i[train_ratio * N:] for i in indices], dim=0)
    rest_index = rest_index[torch.randperm(rest_index.size(0))]

    data.train_mask = index_to_mask(train_index, size=data.num_nodes)
    data.val_mask = index_to_mask(rest_index[:0.5*len(rest_index)], size=data.num_nodes)
    data.test_mask = index_to_mask(rest_index[0.5*len(rest_index):], size=data.num_nodes)

    return data


def run(dataset, model, runs, epochs, train_ratio, lr, weight_decay, early_stopping,
        permute_masks=None, logger=False):

    val_losses, accs, durations = [], [], []
    for _ in range(runs):
        data = dataset[0]
        if permute_masks is not None:
            data = permute_masks(data, dataset.num_classes, train_ratio)
        data = data.to(device)

        model.to(device).reset_parameters()
        optimizer = Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

        if torch.cuda.is_available():
            torch.cuda.synchronize()

        t_start = time.perf_counter()

        best_val_loss = float('inf')
        test_acc = 0
        val_loss_history = []

        for epoch in range(1, epochs + 1):
            train(model, optimizer, data)
            eval_info = evaluate(model, data)
            eval_info['epoch'] = epoch

            # if logger is not None:
            #     logger(eval_info)

            if eval_info['val_loss'] < best_val_loss:
                best_val_loss = eval_info['val_loss']
                test_acc = eval_info['test_acc']

            val_loss_history.append(eval_info['val_loss'])
            # if early_stopping > 0 and epoch > epochs // 2:
            #     tmp = tensor(val_loss_history[-(early_stopping + 1):-1])
            #     if eval_info['val_loss'] > tmp.mean().item():
            #         break

        if torch.cuda.is_available():
            torch.cuda.synchronize()

        t_end = time.perf_counter()

        val_losses.append(best_val_loss)
        accs.append(test_acc)
        durations.append(t_end - t_start)

    loss, acc, duration = tensor(val_losses), tensor(accs), tensor(durations)

    print('Val Loss: {:.4f}, Test Accuracy: {:.3f} ± {:.3f}, Duration: {:.3f}'.
          format(loss.mean().item(),
                 acc.mean().item(),
                 acc.std().item(),
                 duration.mean().item()))


def train(model, optimizer, data):
    model.train()
    optimizer.zero_grad()
    out = model(data)
    loss = F.nll_loss(out[data.train_mask[:, 0]], data.y[data.train_mask[:, 0]])
    loss.backward()
    optimizer.step()


def evaluate(model, data):
    model.eval()

    with torch.no_grad():
        logits = model(data)

    outs = {}
    for key in ['train', 'val', 'test']:
        mask = data['{}_mask'.format(key)]
        loss = F.nll_loss(logits[mask[:, 0]], data.y[mask[:, 0]]).item()
        pred = logits[mask[:, 0]].max(1)[1]
        acc = pred.eq(data.y[mask[:, 0]]).sum().item() / mask.sum().item()

        outs['{}_loss'.format(key)] = loss
        outs['{}_acc'.format(key)] = acc

    return outs

Does it matter which dataset mask I use? How can I repeat FAGCN's results? Thank you very much!

[wsad1]

I don't see self.fc2 being used in forward.
So returning F.log_softmax(self.fc2(x), dim=1) in forward could fix the problem.

[jiaruHithub]

I commented if permute_masks is not None: data = permute_masks(data, dataset.num_classes, train_ratio) in the run function.
But things are not getting better.

[wsad1]

Hmm. Not sure exactly what's going wrong. But, does the training loss go down after each epoch?
Also in the reset_parameters() call self.fc1.reset_parameters() and self.fc2.reset_parameters() to make sure the linear weights are reset after every run.

[jiaruHithub]

I tried reset every parameter in next experiment.
I tried to log losses with logger in the run function of train_eval.py, but when I set logger=True, the system produced an error:

Traceback (most recent call last):
  File "gcn.py", line 43, in <module>
    args.early_stopping, permute_masks)
  File "/home/pytorch_geometric-master/pytorch_geometric-master/benchmark/citation/train_eval.py", line 71, in run
    logger(eval_info)
TypeError: 'bool' object is not callable

What is the reason for this?
Thank you.

[rusty1s]

The argument logger expects a function, not a bool value, i.e.:

def logger(info):
    epoch = info['epoch']
    test_acc = info['test_acc']
    print(epoch, test_acc)

[jiaruHithub]

The argument logger expects a function, not a bool value, i.e.:

def logger(info):
    epoch = info['epoch']
    test_acc = info['test_acc']
    print(epoch, test_acc)

Thank you for your reply. I have tried to print the information by directly printing the key in the dictionary. Besides, do you know why my training accuracy is so low? I had a similar problem with actor datasets with GCN, and I thought it was a problem with my model. But now it seems that the problem is not all models.