Multi-Label Datasets

[RX28666]

Hello,

I am running my code on Yelp which has multi-labels, I made some simple modifications on my original code which is for normal single label dataset:
(1)I use new loss function:loss_op = torch.nn.BCEWithLogitsLoss() loss = loss_op(out, y.float())
(2) I use F-1 instead of accuracy:
y = data.y.to(out.device)
pred = (out > 0).float()
accs = []
for mask in [data.train_mask, data.val_mask, data.test_mask]:
accs.append(f1_score(y[mask], pred[mask], average='micro') if pred[mask].sum() > 0 else 0)
But the results are very bad, I am wondering if I miss something to do experiments on yelp. Thanks for help.

[rusty1s]

Your code looks correct to me. What are the results you see? I briefly remember that you need to normalize your input features on Yelp to get better performance.

[RX28666]

Hello Matthias,

Really appreciate your reply. Actually I just used APPNP with sampling to make a simple test, here is my full code:

path = osp.join(osp.dirname(osp.realpath("__file__")), '..', 'data', 'Yelp')
dataset = Yelp(path)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = dataset[0]
data.x = (data.x - data.x.mean(dim=0)) / data.x.std(dim=0)
data.n_id = torch.arange(data.num_nodes)
kwargs = {'batch_size': 10000, 'num_workers': 16, 'persistent_workers': True}
train_loader = NeighborLoader(data, input_nodes=data.train_mask,
                              num_neighbors=[-1], shuffle=True, **kwargs)
kwargs_test = {'batch_size': 50000, 'num_workers': 16, 'persistent_workers': True}
subgraph_loader = NeighborLoader(copy.copy(data), input_nodes=None,
                                 num_neighbors=[-1], shuffle=False, **kwargs_test)
class Net(torch.nn.Module):
    def __init__(self, dataset):
        super(Net, self).__init__()
        self.lin1 = Linear(dataset.num_features, args.hidden)
        self.lin2 = Linear(args.hidden, dataset.num_classes)
        self.prop1 = APPNP(args.K, args.alpha)

    def reset_parameters(self):
        self.lin1.reset_parameters()
        self.lin2.reset_parameters()

    def forward(self, data):
        x, edge_index = data.x, data.edge_index
        x = F.dropout(x, p=args.dropout, training=self.training)
        x = F.relu(self.lin1(x))
        x = F.dropout(x, p=args.dropout, training=self.training)
        x = self.lin2(x)
        x = self.prop1(x, edge_index)
        return x

    @torch.no_grad()
    def inference(self, x_all, subgraph_loader):
        self.eval()
        xs = []
        for batch in subgraph_loader:
            batch = batch.to(device)
            x = x_all[batch.n_id.to(x_all.device)].to(device)
            x = F.dropout(x, p=args.dropout, training=self.training)
            x = F.relu(self.lin1(x))
            x = F.dropout(x, p=args.dropout, training=self.training)
            x = self.lin2(x)
            x = self.prop1(x, batch.edge_index)
            xs.append(x[:batch.batch_size].cpu())
        out = torch.cat(xs, dim=0)
        return out

model = Net(dataset)
model, data = model.to(device), data.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
loss_op = torch.nn.BCEWithLogitsLoss()

def train(data):
    model.train()
    for batch in train_loader:
        optimizer.zero_grad()
        y = batch.y[:batch.batch_size]
        out = model(batch)[:batch.batch_size]
        loss = loss_op(out, y.float())
        loss.backward(retain_graph=True)
        optimizer.step()
    return float(loss)

@torch.no_grad()
def test():
    model.eval()
    out = model.inference(data.x, subgraph_loader)#.to(device)
    
    y = data.y.to(out.device)#.numpy()
    pred = (out > 0).float()#.numpy()
    
    accs = []
    for mask in [data.train_mask, data.val_mask, data.test_mask]:
        accs.append(f1_score(y[mask], pred[mask], average='micro') if pred[mask].sum() > 0 else 0)

    return accs

acc = []
for j in range(args.runs):
    model.reset_parameters()
    for epoch in range(args.epochs):
        loss = train(data)
        if epoch < 200:
           if (epoch + 1) % 5 == 0:
              train_acc, val_acc, test_acc = test()
              print(f'Epoch: {epoch:02d}, Train: {train_acc:.4f}, Val: {val_acc:.4f}, '
                f'Test: {test_acc:.4f}')
        else:
          train_acc, val_acc, test_acc = test()
          print(f'Epoch: {epoch:02d}, Train: {train_acc:.4f}, Val: {val_acc:.4f}, '
            f'Test: {test_acc:.4f}')

    torch.save(model.state_dict(), f"Yelp.pt")

The accuracy is bad (less than 40%) even after 100 epochs, but when I commented out the APPNP, only MLP left, it seems the performance can achieve over 60%, which should be in a reasonable range. Is there anything wrong?
By the way, I am also trying to run on PPI, I am also wondering if there are some methods can help to combine multiple graphs into one graph just like those datasets with single graph, eg, Reddit or OGB-products
Thanks for your help again.

[rusty1s]

If you want to combine graphs into one, you can use Batch.from_data_list(...) to do that. Not really sure what's the problem with your code though :(