Using std as aggr

[MichaelHopwood]

I see that if I use mean as aggr in MessagePassing, everything functions. But, if I use std, then I

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [3026, 3]], which is output 0 of SqrtBackward0, is at version 1; expected version 0 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

Is this a known bug? Is there a workaround? Standard deviation aggregation makes a lot of sense for my use case.

[EdisonLeeeee]

There seems no inplaced operation for StdAggregation and it works fine for me. Could you provide a minimal example to reproduce this?

[MichaelHopwood]

Here, I adapted examples/gcn.py. All I changed was I added aggr = StdAggregation() and put aggr=copy.deepcopy(aggr) in each GCNConv instantiation.

import argparse
import os.path as osp

import torch
import torch.nn.functional as F

import torch_geometric.transforms as T
from torch_geometric.datasets import Planetoid
from torch_geometric.logging import init_wandb, log
from torch_geometric.nn import GCNConv
from torch_geometric.nn.aggr.basic import StdAggregation, MeanAggregation
import copy


class Args:
    def __init__(self, args_dict):
        for key in args_dict:
            setattr(self, key, args_dict[key])

args_dict = {
    'dataset': 'Cora',
    'hidden_channels': 16,
    'lr': 0.01,
    'epochs': 200,
    'use_gdc': False,
    'wandb': False,
}
args = Args(args_dict)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
init_wandb(name=f'GCN-{args.dataset}', lr=args.lr, epochs=args.epochs,
           hidden_channels=args.hidden_channels, device=device)

path = osp.join('dataefa', 'Planetoid')
dataset = Planetoid(path, args.dataset, transform=T.NormalizeFeatures())
data = dataset[0]

if args.use_gdc:
    transform = T.GDC(
        self_loop_weight=1,
        normalization_in='sym',
        normalization_out='col',
        diffusion_kwargs=dict(method='ppr', alpha=0.05),
        sparsification_kwargs=dict(method='topk', k=128, dim=0),
        exact=True,
    )
    data = transform(data)

aggr = StdAggregation()
    
class GCN(torch.nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels):
        super().__init__()
        self.conv1 = GCNConv(in_channels, hidden_channels, cached=True,
                             normalize=not args.use_gdc, aggr=copy.deepcopy(aggr))
        self.conv2 = GCNConv(hidden_channels, out_channels, cached=True,
                             normalize=not args.use_gdc, aggr=copy.deepcopy(aggr))

    def forward(self, x, edge_index, edge_weight=None):
        x = F.dropout(x, p=0.5, training=self.training)
        x = self.conv1(x, edge_index, edge_weight).relu()
        x = F.dropout(x, p=0.5, training=self.training)
        x = self.conv2(x, edge_index, edge_weight)
        return x


model = GCN(dataset.num_features, args.hidden_channels, dataset.num_classes)
model, data = model.to(device), data.to(device)
optimizer = torch.optim.Adam([
    dict(params=model.conv1.parameters(), weight_decay=5e-4),
    dict(params=model.conv2.parameters(), weight_decay=0)
], lr=args.lr)  # Only perform weight-decay on first convolution.


def train():
    model.train()
    optimizer.zero_grad()
    out = model(data.x, data.edge_index, data.edge_weight)
    loss = F.cross_entropy(out[data.train_mask], data.y[data.train_mask])
    loss.backward()
    optimizer.step()
    return float(loss)


@torch.no_grad()
def test():
    model.eval()
    pred = model(data.x, data.edge_index, data.edge_weight).argmax(dim=-1)

    accs = []
    for mask in [data.train_mask, data.val_mask, data.test_mask]:
        accs.append(int((pred[mask] == data.y[mask]).sum()) / int(mask.sum()))
    return accs


best_val_acc = final_test_acc = 0
for epoch in range(1, args.epochs + 1):
    loss = train()
    train_acc, val_acc, tmp_test_acc = test()
    if val_acc > best_val_acc:
        best_val_acc = val_acc
        test_acc = tmp_test_acc

The error is as follows:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_23737/409600142.py in <cell line: 97>()
     96 best_val_acc = final_test_acc = 0
     97 for epoch in range(1, args.epochs + 1):
---> 98     loss = train()
     99     train_acc, val_acc, tmp_test_acc = test()
    100     if val_acc > best_val_acc:

/tmp/ipykernel_23737/409600142.py in train()
     78     out = model(data.x, data.edge_index, data.edge_weight)
     79     loss = F.cross_entropy(out[data.train_mask], data.y[data.train_mask])
---> 80     loss.backward()
     81     optimizer.step()
     82     return float(loss)

~/anaconda3/envs/python3/lib/python3.8/site-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    394                 create_graph=create_graph,
    395                 inputs=inputs)
--> 396         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    397 
    398     def register_hook(self, hook):

~/anaconda3/envs/python3/lib/python3.8/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    171     # some Python versions print out the first line of a multi-line function
    172     # calls in the traceback and some print out the last line
--> 173     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
    174         tensors, grad_tensors_, retain_graph, create_graph, inputs,
    175         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [2708, 7]], which is output 0 of SqrtBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

[EdisonLeeeee]

Thanks for your code. This is indeed a bug but was fixed in master. You can install PyG from master and try it again :)

[MichaelHopwood]

Yep, that did it. Thanks!