Regression Model Seems to be Random Chance

[anthonysirico]

Here are the model, train, test, and predict functions that I have:

class Net(torch.nn.Module):
    def __init__(self, hidden_channels):
        super().__init__()
        torch.manual_seed(12345)
        self.lin0 = torch.nn.Linear(1, hidden_channels)
        self.conv1 = GraphConv(hidden_channels, hidden_channels)
        self.conv2 = GraphConv(hidden_channels, hidden_channels)
        self.conv3 = GraphConv(hidden_channels, hidden_channels)
        self.gru = GRU(hidden_channels, hidden_channels)

        self.set2set = Set2Set(hidden_channels, processing_steps=3)
        self.lin1 = torch.nn.Linear(2 * hidden_channels, hidden_channels)
        self.lin2 = torch.nn.Linear(hidden_channels, 1)

    def forward(self, data):
        out = F.relu(self.lin0(data.x.to(torch.float)))
        h = out.unsqueeze(0)

        for i in range(3):
            m = F.relu(self.conv1(out, data.edge_index))
            out = F.relu(self.conv2(m, data.edge_index))
            out = F.relu(self.conv3(out, data.edge_index))
            out, h = self.gru(out.unsqueeze(0), h)
            out = out.squeeze(0)

        out = self.set2set(out, data.batch)
        out = F.relu(self.lin1(out))
        out = self.lin2(out)
        return out.view(-1)

def train(epoch):
    model.train()
    loss_all = 0

    for data in training_loader:
        data = data.to(device)
        optimizer.zero_grad()
        loss = F.mse_loss(model(data), data.Utility.to(torch.float32))
        loss.backward()
        loss_all += loss.item() * data.num_graphs
        optimizer.step()
    return loss_all / len(training_loader.dataset)


def test(loader):
    model.eval()
    error = 0
    for data in loader:
        data = data.to(device)
        out = model(data)
        error += (out.cpu() * std - data.Utility.cpu() * std).abs().sum().item()  # MAE
    return error / len(loader.dataset)

def predict(loader):
    model.eval()
    predictions = []
    for data in loader:
        data = data.to(device)
        out = model(data)
        predictions.append(((out.cpu() * std) + mean).item())
    return predictions

But when i do multiple runs to evaluate the effectiveness of the model, it seems to be random chance when it comes to $R^2$, MSE, and MAE. What i am doing is reducing the batch size by half every run, and one round of training, a training size of 80% would do worse than a training set size of 40%, where the 80% would have an $R^2$ of -0.14. And other times it would be the other way around. Its taking 1000 epochs to see any good results. Is there anything wrong with the model i have come up with? Note: The $R^2$ results are based on the test set.

[rusty1s]

Your model looks like. Can you try to use GraphConv(aggr="mean") and replace the Set2Set+GRU with something like global_mean_pool?