Model Outputs The Same Value for All Graphs in Binary Classification

[AlexNiko47]

Hi!

I'm trying to build a binary graph classification model where I have graphs of fixed size (each having 978 nodes, with each one having one feature) where the adjacency matrix represents the similarity between the nodes. I obtained pretty bad results on my actual data so I tried creating a mock dataset where I tried to make the two classes more distinctive (with the differences being not too subtle but not huge as well). Here is my implementation below:

import torch_geometric
from torch_geometric.data import Data
from torch_geometric.loader import DataLoader
from torch_geometric.nn import global_mean_pool, global_max_pool, GCNConv, MeanAggregation, SoftmaxAggregation
from torch.nn import Linear
import pandas as pd
from scipy import sparse
import numpy as np
from torch_geometric.utils import from_scipy_sparse_matrix
import torch
import torch.nn.functional as F

np.random.seed(123)

# Load adjacency matrix and preprocess it
adjacency_matrix = pd.read_csv("Adj_Similarity.csv", index_col=None).to_numpy()
adjacency_matrix = np.where(adjacency_matrix > 0.5, adjacency_matrix, 0)
adjacency_matrix = sparse.coo_matrix(adjacency_matrix)
adjacency_matrix = from_scipy_sparse_matrix(adjacency_matrix)


# Initialize training, validation and test set from normal distribution with mean 1 and sd 0.5
class1_train_dataset = np.random.normal(1,0.5,size=(200,978))
class2_train_dataset = np.random.normal(1,0.5,size=(200,978))

class1_test_dataset = np.random.normal(1,0.5,size=(50,978))
class2_test_dataset = np.random.normal(1,0.5,size=(50,978))

class1_validation_dataset = np.random.normal(1,0.5,size=(25,978))
class2_validation_dataset = np.random.normal(1,0.5,size=(25,978))

# Generate indexes for features that are randomly distinctive for both classes
class1_features = np.random.choice(range(1,978),30)
class2_features = np.random.choice(range(1,978),30)
print(class1_features)
print(class2_features)

# Make the features distinctive using a normal distribution with slightly higher mean
for i in class1_features:
    class1_train_dataset[:,i] = np.random.normal(3,0.5,size=200)
    class1_test_dataset[:,i] = np.random.normal(3,0.5,size=50)
    class1_validation_dataset[:,i] = np.random.normal(3,0.5,size=25)

for i in class2_features:
    class2_train_dataset[:,i] = np.random.normal(3,0.5,size=200)
    class2_test_dataset[:,i] = np.random.normal(3,0.5,size=50)
    class2_validation_dataset[:,i] = np.random.normal(3,0.5,size=25)

# Generate the list of graphs required for the PyTorch Geometric DataLoader
list_of_training_graphs = []
list_of_validation_graphs = []
list_of_test_graphs = []

for row in class1_train_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([0], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_training_graphs.append(graph)

for row in class2_train_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([1], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_training_graphs.append(graph)

###
for row in class1_validation_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([0], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_validation_graphs.append(graph)

for row in class2_validation_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([1], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_validation_graphs.append(graph)

##
for row in class1_test_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([0], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_test_graphs.append(graph)

for row in class2_test_dataset:
    x = torch.as_tensor(row, dtype=torch.float32)
    x = x.view(-1,1)
    y = torch.tensor([1], dtype=torch.float)
    graph = Data( x = x, edge_index = adjacency_matrix[0], edge_weight = adjacency_matrix[1].view(-1,1).float(), y = y)

    list_of_test_graphs.append(graph)

# Generate DataLoaders
TrainLoader = DataLoader(list_of_training_graphs,batch_size= 24, shuffle=True)
ValdationLoader = DataLoader(list_of_validation_graphs, batch_size=24, shuffle=False)
TestLoader = DataLoader(list_of_test_graphs, batch_size=24, shuffle=False)

# GCN Model
class GraphGCN(torch.nn.Module):
    def __init__(self, in_channels : int, hidden_channels: int, out_channels : int):
        super().__init__()
        torch.manual_seed(12345)
        self.linear1 = Linear(in_channels, hidden_channels)
        self.conv1 = GCNConv(hidden_channels, hidden_channels, add_self_loops=False)
        self.linear2 = Linear(hidden_channels,out_channels)
        self.conv2 = GCNConv(out_channels,out_channels, add_self_loops=False)
        self.linear3 = Linear(out_channels,out_channels)
        self.linear4 = Linear(out_channels,out_channels)

    def forward(self, x, edge_index, edge_weight, batch):
        print("Shape of x before first linear layer:", x.shape)
        x = self.linear1(x)
        x = x.relu()

        print("Shape of x after first linear layer is:", x.shape)
        #x = self.conv1(x, edge_index, edge_weight)

        #print("Shape of x after conv is:", x.shape)
        x = self.linear2(x)
        x = x.relu()

        print("Shape of x after second linear layer is:", x.shape)
        #x = self.conv2(x, edge_index, edge_weight)

        x = global_mean_pool(x, batch)
        print("Shape of x after pooling is:", x.shape)

        x = self.linear3(x)
        x = x.relu()

        x = F.dropout(x, 0.5, training=self.training)
        x = self.linear4(x)
        print("After sigmoid, shape is:", torch.sigmoid(x).view(-1).shape)
        return torch.sigmoid(x).view(-1)

# Training Loop
model = GraphGCN(in_channels=1, hidden_channels=16, out_channels=1)
optimizer = torch.optim.Adam(model.parameters(), lr=0.05)
loss_fn = torch.nn.BCELoss()
device = "cpu"
def train():
    model.train()

    total_loss = 0
    for data in TrainLoader:
        data = data.to(device)
        optimizer.zero_grad()
        out = model(data.x, data.edge_index, data.edge_weight, data.batch)
        print("Data.y is:", data.y)
        print("Out is:", out)
        loss = loss_fn(out,data.y.float())
        loss.backward()
        optimizer.step()
        total_loss += float(loss) * data.num_graphs
    return total_loss / len(TrainLoader.dataset)

for epoch in range(1, 50):
    loss = train()
    print("Loss is:", loss)

However, whenever I try to train the model, the graph outputs are exactly the same after just a couple of epochs. I've tried just about every pooling function PyG has and tinkered with each and every single hyperparameter (lr, wd, adding a scheduler etc.) and the outputs still remain exactly the same even with this simple model which only uses Linear layers and a pooling layer. Here is a snippet of one batch in the training run:

Shape of x before first linear layer: torch.Size([23472, 1])
Shape of x after first linear layer is: torch.Size([23472, 16])
Shape of x after second linear layer is: torch.Size([23472, 1])
Shape of x after pooling is: torch.Size([24, 1])
After sigmoid, shape is: torch.Size([24])
Data.y is: tensor([1., 1., 1., 1., 0., 1., 0., 0., 1., 1., 1., 1., 0., 1., 0., 0., 0., 1.,
        0., 0., 0., 1., 0., 0.])
Out is: tensor([0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120,
        0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120,
        0.5120, 0.5120, 0.5120, 0.5120, 0.5120, 0.5120],
       grad_fn=<ViewBackward0>)

Is there anything wrong in my implementation or in the way I pose the problem? I've tried a regular PyTorch MLP and found it pretty easy to classify each "graph" if I was just treating everything as tabular data, so I really have no idea what I'm doing wrong.
Adj_Similarity.csv

[EdisonLeeeee]

This is most likely due to the fact that the input data x is randomly generated and the model cannot distinguish between them, so the final output converges to the same.

[AlexNiko47]

I have tried initializing the datasets as np.ones() and then populate random features from a normal distribution with higher mean and very low sd but I'm still getting the same issue:

class1_train_dataset = np.ones((200,978), dtype=np.float32)
class2_train_dataset = np.ones((200,978), dtype=np.float32)

for i in class1_features:
    class1_train_dataset[:,i] = np.random.normal(10,0.25,size=200)
    class1_test_dataset[:,i] = np.random.normal(10,0.25,size=50)
    class1_validation_dataset[:,i] = np.random.normal(10,0.25,size=25)

# This is one of the features which I consider significant for classifying class 1 (chosen randomly from class1_features)
print(class1_train_dataset[:,511])
# This is a random feature which should only have 1s
print(class1_train_dataset[:,22])

The print(class1_train_dataset[:,511]) returns [10.228439 10.116567 10.03126 9.950587 9.817991 9.847422 ... ] while print(class1_train_dataset[:,22]) returns [1. 1. 1. 1. 1. 1. 1. 1. ...] so I'm successfully creating these datasets, however the issue still persists:

Data.y is: tensor([1., 1., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 0., 1., 1., 1.,
        1., 1., 1., 0., 0., 1.])
Out is: tensor([0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098,
        0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098,
        0.5098, 0.5098, 0.5098, 0.5098, 0.5098, 0.5098],
       grad_fn=<ViewBackward0>)

I've also tried initializing everything using np.zeros() and increasing the number of "significant" features using class1_features = np.random.choice(range(1,978),60) or even np.random.choice(range(1,978),200) but the same thing keeps happening during training. Maybe it's a problem with the adjacency matrix I'm using?

[rusty1s]

Can you try to replace GCNConv with SAGEConv? Depending on how the adjacency looks like, GCNConv will let node features converge to a stationary point.