Weird Batch behaviour when Testing

[stefansei]

I am doing classification on bipartite Graphs on custom data and achieved a accuracy of ~97% while training with a batch size of 32.
However, I have realized that when i am testing that I one the one hand get different results depending on the batch_size I use in the Dataloader and on the other hand I get different results on the SAME instance depending on whether it is the first element in the batch or not. I.e. if I do a batch of 3 with the same instance, instance 2 und 3 will have the same result. This is also the case for untrained models see below.

The example tt/data_pickle/data1 belongs to class 10 with 7.66%
The probability distribution is
[0.05388600379228592, 0.053171832114458084, 0.042920153588056564, 0.03638165444135666, 0.06454285979270935, 0.04660305753350258, 0.05731629952788353, 0.052408184856176376, 0.07615913450717926, 0.04408358410000801, 0.07660436630249023, 0.04978596046566963, 0.05435585230588913, 0.046954553574323654, 0.04687280207872391, 0.047345902770757675, 0.04439355060458183, 0.054442524909973145, 0.051771655678749084]

The example tt/data_pickle/data1 belongs to class 10 with 7.94%
The probability distribution is
[0.059912800788879395, 0.04194619879126549, 0.04149472713470459, 0.03482253849506378, 0.06477014720439911, 0.04839891940355301, 0.0508909597992897, 0.04741313308477402, 0.07316319644451141, 0.04465777054429054, 0.07944633811712265, 0.059452444314956665, 0.06273962557315826, 0.04157734662294388, 0.05337315425276756, 0.046813756227493286, 0.04888582602143288, 0.05137430131435394, 0.048866838216781616]

My Network architecture is:

def __init__(self, in_channels=[num_features_vars, num_features_constr], out_channels=num_classes, aggr="mean"):
        """
        :param in_channels: List [f_v, f_c] where f_v and f_c are the length of the different features
        :param out_channels: Number of classes, which a Graph can belong to
        :param aggr: Mode of aggregation for the collected features from the neighbourhood of each node
        """
        super(GCNNetBip, self).__init__()
        # Define Layers for the graph part of the network
        self.num_features_v = in_channels[0]
        self.num_features_c = in_channels[1]
        self.out_channels = out_channels

        self.conv_1_c = GraphConv((self.num_features_v, self.num_features_c), 32, aggr=aggr)
        self.conv_1_v = GraphConv((self.num_features_c, self.num_features_v), 32, aggr=aggr)
        self.conv_act1 = Tanh()

        self.conv_2_c = GraphConv((32, 32), 64, aggr=aggr)
        self.conv_2_v = GraphConv((32, 32), 64, aggr=aggr)
        self.conv_act2 = Tanh()

        # Fully connectes layers for classification to reduce the feature size before softmaxing
        self.lin_c1 = Linear(128, 64)
        self.act1 = ReLU()
        self.lin_c2 = Linear(64, self.out_channels)

def forward(self, data):
        """
        Forward pass for the network
        :param data: Data object
        :return: The values for every class NOT softmaxed
        """
        v, c, edge_index, x_s_batch, x_t_batch = data.x_s, data.x_t, data.edge_index, data.x_s_batch, data.x_t_batch

        trans_edge_index = t(edge_index)

        # Convolution with transposed edges for one site and regular edges for the other site
        v_ = self.conv_1_v((c, v), trans_edge_index)
        c_ = self.conv_1_c((v, c), edge_index)
        c, v = self.conv_act1(c_), self.conv_act1(v_)

        c_ = self.conv_2_c((v, c), edge_index)
        v_ = self.conv_2_v((c, v), trans_edge_index)
        c, v = self.conv_act2(c_), self.conv_act2(v_)

        # c_ = self.conv_3_c((v, c), edge_index)
        # v_ = self.conv_3_v((c, v), trans_edge_index)
        # c, v = self.conv_act3(c_), self.conv_act3(v_)

        # Pool the nodes independently
        v = global_mean_pool(v, x_s_batch, size=data.num_graphs)
        c = global_mean_pool(c, x_t_batch, size=data.num_graphs)

        # Concatenate the pooled features into a single feature vector
        x = torch.cat((v, c), dim=1)

        # Use 2 FC layers with relu and softmax at the end for classification purposes
        x = self.lin_c1(x)
        x = self.act1(x)
        x = self.lin_c2(x)

        return x

A shortest example would be something like this I suppose:

net = GCNNetBip()
loader = DataLoader(data_list, batch_size = 32, follow_batch=["x_s", "x_t"])

# Classify all the example
res = []
dist = []
with torch.no_grad():
    for n, graphs in enumerate(loader):
        x = net(graphs)
        _, class_ = torch.max(x.data, 1)
        probs =  torch.nn.functional.softmax(x.data, 1)
        dist += probs.detach().tolist()
        res += class_.tolist()

for i in range(len(res)):
    print(f"The example {name_list[i]} belongs to class {CLASSES[res[i]]} with {round(100 * dist[i][res[i]], 2)}%")

    if verbose:
        print(f"The probability distribution is \n {dist[i]}\n")

Hope someone has a idea what is going on...

[rusty1s]

Did you ensure that edge_index is correctly incremented as denoted here?

[rusty1s]

Can you send me a small example to reproduce? I can look into it.

[stefansei]

Sure, I've send you a mail with an attached pickled file, I'm also sending you a better way to reproduce soon

[stefansei]

import torch
from torch_geometric.data import DataLoader, Data
import pickle
from torch_geometric.nn import GraphConv, SAGEConv, global_max_pool, global_mean_pool
from torch.nn import Module, Softmax, Linear, Tanh, ReLU
from torch_sparse import t

class BipartiteData(Data):
    def __init__(self, x_s, x_t, edge_index, **kwargs):
        super(BipartiteData, self).__init__(**kwargs)
        self.edge_index = edge_index
        self.x_s = x_s
        self.x_t = x_t

class GCNNetBip(Module):
    """
    Class for Classification for bipartite graphs using GraphConv Layers.
    """

    def __init__(self, in_channels, out_channels, aggr="mean"):
        """
        :param in_channels: List [f_v, f_c] where f_v and f_c are the length of the variable features and constraint features respectively
        :param out_channels: Number of classes, which a Graph can belong to
        :param aggr: Mode of aggregation for the collected features from the neighbourhood of each node
        """
        super(GCNNetBip, self).__init__()
        # Define Layers for the graph part of the network
        self.num_features_v = in_channels[0]
        self.num_features_c = in_channels[1]
        self.out_channels = out_channels

        self.conv_1_c = GraphConv((self.num_features_v, self.num_features_c), 32, aggr=aggr)
        self.conv_1_v = GraphConv((self.num_features_c, self.num_features_v), 32, aggr=aggr)
        self.conv_act1 = Tanh()

        self.conv_2_c = GraphConv((32, 32), 64, aggr=aggr)
        self.conv_2_v = GraphConv((32, 32), 64, aggr=aggr)
        self.conv_act2 = Tanh()

        # self.conv_3_c = GraphConv((128, 128), 256, aggr=aggr)
        # self.conv_3_v = GraphConv((128, 128), 256, aggr=aggr)
        # self.conv_act3 = Tanh()

        # Fully connectes layers for classification to reduce the feature size before softmaxing
        self.lin_c1 = Linear(128, 64)
        self.act1 = ReLU()
        self.lin_c2 = Linear(64, self.out_channels)
        #self.softmax = Softmax(dim=1)

    def forward(self, data):
        """
        Forward pass for the network
        :param data: Batch of bipartite Graph with x_t_batch and x_s_batch to send through the Network
        :return: The values for every class NOT softmaxed
        """
        v, c, edge_index, x_s_batch, x_t_batch = data.x_s, data.x_t, data.edge_index, data.x_s_batch, data.x_t_batch
        trans_edge_index = t(edge_index)
        # Convolution with transposed edges for one site and regular edges for the other site
        v_ = self.conv_1_v((c, v), trans_edge_index)
        c_ = self.conv_1_c((v, c), edge_index)
        c, v = self.conv_act1(c_), self.conv_act1(v_)

        c_ = self.conv_2_c((v, c), edge_index)
        v_ = self.conv_2_v((c, v), trans_edge_index)

        c, v = self.conv_act2(c_), self.conv_act2(v_)
        # Pool the nodes independently
        v = global_mean_pool(v, x_s_batch, size = data.num_graphs)
        c = global_mean_pool(c, x_t_batch, size = data.num_graphs)



        # Concatenate the pooled features into a single feature vector
        x = torch.cat((v, c), dim=1)

        # Use 2 FC layers with relu and softmax at the end for classification purposes
        x = self.lin_c1(x)
        x = self.act1(x)
        x = self.lin_c2(x)

        return x


net = GCNNetBip([7,4], 19)
# Insert File here
data_list = [pickle.load(open(PATH", "rb"))]*3

loader = DataLoader(data_list, batch_size = 3, follow_batch=["x_s", "x_t"])

# Classify all the example
res = []
dist = []
with torch.no_grad():
    for n, graphs in enumerate(loader):
        x = net(graphs)
        _, cla = torch.max(x.data, 1)
        probs = torch.nn.functional.softmax(x.data, 1)
        dist += probs.detach().tolist()
        res += cla.tolist()

    for i in range(len(res)):
        print(f"The example belongs to class {res[i]} with {round(100 * dist[i][res[i]], 2)}%")
        print(f"The probability distribution is \n {dist[i]}\n")

[rusty1s]

I identified the error. It looks like your SparseTensor comes with edge weights, and since you are naming your SparseTensor edge_index, the values will get incremented per example in a mini-batch, see here. Fixable by writing:

class BipartiteData(Data):
    def __init__(self, x_s, x_t, edge_index, **kwargs):
        super(BipartiteData, self).__init__(**kwargs)
        self.edge_index = edge_index
        self.x_s = x_s
        self.x_t = x_t

    def __inc__(self, key, value):
        return 0

[stefansei]

Right! That fixed it, thanks a lot for your help once again 👍