Train Accuracy Doesn't increase in GENConv

[riskiem]

I have the following Data Generation Process, basically I am trying to generate the default of banks based on an algorithm and then classify if a bank defaulted or not:

def EisenbergNoe(Lij,externalAssets,externalLiabilities,nbanks):
     L_ij = Lij;
     Lij_prime = L_ij.T;
     Ae = externalAssets;
     Le = externalLiabilities;
     
     tolerance = 10^(-5);
     max_count = 10^5
     nbanks = nbanks

     p_bar = L_ij.sum(axis=1)
     p_bar = p_bar.reshape(nbanks,1)
     equityZero = Ae - Le + Lij_prime.sum(axis=1) - p_bar;
     Lij_prime_sum = Lij_prime.sum(axis=1)
     Lij_prime_sum = Lij_prime_sum.reshape(nbanks,1)
     equityZero = Ae - Le + Lij_prime_sum - p_bar
     equity = equityZero;

     error = 1;
     count = 1;

     while error > tolerance and count < max_count:
       oldEquity = equity;
       recoveryVector = np.ones((nbanks, 1)) * ((equity >= 0).astype('int') + np.maximum(1 + equity/p_bar,0) * ((equity < 0).astype('int')));
       equity = Ae - Le + np.matmul(Lij_prime,recoveryVector) - p_bar;
       error = norm(equity - oldEquity)/norm(equity);
       count = count + 1;
    
     recoveryVector = np.ones((nbanks, 1)) * ((equity >= 0).astype('int') + np.maximum(1 + equity/p_bar,0) * ((equity < 0).astype('int')));
     equity = Ae - Le + np.matmul(Lij_prime,recoveryVector) - p_bar;
     paymentVector = np.maximum(0,np.minimum(Ae - Le + np.matmul(Lij_prime,recoveryVector), p_bar))
     equityLoss = equity - equityZero
     default = (paymentVector<p_bar).astype('int')
     return equityLoss, equity, equityZero, paymentVector, error, p_bar, default

I generate 50 graphs using the following:
ngraphs = 50
equityLossMat = []
equityMat = []
equityZeroMat = []
paymentVectorMat = []
errorMat = []
p_barMat = []
LijMat = []
AeMat = []
AeMatShock = []
AlMat = []
defaultMat = []
data_list = []
for i in range(ngraphs):
    nbanks = randint(50,100)
    externalAssets = np.random.randint(1,5,size = nbanks) #generate a vector of external assets
    AeMat.append(externalAssets)
    externalLiabilities = np.random.randint(1,5,size=nbanks) #generate a vector of external liabilities
    AlMat.append(externalLiabilities)
    uniformShock = .0001;
    shock = uniformShock * externalAssets;
    externalAssetsShocked = externalAssets - shock;
    AeMatShock.append(externalAssetsShocked)
    Lij = np.random.randint(1,10,size=nbanks*nbanks)
    Lij = Lij.reshape(nbanks,nbanks)
    np.fill_diagonal(Lij, 0) #A bank's liability to itself is zero
    #Now we have a system in which each node has a liability to another node, which usually will not be the case. So we can randomly set some elements of the Lij matrix to 0, 
   #Number of elements to replace
    
    num_replaced = np.round(.75 * Lij.shape[0]*Lij.shape[1])
    #Random (x, y) coordinates
    indices_x = np.random.randint(0, Lij.shape[0], int(num_replaced))
    indices_y = np.random.randint(0, Lij.shape[1], int(num_replaced))
    Lij[indices_x, indices_y] = 0
    LijMat.append(Lij.reshape(nbanks*nbanks,1))
   ##Call the EisenbergNoe Function to return the values of interest
    equityLoss, equity, equityZero, paymentVector, error, p_bar, default = EisenbergNoe(Lij,externalAssetsShocked.reshape(nbanks,1),externalLiabilities.reshape(nbanks,1),nbanks)
    equityLossMat.append(equityLoss)
    equityMat.append(equity)
    equityZeroMat.append(equityZero)
    paymentVectorMat.append(paymentVector)
    errorMat.append(error)
    p_barMat.append(p_bar )
    defaultMat.append(default)
    f_final = np.stack((externalAssets,externalLiabilities), axis=1)
    x = torch.from_numpy(f_final)
    x = x.type(torch.FloatTensor)
    x = torch.reshape(x,(nbanks,2))
    y =  torch.Tensor(default)          
    y =  torch.reshape(y.type(torch.LongTensor),(nbanks,))
    edge_index = np.transpose(np.nonzero(Lij))
    edge_index = torch.tensor(edge_index,dtype=torch.long)
    edge_attribute = Lij[np.nonzero(Lij > 0)]
    edge_attribute = torch.reshape(torch.tensor(edge_attribute,dtype=torch.float),(len(edge_attribute),1))
    data = Data(x = x, edge_index = edge_index.t().contiguous(), y = y, edge_attr=edge_attribute )
    data_list.append(data)

My GNN is as follows:

class DeeperGCN(torch.nn.Module):
    def __init__(self, hidden_channels, num_layers):
        super(DeeperGCN, self).__init__()

        self.node_encoder = Linear(data.x.size(-1), hidden_channels)
        self.edge_encoder = Linear(data.edge_attr.size(-1), hidden_channels)

        self.layers = torch.nn.ModuleList()
        for i in range(1, num_layers + 1):
            conv = GENConv(hidden_channels, hidden_channels, aggr='softmax_sg',
                           t=1.0, learn_t=True, num_layers=2, norm='layer')
            #conv = GENConv(hidden_channels, hidden_channels, aggr='add',
                            #num_layers=2, norm='layer')
            norm = LayerNorm(hidden_channels, elementwise_affine=True)
            act = ReLU(inplace=True)

            layer = DeepGCNLayer(conv, norm, act, block='res+', dropout=0.8,
                                 ckpt_grad=i % 3)
            self.layers.append(layer)

        self.lin = Linear(hidden_channels, 2)

    def forward(self, x, edge_index, edge_attr):
        x = self.node_encoder(x)
        edge_attr = self.edge_encoder(edge_attr)

        x = self.layers[0].conv(x, edge_index, edge_attr)

        for layer in self.layers[1:]:
            x = layer(x, edge_index, edge_attr)

        x = self.layers[0].act(self.layers[0].norm(x))
        

        return self.lin(x)

model = DeeperGCN(hidden_channels=64, num_layers=100)
#optimizer = torch.optim.Adam(model.parameters(), lr=0.1)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
#optimizer = torch.optim.Adam(model.parameters(), lr=1)
#scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20,
                              #min_lr=0.00001)

The Training looks as follows:

def train(loader):
    model.train()
    loss_all = 0

    for data in loader:
        #data = data.to(device)
        optimizer.zero_grad()
        out = model(data.x, data.edge_index, data.edge_attr)
        loss = criterion(out.squeeze(), data.y.squeeze())
        loss.backward()
        loss_all += loss.item() * data.num_nodes
        optimizer.step()
    #scheduler.step(loss)
    
    return loss_all / len(train_dataset)

And the test is as floows:

def test(loader):
      model.eval()
      test_correct = 0
      pred_len = 0
      for data in loader:
          out = model(data.x, data.edge_index, data.edge_attr)
          pred = out.argmax(dim= -1)  # Use the class with highest probability.
          test_correct += int((pred == data.y).sum()) # Check against ground-truth labels
          pred_len += pred.numel()
      return test_correct/pred_len

I train for 300 epochs. My loss doesnt change by must and my train accuracy decreases. Is there any reason why this could be happening? I have tries playing with batch_sizes and learning rates as well.

[rusty1s]

This is hard to say. I don't think it's a problem with model or optimization as those look correct to me. The only thing that you might wanna fix in that regard is that you are wrongly computing your loss, as you only divide it by the total number of graphs and not the total number of nodes in all training graphs.

[riskiem]

AH, okay. Let me try that out.

[riskiem]

I think a the problem might be due to the balance issues in data. There is a lot of 1s and very very few 0s. Do you think augmentation is a good strategy? I am not sure if augmentation is something done in GNN? If so, at what step should I introduce it?

[rusty1s]

Skewed class distribution is a problem in most neural networks. One way around is to re-weight predictions for specific labels in the loss function, see the weight argument here.

[riskiem]

So, if i have say 90% 1s and 10% 0s, should the weight argument be like torch.Tensor(.9,.1)?

[rusty1s]

Yes