for loop sending data generator to list vice accuracy data

[anthonysirico]

Below is the code, i am running a for loop to train on different training sizes. The first loop works correctly, where when training begins, the training and validation accuracy are sent to a list, then a frame then finally a csv. But on the subsequent loops, a data generator is sent to the list. Can anyone see where the issue is, because I cant find it.

Also if you have a better way of doing this (data compiling for analysis), I'm all ears.

import sys
sys.path.insert(0, 'C:\\Users\\user\\Desktop\\imbalance_exp\\imbalance_exp\\imbalance_exp')
import torch
from torch_geometric.loader import DataLoader
import imb_dataset as imb
from torch.nn import Linear
import torch.nn.functional as F
from torch_geometric.nn import GraphConv
from torch_geometric.nn import global_mean_pool
import neptune.new as neptune
import pandas as pd
from sklearn.metrics import confusion_matrix, matthews_corrcoef
import seaborn as sns
from neptune.new.types import File
from tqdm import tqdm
import time

known = imb.ImbalanceDataset(root='imb_50v2', set='known', split=0.5)
unknown = imb.ImbalanceDataset(root='imb_50v2', set='unknown', split=0.5)
all_data = imb.ImbalanceDataset(root='imb_50v2', set='All', split=None)
torch.manual_seed(12345)
known = known.shuffle()

lr = 0.001
training_perc = 0.9
N = len(known)
mini_batch_size = 32
epochs = 600
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
case = 2

class switcher:
    def sets(self, case):
        default = known
        return getattr(self, 'case_' + str(case), lambda: default)()
    
    def case_1(self):
        training_set = known[:int(training_perc*len(known))]
        validation_set = known[int(training_perc*len(known)):]

        training_loader = DataLoader(training_set, batch_size=mini_batch_size, shuffle=True)
        validation_loader = DataLoader(validation_set, batch_size=mini_batch_size, shuffle=False)

        training_ones = []
        training_zeros = []
        validation_ones = []
        validation_zeros = []

        for i in range(len(training_set)):
            if training_set[i].y == 1:
                training_ones.append(training_set[i])
            else:
                training_zeros.append(training_set[i])
        
        for i in range(len(validation_set)):
            if validation_set[i].y == 1:
                validation_ones.append(validation_set[i])
            else:
                validation_zeros.append(validation_set[i])

        return training_loader, validation_loader, training_ones, training_zeros, validation_ones, validation_zeros
    
    def case_2(self):
        one_index = round(len(known) * 0.25)
        known_ones = known[:one_index].copy()
        known_ones.shuffle()
        known_zeros = known[one_index:].copy()
        known_zeros.shuffle()
        
        training_ones = known_ones[:int(training_perc*len(known_ones))]
        training_zeros = known_zeros[:len(training_ones)]
        training_set = torch.utils.data.ConcatDataset([training_ones, training_zeros])

        validation_ones = known_ones[int(training_perc*len(known_ones)):]
        validation_zeros = known_zeros[len(training_ones):]
        validation_set = torch.utils.data.ConcatDataset([validation_ones, validation_zeros])

        training_loader = DataLoader(training_set, batch_size=mini_batch_size, shuffle=True)
        validation_loader = DataLoader(validation_set, batch_size=mini_batch_size, shuffle=False)

        training_ones = []
        training_zeros = []
        validation_ones = []
        validation_zeros = []

        for i in range(len(training_set)):
            if training_set[i].y == 1:
                training_ones.append(training_set[i])
            else:
                training_zeros.append(training_set[i])
        
        for i in range(len(validation_set)):
            if validation_set[i].y == 1:
                validation_ones.append(validation_set[i])
            else:
                validation_zeros.append(validation_set[i])
        
        return training_loader, validation_loader, training_ones, training_zeros, validation_ones, validation_zeros
class GCN(torch.nn.Module):
    def __init__(self, hidden_channels):
        super(GCN, self).__init__()
        torch.manual_seed(12345)
        self.conv1 = GraphConv(known.num_node_features, hidden_channels)
        self.conv2 = GraphConv(hidden_channels, hidden_channels)
        self.conv3 = GraphConv(hidden_channels, hidden_channels)
        self.lin = Linear(hidden_channels, known.num_classes)

    def forward(self, x, edge_index, batch):
        # 1. Obtain node embeddings
        x = self.conv1(x, edge_index)
        x = x.relu()
        x = self.conv2(x, edge_index)
        x = x.relu()
        x = self.conv3(x, edge_index)

        # 2. Readout layer
        x = global_mean_pool(x, batch)  # [batch_size, hidden_channels]

        # 3. Apply a final classifier
        x = F.dropout(x, p=0.5, training=self.training)
        x = self.lin(x)

        return x

model = GCN(hidden_channels=64).to(device)
print(model)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
criterion = torch.nn.CrossEntropyLoss()

def train():
    model.train()

    total_loss = 0
    for data in training_loader:  # Iterate in batches over the training dataset.
        data = data.to(device)
        out = model(data.x, data.edge_index, data.batch)  # Perform a single forward pass.
        loss = criterion(out, data.y)  # Compute the loss solely based on the training nodes.
        loss.backward()  # Derive gradients.
        optimizer.step()  # Update parameters based on gradients.
        optimizer.zero_grad()  # Clear gradients.

def test(loader):
    model.eval()

    correct = 0
    for data in loader:  # Iterate in batches over the training/test dataset.
        data = data.to(device)
        out = model(data.x, data.edge_index, data.batch)
        pred = out.argmax(dim=1)  # Use the class with highest probability.
        correct += int((pred == data.y).sum())  # Check against ground-truth labels.
    return correct / len(loader.dataset)  # Derive ratio of correct predictions.

output_frame = pd.DataFrame(columns=['epoch', 'lr', 'known', 'unknown', 'train_ones', 'train_zeros', 'val_ones', 'val_zeros', 'tn_all', 'fp_all', 'fn_all', 'tp_all', 'tn_known', 'fp_known', 'fn_known', 'tp_known', 'precision_all', 'recall_all', 'f1_all', 'accuracy_all', 'mcc_all', 'precision_known', 'recall_known', 'f1_known', 'accuracy_known', 'mcc_known', 'time_elapsed'])
for i in range(1,6):

    training_loader, validation_loader, training_ones, training_zeros, validation_ones, validation_zeros = switcher().sets(case)


    train_accuracy = []
    val_accuracy = []
    start_time = time.time()
    for epoch in tqdm(range(1, epochs + 1), total=epochs):
        train()
        train_acc = test(training_loader)
        train_accuracy.append(train_acc)
        val_acc = test(validation_loader)
        val_accuracy.append(val_acc)

    accuracy = pd.DataFrame()
    accuracy['train_acc'] = train_accuracy
    accuracy['val_acc'] = val_accuracy
    accuracy.to_csv(f'C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\imbalance_exp\\csv files\\accuracy_{i}.csv')

    unknown_loader = DataLoader(unknown, batch_size=1, shuffle=False)
    predictions = []
    all_correct = 0
    known_correct = 0
    for test in unknown_loader:
        test = test.to(device)
        out = model(test.x, test.edge_index, test.batch)
        pred = out.argmax(dim=1)
        predictions.append(pred)
        all_correct += int((pred == test.y_all).sum())
        known_correct += int((pred == test.y_known).sum())

    
    pred_df = pd.DataFrame()
    pred_df['y_all_true'] = [i.item() for i in unknown.data.y_all]
    pred_df['y_known_true'] = [i.item() for i in unknown.data.y_known]
    pred_df['y_pred'] = [i.item() for i in predictions]
    pred_df.to_csv(f'C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\imbalance_exp\\csv files\\pred_df_{i}.csv')

    cf_matrix_all = confusion_matrix(pred_df['y_all_true'], pred_df['y_pred'])
    ax = sns.heatmap(cf_matrix_all, annot=True, fmt='g', cmap='Blues')
    ax.title.set_text('Confusion Matrix based on all data')
    tn_all, fp_all, fn_all, tp_all = cf_matrix_all.ravel()

    end_time = time.time()
    time_elapsed = end_time - start_time

    
    precision_all = tp_all / (tp_all + fp_all)

    recall_all = tp_all / (tp_all + fn_all)
    
    f1_all = 2 * (precision_all * recall_all) / (precision_all + recall_all)
    
    accuracy_all = (tp_all + tn_all) / (tp_all + tn_all + fp_all + fn_all)
    
    mcc_all = matthews_corrcoef(pred_df['y_all_true'], pred_df['y_pred'])
    

    cf_matrix_known = confusion_matrix(pred_df['y_known_true'], pred_df['y_pred'])
    ax = sns.heatmap(cf_matrix_known, annot=True, fmt='g', cmap='Blues')
    ax.title.set_text('Confusion Matrix based on known data')
    tn_known, fp_known, fn_known, tp_known = cf_matrix_known.ravel()

    precision_known = tp_known / (tp_known + fp_known)
    
    recall_known = tp_known / (tp_known + fn_known)
    
    f1_known = 2 * (precision_known * recall_known) / (precision_known + recall_known)
    
    accuracy_known = (tp_known + tn_known) / (tp_known + tn_known + fp_known + fn_known)
    
    mcc_known = matthews_corrcoef(pred_df['y_known_true'], pred_df['y_pred'])

    #'epoch', 'lr', 'known', 'unknown', 'train_ones', 'train_zeros', 'val_ones', 'val_zeros', 'tn_all', 'fp_all', 'fn_all', 'tp_all', 'tn_known', 'fp_known', 'fn_known', 'tp_known
    output_frame.loc[i] = [epochs, lr, len(known), len(unknown), len(training_ones), len(training_zeros), len(validation_ones), len(validation_zeros), tn_all, fp_all, fn_all, tp_all, tn_known, fp_known, fn_known, tp_known, precision_all, recall_all, f1_all, accuracy_all, mcc_all, precision_known, recall_known, f1_known, accuracy_known, mcc_known, time_elapsed]
    output_frame.to_csv('C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\imbalance_exp\\csv files\\final_output.csv')
    training_perc -= 0.2

[rusty1s]

What's the issue you are experiencing?

[anthonysirico]

Figured it out, test was used twice, where after the first loop, test was passed to the subsequent loops as a generator and not a graph.

[anthonysirico]

@rusty1s I need clarification on one thing. If you take a look at my code, during each loop, I am training the same model correct? I need to declare a new model at the start of each loop to train a new model, right?

[rusty1s]

Yeah, that is correct. You need to create a new model/or reset its parameters, and create a new optimizer as well.