Second InMemoryDataset in the same notebook comes out as a list

[anthonysirico]

Here is the code that produces the first dataset:

from scipy.io import loadmat
import pandas as pd
from torch_geometric.data import InMemoryDataset
import torch
import numpy as np
import torch_geometric
from tqdm import tqdm
from typing import Union
from torch import Tensor
from collections.abc import Sequence
from torch_geometric.utils import from_networkx
import networkx as nx

IndexType = Union[slice, Tensor, np.ndarray, Sequence]

print(f"Torch version: {torch.__version__}")
print(f"Cuda available: {torch.cuda.is_available()}")
print(f"Torch geometric version: {torch_geometric.__version__}")

class IterationDataset(InMemoryDataset):
    def __init__(self, root: str, transform=None, pre_transform=None, pre_filter=None):
        
        super().__init__(root, transform, pre_transform, pre_filter)

        self.data, self.slices = torch.load(self.processed_paths[0])
    
    @property
    def raw_file_names(self):
        return 'C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\shuffled_data.mat'

    @property
    def processed_file_names(self) -> list[str]:
        return ['data.pt']

    def download(self):
        pass

    def process(self):
        self.raw_data = loadmat(self.raw_file_names, squeeze_me=True)
        data = pd.DataFrame(self.raw_data['Graphs']).sort_values('Labels', ascending=True)
        known_performance_threshold = np.median(data['Labels'])
        
        r'''Create Known Graphs'''
        known_list = []
        for index, cir in tqdm(data.iterrows(), total=len(data)):
            nxg = nx.Graph(data['A'][index])
            nxg = self._get_node_features(nxg, data['Ln'][index])
            pt_graph = self._get_graph_object(nxg)
            pt_graph.y = self._get_known_graph_label(data['Labels'][index], known_performance_threshold)
            pt_graph.performance = torch.tensor(data['Labels'][index], dtype=torch.float)
            pt_graph.complexity = self._get_complexity(data['types'][index])
            pt_graph.orig_index = torch.tensor(index, dtype=torch.long)
            known_list.append(pt_graph)


        if self.pre_filter is not None:
            known_list = [data for data in known_list if self.pre_filter(data)]

        
        if self.pre_transform is not None:
            known_list = [self.pre_transform(data) for data in known_list]

        
        data_known, slices_known = self.collate(known_list)


        torch.save((data_known, slices_known), self.processed_paths[0])


    def _get_complexity(self, types):
        comp = len(types)
        return torch.tensor(comp, dtype=torch.float)

    def _get_node_features(self, nx_graph, node_labels):
        betweenness = list(nx.betweenness_centrality(nx_graph).values())
        eigenvector = list(nx.eigenvector_centrality(nx_graph, max_iter=600).values())
        node_label_dict = dict(enumerate(node_labels))

        mapping_dict = {'C': 0, 'G': 1, 'I': 2, 'O': 3, 'P': 4, 'R': 5}
        component_labels = []

        for value in node_label_dict.values():
            if value in mapping_dict:
                component_labels.append(mapping_dict[value])

        all_features = zip(betweenness, eigenvector, component_labels)
        all_features = dict(enumerate(all_features))

        nx.set_node_attributes(nx_graph, all_features, 'features')

        return nx_graph

    
    def _get_graph_object(self, nx_graph):
        nxg = from_networkx(nx_graph, group_node_attrs=['features'])
        return nxg

    def _get_known_graph_label(self, performance, threshold):
        if performance < threshold:
            return torch.tensor(1, dtype=torch.long)
        else:
            return torch.tensor(0, dtype=torch.long)

    def _get_all_graph_label(self, performance, threshold):
        if performance < threshold:
            return torch.tensor(1, dtype=torch.long)
        else:
            return torch.tensor(0, dtype=torch.long)
    
    @property
    def num_node_features(self) -> int:
        return 3

    @property
    def num_classes(self) -> int:
        return 2

Then, I run and train a model successfully, and make predictions:

import pandas as pd
output = pd.DataFrame()
predictions = []
index = []
y_true = []

known_correct = 0
for test_graph in unknown_loader:
    test_graph = test_graph.to(device)
    out = model(test_graph.x, test_graph.edge_index, test_graph.batch)
    pred = out.argmax(dim=1)
    predictions.append(pred.item())
    index.append(test_graph.orig_index.item())
    y_true.append(test_graph.y.item())
    
    known_correct += int((pred == test_graph.y).sum())

output['index'] = index
output['prediction'] = predictions
output['y_true'] = y_true

ones_index = []
for i in range(len(output)):
    if output['prediction'][i] == 1:
        ones_index.append(output['index'][i])

predicted_ones = All.index_select(ones_index).copy()

All2 = torch.utils.data.ConcatDataset([known, predicted_ones])

All2_indices = []
for i in range(len(All2)):
    All2_indices.append(All2[i].orig_index.item())

I then take that All2_indices and pass it into my second dataset creation script:

from scipy.io import loadmat
import pandas as pd
from torch_geometric.data import InMemoryDataset
import torch
import numpy as np
import torch_geometric
from tqdm import tqdm
from typing import Union
from torch import Tensor
from collections.abc import Sequence
from torch_geometric.utils import from_networkx
import networkx as nx

IndexType = Union[slice, Tensor, np.ndarray, Sequence]

print(f"Torch version: {torch.__version__}")
print(f"Cuda available: {torch.cuda.is_available()}")
print(f"Torch geometric version: {torch_geometric.__version__}")

class IterationDataset2(InMemoryDataset):
    def __init__(self, root: str, indices: list, transform=None, pre_transform=None, pre_filter=None):

        self.indices = indices
        
        super().__init__(root, transform, pre_transform, pre_filter)

        self.data, self.slices = torch.load(self.processed_paths[0])
    
    @property
    def raw_file_names(self):
        return 'C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\shuffled_data.mat'

    @property
    def processed_file_names(self) -> list[str]:
        return ['data.pt']

    def download(self):
        pass

    def process(self):
        self.raw_data = loadmat(self.raw_file_names, squeeze_me=True)
        data = pd.DataFrame(self.raw_data['Graphs']).sort_values('Labels', ascending=True)
        known_performance_threshold = np.median(data['Labels'])
        
        r'''Create Known Graphs'''
        known_list = []
        for index in tqdm(self.indices, total=len(self.indices)):
            nxg = nx.Graph(data['A'][index])
            nxg = self._get_node_features(nxg, data['Ln'][index])
            pt_graph = self._get_graph_object(nxg)
            pt_graph.y = self._get_known_graph_label(data['Labels'][index], known_performance_threshold)
            pt_graph.performance = torch.tensor(data['Labels'][index], dtype=torch.float)
            pt_graph.complexity = self._get_complexity(data['types'][index])
            pt_graph.orig_index = torch.tensor(index, dtype=torch.long)
            known_list.append(pt_graph)


        if self.pre_filter is not None:
            known_list = [data for data in known_list if self.pre_filter(data)]

        
        if self.pre_transform is not None:
            known_list = [self.pre_transform(data) for data in known_list]

        
        data_known, slices_known = self.collate(known_list)


        torch.save((data_known, slices_known), self.processed_paths[0])


    def _get_complexity(self, types):
        comp = len(types)
        return torch.tensor(comp, dtype=torch.float)

    def _get_node_features(self, nx_graph, node_labels):
        betweenness = list(nx.betweenness_centrality(nx_graph).values())
        eigenvector = list(nx.eigenvector_centrality(nx_graph, max_iter=600).values())
        node_label_dict = dict(enumerate(node_labels))

        mapping_dict = {'C': 0, 'G': 1, 'I': 2, 'O': 3, 'P': 4, 'R': 5}
        component_labels = []

        for value in node_label_dict.values():
            if value in mapping_dict:
                component_labels.append(mapping_dict[value])

        all_features = zip(betweenness, eigenvector, component_labels)
        all_features = dict(enumerate(all_features))

        nx.set_node_attributes(nx_graph, all_features, 'features')

        return nx_graph

    
    def _get_graph_object(self, nx_graph):
        nxg = from_networkx(nx_graph, group_node_attrs=['features'])
        return nxg

    def _get_known_graph_label(self, performance, threshold):
        if performance < threshold:
            return torch.tensor(1, dtype=torch.long)
        else:
            return torch.tensor(0, dtype=torch.long)

    def _get_all_graph_label(self, performance, threshold):
        if performance < threshold:
            return torch.tensor(1, dtype=torch.long)
        else:
            return torch.tensor(0, dtype=torch.long)
    
    @property
    def num_node_features(self) -> int:
        return 3

    @property
    def num_classes(self) -> int:
        return 2

Using: data2 = it2.IterationDataset2(root='data2', indices=All2_indices, transform=None, pre_transform=None, pre_filter=None)

But data2 ends up being a list. So when i do something simple like len(data2), i get:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
[~\AppData\Local\Temp\ipykernel_43252\2131915342.py](https://file+.vscode-resource.vscode-cdn.net/c%3A/Users/Anthony%20Sirico/Documents/GitHub/PyGeo_Circuit_exp/PyGeo_Circuit_exp/iterations/~/AppData/Local/Temp/ipykernel_43252/2131915342.py) in ()
----> 1 len(data2)

[c:\Users\Anthony](file:///C:/Users/Anthony) Sirico\anaconda3\lib\site-packages\torch_geometric\data\dataset.py in __len__(self)
    179     def __len__(self) -> int:
    180         r"""The number of examples in the dataset."""
--> 181         return len(self.indices())
    182 
    183     def __getitem__(

TypeError: 'list' object is not callable

Anyone help on this one, sorry but time is a factor.

@rusty1s

[rusty1s]

You are overriding the indices property of InMemoryDataset in your second dataset, which explains this error:

self.indices = indices