class input issue

[anthonysirico]

I am having an error with the super().__init__ line in my code and i cannot figure it out.

class ImbalanceDataset(InMemoryDataset):
    def __init__(self, root: str, split: float, set: str, transform=None, pre_transform=None, pre_filter=None):
        
        self.split = split
        self.known_data = None
        self.unknown_data = None
        self.All = None
        self.raw_data = None
        self.set = set
        
        super().__init__(root, transform, pre_transform, pre_filter)

        if set == 'known':
           path = self.processed_paths[0]
        elif set == 'unknown':
            path = self.processed_paths[1]
        elif set == 'All':
            path = self.processed_paths[2]                 

        self.data, self.slices = torch.load(path)
    
    @property
    def raw_file_names(self):
        return 'C:\\Users\\Anthony Sirico\\Documents\\GitHub\\PyGeo_Circuit_exp\\PyGeo_Circuit_exp\\shuffled_data.mat'

    @property
    def processed_file_names(self) -> list[str]:
        return ['known.pt', 'unknown.pt', 'All.pt']

    def download(self):
        pass

    def process(self):
        self.raw_data = loadmat(self.raw_file_names, squeeze_me=True)
        data = pd.DataFrame(self.raw_data['Graphs'])
        data = data.sample(frac=1).reset_index(drop=True)

        self.known_data = data[:int(len(data)*self.split)]
        self.unknown_data = data[int(len(data)*self.split):]
        self.All = data

        for i in range(len(self.known_data)):
            self.known_data['Ln'][i] = dict(enumerate(self.known_data['Ln'][i]))
        
        for i in range(len(self.unknown_data)):
            self.unknown_data['Ln'][i] = dict(enumerate(self.unknown_data['Ln'][i]))
        
        for i in range(len(self.All)):
            self.All['Ln'][i] = dict(enumerate(self.All['Ln'][i]))
        
        r'''Create Known Graphs'''
        known_list = []
        for index, cir in tqdm(self.known_data.iterrows(), total=len(self.known_data)):
            nxg = nx.Graph(self.known_data['A'][index])
            nx.set_node_attributes(nxg, self.known_data['Ln'][index], 'component')
            pt_graph = self._get_graph_object(nxg)
            pt_graph.x = self._get_node_features(nxg, self.known_data['Ln'][index])
            pt_graph.y = self._get_graph_label(self.known_data['binary'][index])
            pt_graph.performance = torch.tensor(self.known_data['Labels'][index], dtype=torch.float)
            known_list.append(pt_graph)

        r'''Create Unknown Graphs'''
        unknown_list = []
        for index, cir in tqdm(self.unknown_data.iterrows(), total=len(self.unknown_data)):
            nxg = nx.Graph(self.unknown_data['A'][index])
            nx.set_node_attributes(nxg, self.unknown_data['Ln'][index], 'component')
            pt_graph = self._get_graph_object(nxg)
            pt_graph.x = self._get_node_features(nxg, self.unknown_data['Ln'][index])
            pt_graph.y_all = self._get_graph_label(self.unknown_data['binary_all'][index])
            pt_graph.y_known = self._get_graph_label(self.unknown_data['binary_known'][index])
            pt_graph.performance = torch.tensor(self.unknown_data['Labels'][index], dtype=torch.float)
            unknown_list.append(pt_graph)
        
        r'''Create All Graphs'''
        all_list = []
        for index, cir in tqdm(self.All.iterrows(), total=len(self.All)):
            nxg = nx.Graph(self.All['A'][index])
            nx.set_node_attributes(nxg, self.All['Ln'][index], 'component')
            pt_graph = self._get_graph_object(nxg)
            pt_graph.x = self._get_node_features(nxg, self.All['Ln'][index])
            pt_graph.y = self._get_graph_label(self.All['binary'][index])
            pt_graph.performance = torch.tensor(self.All['Labels'][index], dtype=torch.float)
            all_list.append(pt_graph)

        if self.pre_filter is not None:
            known_list = [data for data in known_list if self.pre_filter(data)]
            unknown_list = [data for data in unknown_list if self.pre_filter(data)]
            all_list = [data for data in all_list if self.pre_filter(data)]
        
        if self.pre_transform is not None:
            known_list = [self.pre_transform(data) for data in known_list]
            unknown_list = [self.pre_transform(data) for data in unknown_list]
            all_list = [self.pre_transform(data) for data in all_list]
        
        data_known, slices_known = self.collate(known_list)
        data_unknown, slices_unknown = self.collate(unknown_list)
        data_all, slices_all = self.collate(all_list)

        torch.save((data_known, slices_known), self.processed_paths[0])
        torch.save((data_unknown, slices_unknown), self.processed_paths[1])
        torch.save((data_all, slices_all), self.processed_paths[2])


    def _get_node_features(self, nx_graph, node_labels):
        betweenness = list(nx.betweenness_centrality(nx_graph).values())
        eigenvector = list(nx.eigenvector_centrality(nx_graph, max_iter=600).values())

        mapping_dict = {'C': 0, 'G': 1, 'I': 2, 'O': 3, 'P': 4, 'R': 5}
        component_labels = []

        for value in node_labels.values():
            if value in mapping_dict:
                component_labels.append(mapping_dict[value])

        all_feats = [component_labels, betweenness, eigenvector]

        all_feats = np.asarray(all_feats).transpose()

        return torch.tensor(all_feats, dtype=torch.float)
    
    def _get_graph_object(self, nx_graph):
        nxg = from_networkx(nx_graph)
        return nxg
    
    def _get_graph_label(self, label):
        binary = np.asarray([label])
        return torch.tensor(binary, dtype=torch.int64)
        
    @property
    def num_node_features(self) -> int:
        return 3

    @property
    def num_classes(self) -> int:
        return 2

When I run known = imb.ImbalanceDataset(root='imb_50v2', set='known', split=0.5) it raises the error:

Output exceeds the [size limit](command:workbench.action.openSettings?[). Open the full output data [in a text editor](command:workbench.action.openLargeOutput?f4c2c7bb-3d1a-46d0-8b68-f622f9e1fa3d)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
c:\Users\Anthony Sirico\anaconda3\lib\site-packages\pandas\core\indexes\range.py in get_loc(self, key, method, tolerance)
    391                 try:
--> 392                     return self._range.index(new_key)
    393                 except ValueError as err:

ValueError: 0 is not in range

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_119600\2291953209.py in <cell line: 17>()
     15 from tqdm import tqdm
     16 
---> 17 known = imb.ImbalanceDataset(root='imb_50v2', set='known', split=0.5)
     18 unknown = imb.ImbalanceDataset(root='imb_50v2', set='unknown', split=0.5)
     19 all_data = imb.ImbalanceDataset(root='imb_50v2', set='All')

c:\Users\Anthony Sirico\Documents\GitHub\PyGeo_Circuit_exp\PyGeo_Circuit_exp\imbalance_exp\imb_dataset.py in __init__(self, root, split, set, transform, pre_transform, pre_filter)
     28         self.set = set
     29 
---> 30         super().__init__(root, transform, pre_transform, pre_filter)
     31 
     32         if set == 'known':
...
--> 394                     raise KeyError(key) from err
    395             self._check_indexing_error(key)
    396             raise KeyError(key)

KeyError: 0

[rusty1s]

There seems to be some invalid index access in one of your pandas call - a bit hard for me to debug by just looking at the code :(

[anthonysirico]

So it ended up being the pandas dataframe. After splitting the data into separate sets, I needed to reset the index.