sparse tensors are different after saving and loading dataset

[dottipr]

Hello,

I am trying to implement the neural network of https://github.com/ryanzhangfan/NeuroSAT using Pytorch Geometric, but the loss function is almost always constant. After several attempts of debugging my code, I realised that the loss is constant when I load an already processed dataset and the learning works when I train the network on a freshly created dataset.

This is what my dataset looks like:

class SATDataset(Dataset):
    def __init__(self, root, d):
        # root: location of dataset
        # d: number of features for x_l and x_c
        self.root = root
        self.d = d

        # get dimacs filenames
        self.sorted_file_names = sorted([f for f in listdir(self.root)
                                   if osp.isfile(osp.join(self.root,f))])

        # define name for processed data
        self.processed_name = "data.pt"

        # process dimacs and get list of problems
        #self.data_list = self.process()

        super(SATDataset, self).__init__(root=self.root)

    @property
    def raw_dir(self):
        return self.root

    @property
    def raw_file_names(self):
        sorted_file_names = sorted([f for f in listdir(self.raw_dir) if osp.isfile(osp.join(self.raw_dir,f))])
        return sorted_file_names

    @property
    def processed_file_names(self):
        processed_name = self.processed_name
        return processed_name

    def process(self):
        self.data_list = []
        for raw_path in self.raw_paths:
            n_vars, clauses = self.parse_dimacs(raw_path)
            # n_vars is the number of variables according to the .dimacs file
            # clauses is a list of lists (=clauses) of numbers (=literals)

            y, _ = solve_sat(n_vars, clauses) # get problem label (sat/unsat)

            # create graph instance (Problem)
            p = self.create_problem(n_vars, clauses, y)

            self.data_list.append(p)
        torch.save(self.data_list, osp.join(self.processed_dir,
                                            self.processed_name ))

    def _process(self):
        f = osp.join(self.processed_dir, 'pre_transform.pt')
        if osp.exists(f) and torch.load(f) != __repr__(self.pre_transform):
            logging.warning(
                'The `pre_transform` argument differs from the one used in '
                'the pre-processed version of this dataset. If you really '
                'want to make use of another pre-processing technique, make '
                'sure to delete `{}` first.'.format(self.processed_dir))
        f = osp.join(self.processed_dir, 'pre_filter.pt')
        if osp.exists(f) and torch.load(f) != __repr__(self.pre_filter):
            logging.warning(
                'The `pre_filter` argument differs from the one used in the '
                'pre-processed version of this dataset. If you really want to '
                'make use of another pre-fitering technique, make sure to '
                'delete `{}` first.'.format(self.processed_dir))

        if files_exist(self.processed_paths):  # pragma: no cover
            # if data already processed, load it
            print("Loading saved dataset...")
            f = osp.join(self.processed_dir, self.processed_name )
            self.data_list = torch.load(f)
            return

        print('Processing...')

        makedirs(self.processed_dir)
        self.process()

        path = osp.join(self.processed_dir, 'pre_transform.pt')
        torch.save(__repr__(self.pre_transform), path)
        path = osp.join(self.processed_dir, 'pre_filter.pt')
        torch.save(__repr__(self.pre_filter), path)

        print('Done!')


    # parse dimacs file
    def parse_dimacs(self, filename):
        [...]

    # create Problem instance (graph) from parsed dimacs
    def create_problem(self, n_vars, clauses, y):
        # d is the number of features of x_l and x_c

        n_lits = int(2 * n_vars)
        n_clauses = len(clauses)

        # create feature vectors for lits and clauses
        l_init = torch.normal(mean=0.0, std=1.0, size=(1,self.d))
        c_init = torch.normal(mean=0.0, std=1.0, size=(1,self.d))
        denom = torch.sqrt(torch.tensor(self.d, dtype=torch.float64))

        x_l = (torch.div(l_init, denom)).repeat(n_lits, 1)
        x_c = (torch.div(c_init, denom)).repeat(n_clauses, 1)

        # get graph edges from list of clauses
        edge_index = [[],[]]
        for i,clause in enumerate(clauses):
            # get idxs of lits in clause
            lits_indices = [self.from_lit_to_idx(l, n_vars) for l in clause]
            clauses_indices = len(clause) * [i]

            # add all edges connected to clause i to edge_index
            edge_index[0].extend(lits_indices)
            edge_index[1].extend(clauses_indices)

        # convert edge_index to tensor
        edge_index = torch.tensor(edge_index, dtype=torch.long)

        return Problem(edge_index, x_l, x_c, y)

    def from_lit_to_idx(self, lit, n_vars):
        [...]

    def __len__(self):
        num_samples = len(self.raw_paths)
        return num_samples

    def __getitem__(self, idx):
        data = self.data_list[idx]
        return data

(I decided to modify the _process method because the dataset is very large and saving every graph as a single file was making the dataloader extremely slow...maybe I did it in the wrong way)

And this is how graphs in my dataset are defined:

from torch_sparse import SparseTensor

class Problem(Data):
    # a Problem is a bipartite graph

    def __init__(self, edge_index=None, x_l=None, x_c=None, y=None):
        # edge_index is a bipartite adjacency matrix between lits and clauses
        # x_l is the feature vector of the lits nodes
        # x_c is the feature vector of the clauses nodes
        # y is the label: 1 if sat, 0 if unsat

        super(Problem, self).__init__()
        # nodes features
        self.x_l = x_l
        self.x_c = x_c
        self.y = y

        self.num_literals = x_l.size(0) if x_l is not None else 0
        self.num_clauses = x_c.size(0) if x_c is not None else 0

        # edges
        self.edge_index = edge_index
        self.adj_t = SparseTensor(row = edge_index[1],
                                  col = edge_index[0],
                                  sparse_sizes = [self.num_clauses, self.num_literals]
                                 ) if edge_index is not None else 0

        # compute number of variables
        assert self.num_literals %2 == 0
        self.num_vars = self.num_literals // 2

        self.num_nodes = self.num_literals + self.num_clauses

    def __inc__(self, key, value):
        if key == 'edge_index':
            return torch.tensor([[self.x_l.size(0)], [self.x_c.size(0)]])
        else:
            return super().__inc__(key, value)

I've created a script where I first create and process the dataset (dataset_train = SATDataset(dimacs_dir_train, d)) and then I create another dataset which should be equal (called dataset_train_load), with the only difference being that the samples are loaded from memory.

I compared all the attributes of the samples in the two datasets and found out that the adjacency matrices are not equal (even if they seem so).

logger.info(f"ADJ:\n{dataset_train[0].adj_t}") returns

SparseTensor(row=tensor([ 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3,
3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6,
6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10,
10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 15, 15,
15, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 18, 18, 19, 19, 19, 20, 20,
20, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 24, 24,
25, 25, 25, 25, 25, 25, 26, 26, 26, 27, 27, 27, 27, 28, 28, 28, 29, 29,
29, 30, 30, 30, 30, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34,
35, 35, 35, 35, 35, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 39, 39, 39,
40, 40, 40, 41, 41, 41, 41, 41, 42, 42, 42, 43, 43, 43, 44, 44, 44, 44,
44, 44, 44, 44, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47,
48, 48, 48, 49, 49, 49, 50, 50, 50, 51, 51, 52, 52, 52, 52, 52, 52, 53,
53, 53, 53, 53, 54, 54, 55, 55, 55, 56, 56, 56, 56, 56, 56, 56, 57, 57,
57, 57, 58, 58, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 61, 61,
61, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65,
65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68,
68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 71, 71, 72, 72, 72,
72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 76, 76, 76, 77, 77, 77, 77,
78, 78, 78, 78, 79, 79]),
col=tensor([10, 11, 12, 13, 14, 15, 0, 4, 6, 11, 13, 18, 5, 16, 19, 4, 5, 8,
9, 10, 11, 13, 16, 17, 10, 13, 16, 18, 3, 10, 11, 14, 15, 19, 0, 2,
5, 9, 13, 18, 2, 5, 7, 2, 4, 5, 7, 8, 9, 10, 11, 13, 16, 0,
1, 4, 5, 18, 11, 13, 15, 19, 0, 7, 9, 12, 7, 11, 5, 9, 1, 2,
13, 0, 1, 6, 3, 5, 9, 11, 14, 17, 18, 2, 5, 8, 15, 17, 1, 5,
13, 1, 5, 6, 13, 19, 1, 4, 8, 10, 12, 15, 17, 6, 17, 19, 5, 13,
0, 4, 7, 8, 12, 19, 10, 14, 16, 0, 2, 15, 17, 1, 2, 15, 5, 11,
17, 4, 10, 17, 18, 7, 10, 13, 0, 2, 9, 11, 15, 17, 0, 13, 2, 14,
2, 8, 10, 14, 19, 2, 7, 10, 2, 4, 8, 19, 0, 3, 18, 13, 15, 19,
8, 16, 17, 4, 10, 15, 17, 18, 3, 9, 14, 0, 5, 6, 1, 2, 3, 7,
10, 15, 18, 19, 2, 5, 14, 0, 2, 5, 7, 13, 14, 16, 3, 5, 6, 9,
4, 18, 19, 3, 8, 9, 3, 12, 17, 8, 17, 0, 4, 9, 11, 15, 17, 0,
8, 13, 14, 17, 11, 15, 3, 5, 19, 1, 5, 6, 7, 14, 18, 19, 6, 10,
15, 17, 4, 12, 0, 1, 12, 14, 16, 18, 2, 6, 8, 9, 10, 17, 11, 13,
18, 0, 6, 7, 5, 10, 14, 16, 0, 5, 13, 16, 18, 0, 4, 6, 8, 11,
12, 13, 17, 19, 6, 8, 9, 13, 5, 6, 8, 10, 11, 3, 4, 5, 9, 10,
11, 16, 17, 2, 7, 8, 9, 0, 8, 9, 13, 14, 15, 8, 13, 8, 15, 16,
17, 19, 4, 13, 16, 5, 7, 12, 13, 17, 19, 4, 8, 17, 15, 16, 17, 18,
3, 11, 17, 18, 12, 15]),
size=(80, 20), nnz=330, density=20.62%)

and logger.info(f"ADJ:\n{dataset_train_load[0].adj_t}") returns

ADJ:
SparseTensor(row=tensor([ 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3,
3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6,
6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10,
10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 15, 15,
15, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 18, 18, 19, 19, 19, 20, 20,
20, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 24, 24,
25, 25, 25, 25, 25, 25, 26, 26, 26, 27, 27, 27, 27, 28, 28, 28, 29, 29,
29, 30, 30, 30, 30, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34,
35, 35, 35, 35, 35, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 39, 39, 39,
40, 40, 40, 41, 41, 41, 41, 41, 42, 42, 42, 43, 43, 43, 44, 44, 44, 44,
44, 44, 44, 44, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47,
48, 48, 48, 49, 49, 49, 50, 50, 50, 51, 51, 52, 52, 52, 52, 52, 52, 53,
53, 53, 53, 53, 54, 54, 55, 55, 55, 56, 56, 56, 56, 56, 56, 56, 57, 57,
57, 57, 58, 58, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 61, 61,
61, 62, 62, 62, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65,
65, 65, 65, 65, 66, 66, 66, 66, 67, 67, 67, 67, 67, 68, 68, 68, 68, 68,
68, 68, 68, 69, 69, 69, 69, 70, 70, 70, 70, 70, 70, 71, 71, 72, 72, 72,
72, 72, 73, 73, 73, 74, 74, 74, 75, 75, 75, 76, 76, 76, 77, 77, 77, 77,
78, 78, 78, 78, 79, 79]),
col=tensor([10, 11, 12, 13, 14, 15, 0, 4, 6, 11, 13, 18, 5, 16, 19, 4, 5, 8,
9, 10, 11, 13, 16, 17, 10, 13, 16, 18, 3, 10, 11, 14, 15, 19, 0, 2,
5, 9, 13, 18, 2, 5, 7, 2, 4, 5, 7, 8, 9, 10, 11, 13, 16, 0,
1, 4, 5, 18, 11, 13, 15, 19, 0, 7, 9, 12, 7, 11, 5, 9, 1, 2,
13, 0, 1, 6, 3, 5, 9, 11, 14, 17, 18, 2, 5, 8, 15, 17, 1, 5,
13, 1, 5, 6, 13, 19, 1, 4, 8, 10, 12, 15, 17, 6, 17, 19, 5, 13,
0, 4, 7, 8, 12, 19, 10, 14, 16, 0, 2, 15, 17, 1, 2, 15, 5, 11,
17, 4, 10, 17, 18, 7, 10, 13, 0, 2, 9, 11, 15, 17, 0, 13, 2, 14,
2, 8, 10, 14, 19, 2, 7, 10, 2, 4, 8, 19, 0, 3, 18, 13, 15, 19,
8, 16, 17, 4, 10, 15, 17, 18, 3, 9, 14, 0, 5, 6, 1, 2, 3, 7,
10, 15, 18, 19, 2, 5, 14, 0, 2, 5, 7, 13, 14, 16, 3, 5, 6, 9,
4, 18, 19, 3, 8, 9, 3, 12, 17, 8, 17, 0, 4, 9, 11, 15, 17, 0,
8, 13, 14, 17, 11, 15, 3, 5, 19, 1, 5, 6, 7, 14, 18, 19, 6, 10,
15, 17, 4, 12, 0, 1, 12, 14, 16, 18, 2, 6, 8, 9, 10, 17, 11, 13,
18, 0, 6, 7, 5, 10, 14, 16, 0, 5, 13, 16, 18, 0, 4, 6, 8, 11,
12, 13, 17, 19, 6, 8, 9, 13, 5, 6, 8, 10, 11, 3, 4, 5, 9, 10,
11, 16, 17, 2, 7, 8, 9, 0, 8, 9, 13, 14, 15, 8, 13, 8, 15, 16,
17, 19, 4, 13, 16, 5, 7, 12, 13, 17, 19, 4, 8, 17, 15, 16, 17, 18,
3, 11, 17, 18, 12, 15]),
size=(80, 20), nnz=330, density=20.62%)

However if I test these two sparse matrices for equality (logger.info(f"ADJ:\n{dataset_train[0].adj_t==dataset_train_load[0].adj_t}")) I get False.

Does anybody have an idea of why this is the case and how it could be solved?

[rusty1s]

It seems like you are loading your complete graph dataset to CPU memory anyway, so using InMemoryDataset is probably the preferred way (but shouldn't make any difference except for a little bit faster loading times).

Your two adjacency matrices are identical, but you cannot infer that via == as it will simply check for pointer equivalence. Instead, do something like:

row1, col1, _ = dataset_train[0].adj_t.coo()
row2, col2, _ = dataset_train_load[0].adj_t.coo()
assert row1.tolist() == row2.tolist()
assert col1.tolist() == col2.tolist()

[dottipr]

Thanks a lot for your answer.

I checked with the code you shared and the two matrices are indeed identical. I've also changed the class of the dataset to InMemoryDataset and I will try training the network with it later, thanks for the advice.

Now I am even more puzzled by the fact that the learning only works when I am not loading the processed data from memory... could you maybe notice if there are any mistakes in the definition of my graphs' class (It should represent a bipartite graph) or in the dataset definition?