Single underlying graph

[adhikarirsr]

I have a single underlying graph (let's say for a picture the graph is a grid graph with some edge_weight). So, when I create Dataset, I do not want to have that graph (edge_index and edge_weight) as a part of every data. I should be able to refer to one edge_index and one edge_weight value from the memory. This would save lots of memory, right? Is PyG already doing that? If not then how can I do so?

[rusty1s]

I think you can simply set these values as an attribute to dataset, e.g.:

class MyDataset(InMemoryDataset):
    def __init__(self):
        super().__init__(...)
        self.data, self.slices = torch.load(self.processed_paths[0])
        self.edge_index = torch.load('link to saved edge_index')

    def process(self):
        ...
        torch.save(edge_index, 'link to saved edge index')

[adhikarirsr]

I tried to create out of memory data but I am getting an error. In my case edge_index and edge_weight is same for all the data.

from torch_geometric.data import Dataset, download_url
import os.path as osp
class Gdata_cre(Dataset):
    def __init__(self, root, train=True, transform=None):
        self.train_dataset = Gdata(root, train=True, download=True)
        self.test_dataset = Gdata(root, train=False)
        super(Gdata_cre, self).__init__(root, transform)


    @property
    def processed_file_names(self):
        return ['train.pt', 'test.pt']
  
    def process(self):
        ii = 0
        jj = 0
        edge_index =edge_index2

        data_list = []
        s = 1
        for x, y in self.train_dataset:
            x = ToTensor()(x)
            s = s+1
            xc = x
            x = add_noise(x,s)
            xn = x
            x , m = masker.mask(x.unsqueeze(0),0)
            x = x.reshape((1,784))
            x1 = x.view(-1,1)
            deg1 = deg.view(-1,1)
            avx= torch.tensor(np.matmul(avg_mat,np.array(x1)))
                
            x1 = torch.cat((x1,deg1,avx),1)

            data = Data(x=x1.type(torch.float), edge_index=edge_index, edge_weight=edge_weight1, xn =xn.view(-1,1),m = m.view(-1,1),xc =xc.view(-1,1))
            torch.save(data, osp.join(self.processed_dir, 'data_{}.pt'.format(ii)))
            ii += 1
        
        data_list = []
        b = 3425
        for x, y in self.test_dataset:
            xc = ToTensor()(x)
            b = b + 1
            x = add_noise(xc,b)
            x1 = x.view(-1,1)
            deg1 = deg.view(-1,1)
            avx= torch.tensor(np.matmul(avg_mat,np.array(x1)))                
            x1 = torch.cat((x1,deg1,avx),1)
            
            data = Data(x=x1.type(torch.float), edge_index=edge_index,edge_weight=edge_weight1,xc=xc)
            torch.save(data, osp.join(self.processed_dir, 'data_{}.pt'.format(jj)))
            jj += 1

    def len(self):
        return len(self.processed_file_names)

    def get(self, idx):
        data = torch.load(osp.join(self.processed_dir, 'data_{}.pt'.format(idx)))
        return data

This runs fine but takes a lot of time:

train_dataset = Gdata_cre('/home/graph/tester4_corr_rad_2', train=True)
test_dataset = Gdata_cre('/home/graph/tester4_corr_rad_2', train=False)

and this also runs fine:

bt = DataLoader(train_dataset, batch_size=1, shuffle=True)
bt = next(iter(bt))
bt

However,
train_set, val_set = torch.utils.data.random_split(train_dataset, [51000, 2000], generator=torch.Generator().manual_seed(42))
throws:

~/jupy37/lib/python3.7/site-packages/torch/utils/data/dataset.py in random_split(dataset, lengths, generator)
    349     # Cannot verify that dataset is Sized
    350     if sum(lengths) != len(dataset):  # type: ignore
--> 351         raise ValueError("Sum of input lengths does not equal the length of the input dataset!")
    352 
    353     indices = randperm(sum(lengths), generator=generator).tolist()

ValueError: Sum of input lengths does not equal the length of the input dataset!

[adhikarirsr]

I also tried your suggestion:

class Gdata_cre(InMemoryDataset):
    def __init__(self, root, train=True, transform=None):
        self.train_dataset = Gdata(root, train=True, download=True)
        self.test_dataset = Gdata(root, train=False)
        super(Gdata_cre, self).__init__(root, transform)
        path = self.processed_paths[0] if train else self.processed_paths[1]
        self.data, self.slices = torch.load(path)
        self.edge_index = edge_index2


    @property
    def processed_file_names(self):
        return ['train.pt', 'test.pt']
    
    
    def process(self):

        data_list = []
        s = 1
        for x, y in self.train_dataset:
            x = ToTensor()(x)
            s = s+1
            xc = x
            x = add_noise(x,s)
            xn = x
            x , m = masker.mask(x.unsqueeze(0),0)
            x = x.reshape((1,784))
            x1 = x.view(-1,1)
            deg1 = deg.view(-1,1)
            avx= torch.tensor(np.matmul(avg_mat,np.array(x1)))
                
            x1 = torch.cat((x1,deg1,avx),1)

            data = Data(x=x1.type(torch.float), edge_weight=edge_weight1, xn =xn.view(-1,1),m = m.view(-1,1),xc =xc.view(-1,1))
            data_list.append(data)
        torch.save(self.collate(data_list), self.processed_paths[0])
        torch.save(edge_index, edge_index2)

        data_list = []
        b = 3425
        for x, y in self.test_dataset:
            xc = ToTensor()(x)
            b = b + 1
            x = add_noise(xc,b)
            x1 = x.view(-1,1)
            deg1 = deg.view(-1,1)
            avx= torch.tensor(np.matmul(avg_mat,np.array(x1)))
            x1 = torch.cat((x1,deg1,avx),1)           
            data = Data(x=x1.type(torch.float),edge_weight=edge_weight1,xc=xc)
            data_list.append(data)
        torch.save(self.collate(data_list), self.processed_paths[1])
        torch.save(edge_index, edge_index2)

I get this error:
NameError: name 'edge_index' is not defined

[rusty1s]

It seems that edge_index is never be loaded or created, that's why it is undefined. My above code just showed the blueprint, you still need to take care of loading or creating the initial edge_index tensor somewhere.