Converting a data object to a dataset for reusing such as Planetoid, TUDataset

[abose17]

Hi Altruists,
Can anyone tell me how to make a custom dataset from a data object? I am needing this to save it in a local machine such as Planetoid, TUDataset? Following is my data object created by the following snippet of code

data = Data(x=id_matrix, edge_index=edge_index.contiguous(), y = target_tensor)
-> Data(x=[8114, 8114], edge_index=[2, 303074], y=[8114], train_mask=[5000], test_mask=[1000], val_mask=[500])

The neighborLoader also works fine with my created PYG data object

loader = NeighborLoader(
    data,
    # Sample 30 neighbors for each node for 2 iterations
    num_neighbors=[30] * 2,
    # Use a batch size of 128 for sampling training nodes
    batch_size=128,
    input_nodes=data.train_mask,
)
sampled_data = next(iter(loader))
print(sampled_data.batch_size)

-> 128

I am struggling to make my data object to a dataset using the following skeleton code

import os.path as osp
import torch
from torch_geometric.data import Dataset, download_url

class MyOwnDataset(Dataset):
    def __init__(self, root, transform=None, pre_transform=None, pre_filter=None):
        super().__init__(root, transform, pre_transform, pre_filter)

    @property
    def raw_file_names(self):
        return ['some_file_1', 'some_file_2', ...]

    @property
    def processed_file_names(self):
        return ['data_1.pt', 'data_2.pt', ...]

    def download(self):
        # Download to `self.raw_dir`.
        path = download_url(url, self.raw_dir)
        ...

    def process(self):
        idx = 0
        for raw_path in self.raw_paths:
            # Read data from `raw_path`.
            data = Data(...)

            if self.pre_filter is not None and not self.pre_filter(data):
                continue

            if self.pre_transform is not None:
                data = self.pre_transform(data)

            torch.save(data, osp.join(self.processed_dir, f'data_{idx}.pt'))
            idx += 1

    def len(self):
        return len(self.processed_file_names)

    def get(self, idx):
        data = torch.load(osp.join(self.processed_dir, f'data_{idx}.pt'))
        return data

This question is also asked here https://torchgeometricco.slack.com/archives/CRAR9HK2P/p1661978641653829

[abose17]

@rusty1s
@Matthias Fey
I want to use it like other benchmark data sets so that I can generate results chronologically with other data sets. My code is written like that.
I have a question: Does saving the data object using torch.save() retain all graph information?
I found these two following issue issues

https://github.com/pyg-team/pytorch_geometric/issues/1834
https://github.com/pyg-team/pytorch_geometric/issues/1359

Data transform function

def transform(data):
    row, col = data.edge_index
    deg = degree(col, data.num_nodes)
    data.edge_attr = torch.stack(
        [1 / torch.sqrt(deg[row]), 1 / torch.sqrt(deg[col])], dim=-1)
    print("data.edge_attr size-> ", data.edge_attr.size())
    return data
transform = transform(data)

Saving the data object using this

torch.save(data, "my_path/ctiRaw/ctiRaw_V1.pt")

I used your provided skeleton

class CTIDataset(InMemoryDataset):
    def __init__(self, root, transform):
        super(CTIDataset, self).__init__(root, transform)
        self.data, self.slices = torch.load(self.processed_paths[0])

    def processed_file_names(self):
        return ['data.pt']

    def process(self):
        # Generate data here or expect it to be in global scope.
        self.collate([data], self.processed_paths[0])

I tried to use the above class to make the custom data set

ctData = CTIDataset("my_path/ctiRaw", transform)

Generates this error

input In [231], in CTIDataset.__init__(self, root, transform)
      2 def __init__(self, root, transform):
----> 3     super(CTIDataset, self).__init__(root, transform)
      4     self.data, self.slices = torch.load(self.processed_paths[0])

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/site-packages/torch_geometric/data/in_memory_dataset.py:56, in InMemoryDataset.__init__(self, root, transform, pre_transform, pre_filter)
     52 def __init__(self, root: Optional[str] = None,
     53              transform: Optional[Callable] = None,
     54              pre_transform: Optional[Callable] = None,
     55              pre_filter: Optional[Callable] = None):
---> 56     super().__init__(root, transform, pre_transform, pre_filter)
     57     self.data = None
     58     self.slices = None

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/site-packages/torch_geometric/data/dataset.py:87, in Dataset.__init__(self, root, transform, pre_transform, pre_filter)
     84     self._download()
     86 if 'process' in self.__class__.__dict__:
---> 87     self._process()

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/site-packages/torch_geometric/data/dataset.py:164, in Dataset._process(self)
    157 if osp.exists(f) and torch.load(f) != _repr(self.pre_filter):
    158     warnings.warn(
    159         "The `pre_filter` argument differs from the one used in the "
    160         "pre-processed version of this dataset. If you want to make "
    161         "use of another pre-fitering technique, make sure to delete "
    162         "'{self.processed_dir}' first")
--> 164 if files_exist(self.processed_paths):  # pragma: no cover
    165     return
    167 print('Processing...', file=sys.stderr)

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/site-packages/torch_geometric/data/dataset.py:138, in Dataset.processed_paths(self)
    135 r"""The absolute filepaths that must be present in order to skip
    136 processing."""
    137 files = to_list(self.processed_file_names)
--> 138 return [osp.join(self.processed_dir, f) for f in files]

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/site-packages/torch_geometric/data/dataset.py:138, in <listcomp>(.0)
    135 r"""The absolute filepaths that must be present in order to skip
    136 processing."""
    137 files = to_list(self.processed_file_names)
--> 138 return [osp.join(self.processed_dir, f) for f in files]

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/posixpath.py:90, in join(a, *p)
     88             path += sep + b
     89 except (TypeError, AttributeError, BytesWarning):
---> 90     genericpath._check_arg_types('join', a, *p)
     91     raise
     92 return path

File ~/anaconda3/envs/pyTorch_Geo/lib/python3.9/genericpath.py:152, in _check_arg_types(funcname, *args)
    150         hasbytes = True
    151     else:
--> 152         raise TypeError(f'{funcname}() argument must be str, bytes, or '
    153                         f'os.PathLike object, not {s.__class__.__name__!r}') from None
    154 if hasstr and hasbytes:
    155     raise TypeError("Can't mix strings and bytes in path components") from None

TypeError: join() argument must be str, bytes, or os.PathLike object, not 'method'

[rusty1s]

There are two bugs in the code example:

• One needs to set processed_file_names to a property
• There was a torch.save missing in process()
  Here is the updated code that runs through for me:

import torch

from torch_geometric.data import Data, InMemoryDataset

data = Data(x=torch.randn(100, 24))


class CTIDataset(InMemoryDataset):
    def __init__(self, root, transform=None):
        super().__init__(root, transform)
        self.data, self.slices = torch.load(self.processed_paths[0])

    @property
    def processed_file_names(self):
        return ['data.pt']

    def process(self):
        # Generate data here or expect it to be in global scope.
        torch.save(self.collate([data]), self.processed_paths[0])


dataset = CTIDataset('/tmp/bla')
print(dataset)
print(dataset[0])

[abose17]

@rusty1s Thanks much!
The code works fine without showing any error.