MNIST Superpixel Accuracy is Low

[firqaaa]

Hi, i am very new to graph, in my case i am trying to build custom dataset from MNIST using both grid and superpixels. But when i train the models, the accuracy is very low, it about 0.22. Why can this happen?

Below is my GCN model building for this case :

import torch.nn.functional as F
from torch.nn import Linear, Flatten
from torch_geometric.nn import GCNConv
from torch_geometric.nn import global_mean_pool

class GCN(torch.nn.Module):
    def __init__(self, hidden_channels):
        super(GCN, self).__init__()
        torch.manual_seed(12345)
        self.conv1 = GCNConv(train_dataset.num_features, hidden_channels)
        self.conv2 = GCNConv(hidden_channels, hidden_channels)
        self.conv3 = GCNConv(hidden_channels, hidden_channels)
        self.flatten = Flatten()
        self.lin = Linear(hidden_channels, 10)

    def forward(self, x, edge_index, batch):
        x = self.conv1(x, edge_index)
        x = x.relu()
        x = self.conv2(x, edge_index)
        x = x.relu()
        x = self.conv3(x, edge_index)
        x = global_mean_pool(x, batch)
        x = self.flatten(x)
        x = F.dropout(x, p=0.5, training=self.training)
        x = self.lin(x)

        return x

model = GCN(hidden_channels=32)
print(model)

I use the code below to generate superpixels representation :

import numpy as np
from skimage import color, segmentation

NP_TORCH_FLOAT_DTYPE = np.float32
NP_TORCH_LONG_DTYPE = np.int64

NUM_FEATURES = 3
NUM_CLASSES = 10

def _preprocess(image):
    if len(image.shape) == 2 or image.shape[2] == 1:
        return color.gray2rgb(
            np.reshape(image, (image.shape[0], image.shape[1])))
    else:
        return image

def get_graph_from_image(image,desired_nodes=75):
    # load the image and convert it to a floating point data type
    image = _preprocess(image)
    segments = segmentation.slic(image, desired_nodes, slic_zero = True)
    asegments = np.array(segments)

    num_nodes = np.max(asegments)
    nodes = {
        node: {
            "rgb_list": [],
            "pos_list": []
        } for node in range(num_nodes+1)
    }

    height = image.shape[0]
    width = image.shape[1]
    for y in range(height):
        for x in range(width):
            node = asegments[y,x]
            rgb = image[y,x,:]
            pos = np.array([float(x)/width,float(y)/height])
            nodes[node]["rgb_list"].append(rgb)
            nodes[node]["pos_list"].append(pos)

    G = nx.Graph()
    for node in nodes:
        try:
            nodes[node]["rgb_list"] = np.stack(nodes[node]["rgb_list"])
            nodes[node]["pos_list"] = np.stack(nodes[node]["pos_list"])
            rgb_mean = np.mean(nodes[node]["rgb_list"], axis=0)
            pos_mean = np.mean(nodes[node]["pos_list"], axis=0)
            features = np.concatenate(
                [
                    np.reshape(rgb_mean, -1),
                    np.reshape(pos_mean, -1),
                ]
            )
            G.add_node(node, features = list(features))
        except:
            pass


    # From https://stackoverflow.com/questions/26237580/skimage-slic-getting-neighbouring-segments
    segments_ids = np.unique(segments)

    # centers
    centers = np.array([np.mean(np.nonzero(segments==i),axis=1) for i in segments_ids])

    vs_right = np.vstack([segments[:,:-1].ravel(), segments[:,1:].ravel()])
    vs_below = np.vstack([segments[:-1,:].ravel(), segments[1:,:].ravel()])
    bneighbors = np.unique(np.hstack([vs_right, vs_below]), axis=1)

    # Adjacency loops
    for i in range(bneighbors.shape[1]):
        if bneighbors[0,i] != bneighbors[1,i]:
            G.add_edge(bneighbors[0,i],bneighbors[1,i])

    # Self loops
    for node in nodes:
        G.add_edge(node,node)

    n = G.number_of_nodes()
    m = G.number_of_edges()
    h = np.zeros([n,NUM_FEATURES]).astype(NP_TORCH_FLOAT_DTYPE)
    edges = np.zeros([2*m,2]).astype(NP_TORCH_LONG_DTYPE)
    # print(G.edges())
    for e,(s,t) in enumerate(G.edges()):
        edges[e,0] = s
        edges[e,1] = t

        edges[m+e,0] = t
        edges[m+e,1] = s

    for i in G.nodes():
        try:
            h[i,:] = list(G.nodes(data='features'))[i][1]['features'][:NUM_FEATURES]
        except:
            pass
    #end for
    del G
    return h, edges

This code below is used to create custom dataset in PyG :

import os
import torch
from tqdm.auto import tqdm
from torch_geometric.data import Data, Dataset

class ImageSuperpixelsDataset(Dataset):
    def __init__(self, root, test=False, transform=None, pre_transform=None):
        self.test = test
        super(ImageSuperpixelsDataset, self).__init__(root, transform, pre_transform)

    @property
    def raw_file_names(self):
        return 'ImagesSuperpixels.pt'

    @property
    def processed_file_names(self):
        if self.test:
            return [f'data_test_{i}.pt' for i in range(len(X_test))]
        else:
            return [f'data_{i}.pt' for i in range(len(X_train))]

    def download(self):
        pass

    def process(self):
        if self.test:
            for i, j in tqdm(enumerate(X_test), total=len(X_test)):
                edge_attr, edge_index = get_graph_from_image(X_test[i])
                # Create Data Object
                data = Data(x=torch.tensor(X_test[i].reshape(784, 1), dtype=torch.float),
                            edge_index=torch.tensor(edge_index).t().to(torch.long).view(2, -1),
                            # edge_attr=torch.tensor(edge_attr, dtype=torch.float),
                            y=torch.tensor(y_test[i], dtype=torch.int64))

                if self.test:
                    torch.save(data, os.path.join(self.processed_dir, f'data_test_{i}.pt'))
                else:
                    torch.save(data, os.path.join(self.processed_dir, f'data_{i}.pt'))
        else:
            for i, j in tqdm(enumerate(X_train), total=len(X_train)):
                edge_attr, edge_index = get_graph_from_image(X_train[i])

                # Create Data Object
                data = Data(x=torch.tensor(X_train[i].reshape(784, 1), dtype=torch.float),
                            edge_index=torch.tensor(edge_index).t().to(torch.long).view(2, -1),
                            # edge_attr=torch.tensor(edge_attr, dtype=torch.float),
                            y=torch.tensor(y_train[i], dtype=torch.int64))

                if self.test:
                    torch.save(data, os.path.join(self.processed_dir, f'data_test_{i}.pt'))
                else:
                    torch.save(data, os.path.join(self.processed_dir, f'data_{i}.pt'))

    def len(self):
        if self.test:
            return len(X_test)
        else:
            return len(X_train)

    def get(self, idx):
        if self.test:
            data = torch.load(os.path.join(self.processed_dir, f'data_test_{idx}.pt'))
        else:
            data = torch.load(os.path.join(self.processed_dir, f'data_{idx}.pt'))
        return data

Also it run very slow in my local PC

[rusty1s]

I think the accuracy is low because node positions are not captured in your model. One way to overcome this is to concatenate both RGB values and node positions to a single node feature matrix.

[firqaaa]

this one, right?

            nodes[node]["rgb_list"] = np.stack(nodes[node]["rgb_list"])
            nodes[node]["pos_list"] = np.stack(nodes[node]["pos_list"])
            rgb_mean = np.mean(nodes[node]["rgb_list"], axis=0)
            pos_mean = np.mean(nodes[node]["pos_list"], axis=0)
            features = np.concatenate(
                [
                    np.reshape(rgb_mean, -1),
                    np.reshape(pos_mean, -1),
                ]
            )
            G.add_node(node, features = list(features))

[rusty1s]

Oh, looks like you already do that, sorry for missing this part. Here is an alternative code or applying GNNs on images: https://gist.github.com/rusty1s/866ca8b830f22aa34923a0f8164f6e64

[firqaaa]

Okay, thanks for the reference ...
but can you give me an alternative code to convert image to superpixels graph representation? so i can follow it

[rusty1s]

I don‘t have such an example, sorry. The only example I can point you to is the superpixel MNIST example under examples and the transforms.ToSLIC transform.