updated docs

Author: adhocmaster

ChangeLog.txt

@@ -2,4 +2,6 @@ Oct 14, 2022
 1. generate_patches has a new parameter for output directory
 2. default output directory now has a meaningful structure with patch sizes and step sizes
 3. Refactored ImageGroupReader
-4. generate_patches normalizes the reader data once.
+4. generate_patches normalizes the reader data once.
+5. window_size -> patch_size
+6. step_size -> stride

README.md

@@ -13,7 +13,72 @@ Generate and load dataset of road network maps.
 pip install mapdatasetgenerator
 ```
 
-# Usage
+# Creating patches
 ```
-import mapdataset
+# Run this script to generate data in /output directory.
+import logging
+import sys
+
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+root.addHandler(handler)
+
+from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader
+
+
+sfMap = MapReader('./data/input/sf_layered.txt', "SF_Layered")
+mapsDataset = MapsDataset(
+    patch_size=(32, 32), 
+    stride=10, 
+    sample_group_size=1280, 
+    converter=single_layer_converter,
+    outputDir="./data/output"
+    ) 
+    
+mapsDataset.generate_patches(sfMap) #This will generate dill files which contain the saved sample lists.
+
+```
+
+# Reading patches
+
 ```
+# Script to read dill data objects as numpy arrays.
+from PIL import Image
+import os
+import sys
+import logging
+
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+root.addHandler(handler)
+
+from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader
+
+
+
+dillFolder = "./data/output/SF_Layered/32x32/stride-10"
+nGroups = 0
+# Iterate directory
+for path in os.listdir(dillFolder):
+    # check if current path is a file
+    if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"):
+        nGroups += 1
+
+for i in range(nGroups):
+    reader = ImageGroupReader(dillFolder)
+    data = reader.load_group(groupNo=i)
+    patchImgArray = reader.asImg(data[0])
+    im = Image.fromarray(patchImgArray)
+    path = os.path.join(dillFolder, f"{i}-0.png")
+    im.save(path)
+```

src/mapdataset/lib/__init__.py

@@ -1 +1,2 @@
-from .ImageGroupReader import ImageGroupReader
+from .ImageGroupReader import ImageGroupReader
+from .maps import *

src/mapdataset/maps.py src/mapdataset/lib/maps.pysrc/mapdataset/maps.py renamed to src/mapdataset/lib/maps.py

@@ -53,15 +53,15 @@ def get_char(self, layer):
 # Modifications - Ishaan, Muktadir
 
 class MapsDataset(Dataset):
-    def __init__(self, window_size, step_size, sample_group_size, converter, outputDir="../data/output"):
+    def __init__(self, patch_size, stride, sample_group_size, converter, outputDir="../data/output"):
         self.outputDir = outputDir
         self.char_size = converter.char_size
         self.converter = converter
-        self.window_size = window_size
-        self.step_size = step_size
+        self.patch_size = patch_size
+        self.stride = stride
         self.sample_group_size = sample_group_size
         self.samples = []
-        self.block_size = self.window_size[0] * self.window_size[1] - 1
+        self.block_size = self.patch_size[0] * self.patch_size[1] - 1
 
         os.makedirs(self.outputDir, exist_ok=True)
 
@@ -79,12 +79,12 @@ def __getitem__(self, idx):
         #return flat
 
     def add(self, mapReader):
-        for i in range(0, mapReader.size[0] - self.window_size[0] + 1, self.step_size):
-            for j in range(0, mapReader.size[1] - self.window_size[1] + 1, self.step_size):
+        for i in range(0, mapReader.size[0] - self.patch_size[0] + 1, self.stride):
+            for j in range(0, mapReader.size[1] - self.patch_size[1] + 1, self.stride):
                 self.samples.append([[
                     (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1
-                    for y in range(self.window_size[1])]
-                    for x in range(self.window_size[0])])
+                    for y in range(self.patch_size[1])]
+                    for x in range(self.patch_size[0])])
 
     #Generate image patches and write to data/output directory
     def generate_patches(self, mapReader, image_groups=3, outDirectory=None):
@@ -98,12 +98,12 @@ def generate_patches(self, mapReader, image_groups=3, outDirectory=None):
         mapReader.standardize(converter=self.converter)
 
         if outDirectory is None:
-            outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.window_size[0]}x{self.window_size[1]}", f"stride-{self.step_size}")
+            outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.patch_size[0]}x{self.patch_size[1]}", f"stride-{self.stride}")
         os.makedirs(outDirectory, exist_ok=True)
 
         img_group_number = 0
-        for i in range(0, mapReader.size[0] - self.window_size[0] + 1, self.step_size):
-            for j in range(0, mapReader.size[1] - self.window_size[1] + 1, self.step_size):
+        for i in range(0, mapReader.size[0] - self.patch_size[0] + 1, self.stride):
+            for j in range(0, mapReader.size[1] - self.patch_size[1] + 1, self.stride):
 
                 sample = self.extractSample(mapReader, topLeft=(i, j))
 
@@ -125,16 +125,16 @@ def extractSample(self, mapReader, topLeft):
         # sample = [
         #             [
         #                     (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 # TODO this conversion should be done once in the original data instead of patches.
-        #                 for y in range(self.window_size[1])
+        #                 for y in range(self.patch_size[1])
         #             ]
-        #             for x in range(self.window_size[0])
+        #             for x in range(self.patch_size[0])
         #         ]
         sample = [
                     [
                             mapReader.data[i + x][j + y]
-                        for y in range(self.window_size[1])
+                        for y in range(self.patch_size[1])
                     ]
-                    for x in range(self.window_size[0])
+                    for x in range(self.patch_size[0])
                 ]
         return np.asarray(sample)
 
@@ -143,8 +143,8 @@ def shuffle(self):
         random.shuffle(self.samples)
 
     def get_train_test(self, train_ratio):
-        train_dataset = MapsDataset(self.window_size, self.step_size, self.converter)
-        test_dataset = MapsDataset(self.window_size, self.step_size, self.converter)
+        train_dataset = MapsDataset(self.patch_size, self.stride, self.converter)
+        test_dataset = MapsDataset(self.patch_size, self.stride, self.converter)
         train_size = int(len(self.samples) * train_ratio)
         train_dataset.samples = self.samples[:train_size]
         test_dataset.samples = self.samples[train_size:]

src/MapDatasetGenerator/read.py src/mapdataset/read.pysrc/MapDatasetGenerator/read.py renamed to src/mapdataset/read.py

@@ -11,7 +11,7 @@
 handler.setFormatter(formatter)
 root.addHandler(handler)
 
-import maps
+from lib import maps
 
 sfMap = maps.MapReader('../data/input/sf_layered.txt', "SF_Layered")
 mapsDataset = maps.MapsDataset((32, 32), 10, 1280, maps.single_layer_converter) #Third parameter is the group size

src/mapdataset/run.py

@@ -1,10 +1,32 @@
 # Script to read dill data objects as numpy arrays.
-import dill
-import numpy as np
 from PIL import Image
 import os
+import sys
+import logging
 
-from mapdataset import ImageGroupReader
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+root.addHandler(handler)
+
+from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader
+
+
+
+sfMap = MapReader('./data/input/sf_layered.txt', "SF_Layered")
+mapsDataset = MapsDataset(
+    patch_size=(32, 32), 
+    stride=10, 
+    sample_group_size=1280, 
+    converter=single_layer_converter,
+    outputDir="./data/output"
+    ) 
+    
+mapsDataset.generate_patches(sfMap) #This will generate dill files which contain the saved sample lists.
 
 
 dillFolder = "./data/output/SF_Layered/32x32/stride-10"