new reader

Author: adhocmaster

ChangeLog.txt

@@ -0,0 +1,4 @@
+Oct 14, 2022
+1. generate_patches has a new parameter for output directory
+2. default output directory now has a meaningful structure with patch sizes and step sizes
+3. Refactored ImageGroupReader

src/__init__.py

@@ -0,0 +1,36 @@
+import logging
+import dill
+import numpy as np
+import os
+import logging
+
+class ImageGroupReader:
+    def __init__(self, directory_path, location='SF'):
+        self.data = []
+        self.dir_path = directory_path
+        self.location = location
+
+    def getGroupPath(self, groupNo):
+        return os.path.join(self.dir_path, f"{groupNo}.dill")
+
+    def load_group(self, groupNo=0):
+        with open(self.getGroupPath(groupNo), 'rb') as d:
+            return dill.load(d)
+
+           
+    def groupAsImg(self, group):
+        """Assumes the raw data is standardized. So, we project it back into PIL RGB image format
+
+        Returns:
+            _type_: _description_
+        """
+        # print(data[0])
+        data = np.asarray(group)
+        data = np.clip((data + 1) / 2, 0, 1) * 255
+        data = data.astype(np.uint8)
+        return data
+    
+    def asImg(self, patch):
+        data = np.clip((patch + 1) / 2, 0, 1) * 255
+        data = data.astype(np.uint8)
+        return data

src/lib/ImageGroupReader.py

@@ -0,0 +1 @@
+from .ImageGroupReader import ImageGroupReader

src/lib/__init__.py

@@ -94,7 +94,7 @@ def generate_patches(self, mapReader, image_groups=3, outDirectory=None):
         mapReader.standardize(converter=self.converter)
 
         if outDirectory is None:
-            outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.window_size[0]}x{self.window_size[1]}", f"{self.step_size}x{self.step_size}")
+            outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.window_size[0]}x{self.window_size[1]}", f"stride-{self.step_size}")
         os.makedirs(outDirectory, exist_ok=True)
 
         img_group_number = 0
@@ -118,21 +118,21 @@ def generate_patches(self, mapReader, image_groups=3, outDirectory=None):
     def extractSample(self, mapReader, topLeft):
         i = topLeft[0]
         j = topLeft[1]
-        # sample = [
-        #             [
-        #                     (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 # TODO this conversion should be done once in the original data instead of patches.
-        #                 for y in range(self.window_size[1])
-        #             ]
-        #             for x in range(self.window_size[0])
-        #         ]
         sample = [
                     [
-                            mapReader.data[i + x][j + y]
+                            (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 # TODO this conversion should be done once in the original data instead of patches.
                         for y in range(self.window_size[1])
                     ]
                     for x in range(self.window_size[0])
                 ]
-        return sample
+        # sample = [
+        #             [
+        #                     mapReader.data[i + x][j + y]
+        #                 for y in range(self.window_size[1])
+        #             ]
+        #             for x in range(self.window_size[0])
+        #         ]
+        return np.asarray(sample)
 
 
     def shuffle(self):

src/maps.py

@@ -2,41 +2,25 @@
 import dill
 import numpy as np
 from PIL import Image
-import torch
-
-
-class ImgGroupReader:
-    def __init__(self, directory_path, location='SF'):
-        self.data = []
-        self.dir_path = directory_path
-        self.location = location
-
-    def load_group(self, group=0):
-        if self.dir_path[-1]=='/':
-            with open(self.dir_path+str(group)+".dill", 'rb') as d:
-                self.data = dill.load(d)
-                d.close()
-        else:
-            self.dir_path = self.dir_path + "/"
-            with open(self.dir_path+str(group)+".dill", 'rb') as d:
-                self.data = dill.load(d)
-                d.close()
-           
-    def read_numpy(self):
-        if(self.data == []): return print("Image group file not loaded! Call load_image_group() first.")
-        else:
-            self.data = np.array(self.data)
-            self.data = np.clip((self.data+1)/2, 0, 1)*255
-            self.data = self.data.astype(np.uint8)
-        return self.data
-
-sf_groups = 117
-for i in range(sf_groups):
-    reader = ImgGroupReader("../data/output/city_patches/dill/washington")
-    reader.load_group(group=i)
-    data = reader.read_numpy()
-    im = Image.fromarray(data[0])
-    im.save('../data/output/city_patches/png/washington/'+str(i)+'.png')
+import os
+
+from lib import ImageGroupReader
+
+dillFolder = "../data/output/SF_Layered/32x32/stride-10"
+nGroups = 0
+# Iterate directory
+for path in os.listdir(dillFolder):
+    # check if current path is a file
+    if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"):
+        nGroups += 1
+
+for i in range(nGroups):
+    reader = ImageGroupReader(dillFolder)
+    data = reader.load_group(groupNo=i)
+    patchImgArray = reader.asImg(data[0])
+    im = Image.fromarray(patchImgArray)
+    path = os.path.join(dillFolder, f"{i}.png")
+    im.save(path)