Store tiles/patches in data/output and read the same as numpy

Author: ishaan95

.gitignore

@@ -0,0 +1,2 @@
+data/output/*
+__pycache__/*

maps.py

@@ -6,6 +6,8 @@
 import torch
 import numpy as np
 import random
+import dill
+
 
 # Credit - Bahar
 class MapReader:
@@ -36,11 +38,12 @@ def get_char(self, layer):
 # Modifications - Ishaan
 
 class MapsDataset(Dataset):
-    def __init__(self, window_size, step_size, converter):
+    def __init__(self, window_size, step_size, sample_group_size, converter):
         self.char_size = converter.char_size
         self.converter = converter
         self.window_size = window_size
         self.step_size = step_size
+        self.sample_group_size = sample_group_size
         self.samples = []
         self.block_size = self.window_size[0] * self.window_size[1] - 1
 
@@ -62,6 +65,25 @@ def add(self, mapReader):
                     for y in range(self.window_size[1])]
                     for x in range(self.window_size[0])])
 
+    #Generate image patches and write to data/output directory
+    def generate_patches(self, mapReader, image_groups=3):
+        img_group_number = 0
+        for i in range(0, mapReader.size[0] - self.window_size[0] + 1, self.step_size):
+            for j in range(0, mapReader.size[1] - self.window_size[1] + 1, self.step_size):
+                self.samples.append([[
+                (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1
+                for y in range(self.window_size[1])]
+                for x in range(self.window_size[0])])
+
+                if len(self.samples)==self.sample_group_size:
+                    with open("data/output/SF_group"+str(img_group_number)+".dill", 'wb+') as f:
+                        dill.dump(self.samples, f)
+                        f.close()    
+                    print("Image group {} saved in data/output/ !".format(img_group_number))
+                    self.samples.clear()
+                    img_group_number+=1
+        
+
     def shuffle(self):
         random.shuffle(self.samples)
 

read.py

@@ -0,0 +1,39 @@
+# Script to read dill data objects as numpy arrays.
+import dill
+import numpy as np
+from PIL import Image
+import torch
+
+
+class ImgGroupReader:
+    def __init__(self, directory_path, location='SF'):
+        self.data = []
+        self.dir_path = directory_path
+        self.location = location
+
+    def load_group(self, group=0):
+        if self.dir_path[-1]=='/':
+            with open(self.dir_path+self.location+"_group"+str(group)+".dill", 'rb') as d:
+                self.data = dill.load(d)
+                d.close()
+        else:
+            self.dir_path = self.dir_path + "/"
+            with open(self.dir_path+self.location+"_group"+str(group)+".dill", 'rb') as d:
+                self.data = dill.load(d)
+                d.close()
+           
+    def read_numpy(self):
+        if(self.data == []): return print("Image group file not loaded! Call load_image_group() first.")
+        else:
+            self.data = np.array(self.data)
+            self.data = np.clip((self.data+1)/2, 0, 1)*255
+            self.data = self.data.astype(np.uint8)
+        return self.data
+
+reader = ImgGroupReader("data/output/")
+reader.load_group()
+data = reader.read_numpy()
+im = Image.fromarray(data[0])
+im.show()
+
+

requirements.txt

@@ -1,4 +1,4 @@
-matplotlib==3.5.3
+dill==0.3.5.1
 numpy==1.23.3
 Pillow==9.2.0
 torch==1.12.1

run.py

@@ -0,0 +1,6 @@
+# Run this script to generate data in /output directory.
+import maps
+
+sfMap = maps.MapReader('data/input/sf_layered.txt')
+mapsDataset = maps.MapsDataset((32, 32), 2, 1280, maps.single_layer_converter) #Third parameter is the group size
+mapsDataset.generate_patches(sfMap) #This will generate dill files which contain the saved sample lists.