Cangjie

.gitignore

@@ -1,4 +1,5 @@
 **/__pycache__
 data
-checkpoint
-runs
+checkpoint*
+runs
+net_ver

cangjie_dataset.py

@@ -0,0 +1,91 @@
+import os
+import numpy as np
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms
+
+class ETL952Dataset(Dataset):
+    def __init__(self, root_dir, folder_name, transform=None):
+        self.root_dir = os.path.join(root_dir, 'data', 'etl_952_singlechar_size_64', 'etl_952_singlechar_size_64', folder_name)
+
+        if transform is None:
+            self.transform = transforms.Compose([
+                transforms.Resize((32, 32)),  # Resize to match SqueezeNet input size
+                transforms.ToTensor(),
+                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+            ])
+        else:
+            self.transform = transform
+
+        self.classes = [str(i) for i in range(952)]
+        self.data = []  # Will store image data
+        self.labels = []  # Will store labels
+
+        # Load all images into memory
+        for class_idx, class_name in enumerate(self.classes):
+            if class_idx%10==0:
+                print(f"start loading class {class_name}/952 from {folder_name}")
+            class_path = os.path.join(self.root_dir, class_name)
+            for img_name in os.listdir(class_path):
+                if img_name.lower().endswith(('.png', )):
+                    img_path = os.path.join(class_path, img_name)
+                    image = Image.open(img_path).convert('RGB')
+                    image = image.resize((32, 32))  # Resize to 32x32
+                    image_array = np.array(image)
+                    self.data.append(image_array)
+                    self.labels.append(class_idx)
+
+        # Convert to numpy arrays
+        self.data = np.array(self.data)
+        self.labels = np.array(self.labels)
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, idx):
+        image = self.data[idx]
+        label = self.labels[idx]
+
+        # Convert numpy array to PIL Image
+        image = Image.fromarray(image)
+
+        if self.transform:
+            image = self.transform(image)
+
+        return image, label
+
+class ETL952Train(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Train, self).__init__(root_dir, folder_name="952_train", transform=transform)
+class ETL952Test(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Test, self).__init__(root_dir, folder_name="952_test", transform=transform)
+class ETL952Val(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Val, self).__init__(root_dir, folder_name="952_val", transform=transform)
+
+import pandas as pd
+class ETL952Labels():
+
+    def __init__(self,path=os.path.join("data","etl_952_singlechar_size_64","etl_952_singlechar_size_64","952_labels.txt")):
+        self.path=path
+        self.data=pd.read_csv(path,sep=" ",header=0,names=['label', 'character', 'JISx0208', 'UTF8', 'Cangjie'])
+        self.data=self.data.to_numpy()
+        # for i in range(len(self.data)):
+        #     self.data[i][4]=(self.data[i][4]+"      ")[:5]
+
+
+# from torch.utils.data import DataLoader
+# # train_set = ETL952Train(root_dir="pytorch-cifar100", transform=transforms.ToTensor())
+
+# train_set = ETL952Train(root_dir="", transform=transforms.ToTensor())
+# train_loader = DataLoader(train_set, batch_size=128, shuffle=True,num_workers=4)
+# train_loader = DataLoader(train_set, batch_size=64, shuffle=True)
+
+# print("total number of train data: ", len(train_set))
+
+# for images, labels in train_loader:
+#     print("images shape: ", images.shape)
+#     print("labels shape: ", labels.shape)
+#     break

cangjie_dataset_b3.py

@@ -0,0 +1,91 @@
+import os
+import numpy as np
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms
+import pandas as pd
+
+class ETL952Dataset(Dataset):
+    def __init__(self, root_dir, folder_name, transform=None):
+        self.root_dir = os.path.join(root_dir, 'data', 'etl_952_singlechar_size_64', 'etl_952_singlechar_size_64', folder_name)
+        self.dict_file_path = os.path.join(root_dir, 'data', 'etl_952_singlechar_size_64', 'etl_952_singlechar_size_64',"952_labels.txt")
+        if transform is None:
+            self.transform = transforms.Compose([
+                transforms.Resize((32, 32)),  # Resize to match SqueezeNet input size
+                transforms.ToTensor(),
+                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+            ])
+        else:
+            self.transform = transform
+
+        self.classes = [str(i) for i in range(952)]
+        self.data = []  # Will store image data
+        self.labels = []  # Will store labels
+        self.string_labels = []  # Will store string labels
+
+        self.string_label_data = pd.read_csv(self.dict_file_path, sep=" ", header=0, names=['label', 'character', 'JISx0208', 'UTF8', 'Cangjie']).to_numpy()
+
+
+        # Load all images into memory
+        for class_idx, class_name in enumerate(self.classes):
+            if class_idx%10==0:
+                print(f"start loading class {class_name}/952 from {folder_name}")
+            class_path = os.path.join(self.root_dir, class_name)
+            for img_name in os.listdir(class_path):
+                if img_name.lower().endswith(('.png', )):
+                    img_path = os.path.join(class_path, img_name)
+                    image = Image.open(img_path).convert('RGB')
+                    image = image.resize((32, 32))  # Resize to 32x32
+                    image_array = np.array(image)
+                    self.data.append(image_array)
+                    self.labels.append(class_idx)
+                    self.string_labels.append(self.string_label_data[class_idx][4])
+
+        # Convert to numpy arrays
+        self.data = np.array(self.data)
+        self.labels = np.array(self.labels)
+        self.string_labels = np.array(self.string_labels)
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, idx):
+        image = self.data[idx]
+        label = self.labels[idx]
+        string_label = self.string_labels[idx]
+
+        # Convert numpy array to PIL Image
+        image = Image.fromarray(image)
+
+        if self.transform:
+            image = self.transform(image)
+
+        return image, label, string_label
+
+class ETL952Train(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Train, self).__init__(root_dir, folder_name="952_train", transform=transform)
+class ETL952Test(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Test, self).__init__(root_dir, folder_name="952_test", transform=transform)
+class ETL952Val(ETL952Dataset):
+    def __init__(self, root_dir, transform=None):
+        super(ETL952Val, self).__init__(root_dir, folder_name="952_val", transform=transform)
+
+
+
+
+# from torch.utils.data import DataLoader
+# # train_set = ETL952Train(root_dir="pytorch-cifar100", transform=transforms.ToTensor())
+
+# train_set = ETL952Train(root_dir="", transform=transforms.ToTensor())
+# train_loader = DataLoader(train_set, batch_size=128, shuffle=True,num_workers=4)
+# train_loader = DataLoader(train_set, batch_size=64, shuffle=True)
+
+# print("total number of train data: ", len(train_set))
+
+# for images, labels in train_loader:
+#     print("images shape: ", images.shape)
+#     print("labels shape: ", labels.shape)
+#     break

cangjie_models/sqnetC3579.py

@@ -0,0 +1,97 @@
+"""squeezenet in pytorch
+
+
+
+[1] Song Han, Jeff Pool, John Tran, William J. Dally
+
+    squeezenet: Learning both Weights and Connections for Efficient Neural Networks
+    https://arxiv.org/abs/1506.02626
+"""
+
+import torch
+import torch.nn as nn
+
+
+class Fire(nn.Module):
+
+    def __init__(self, in_channel, out_channel, squzee_channel):
+
+        super().__init__()
+        self.squeeze = nn.Sequential(
+            nn.Conv2d(in_channel, squzee_channel, 1),
+            nn.BatchNorm2d(squzee_channel),
+            nn.ReLU(inplace=True)
+        )
+
+        self.expand_1x1 = nn.Sequential(
+            nn.Conv2d(squzee_channel, int(out_channel / 2), 1),
+            nn.BatchNorm2d(int(out_channel / 2)),
+            nn.ReLU(inplace=True)
+        )
+
+        self.expand_3x3 = nn.Sequential(
+            nn.Conv2d(squzee_channel, int(out_channel / 2), 3, padding=1),
+            nn.BatchNorm2d(int(out_channel / 2)),
+            nn.ReLU(inplace=True)
+        )
+
+    def forward(self, x):
+
+        x = self.squeeze(x)
+        x = torch.cat([
+            self.expand_1x1(x),
+            self.expand_3x3(x)
+        ], 1)
+
+        return x
+
+class SqNetC3579(nn.Module):
+
+    """mobile net with simple bypass"""
+    def __init__(self, class_num=952):
+
+        super().__init__()
+        self.stem = nn.Sequential(
+            # nn.MaxPool2d(2, 2),
+            nn.Conv2d(3, 96, 3, padding=1),
+            nn.BatchNorm2d(96),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2)
+        )
+
+        self.fire2 = Fire(96, 128, 16)
+        # self.fire3 = Fire(128, 128, 16)
+        self.fire4 = Fire(128, 256, 32)
+        # self.fire5 = Fire(256, 256, 32)
+        self.fire6 = Fire(256, 384, 48)
+        # self.fire7 = Fire(384, 384, 48)
+        self.fire8 = Fire(384, 512, 64)
+        # self.fire9 = Fire(512, 512, 64)
+        self.conv10 = nn.Conv2d(512, class_num, 1)
+        self.avg = nn.AdaptiveAvgPool2d(1)
+        self.maxpool = nn.MaxPool2d(2, 2)
+
+    def forward(self, x):
+        x = self.stem(x)       
+
+        f2 = self.fire2(x)
+        # f3 = self.fire3(f2) + f2
+        f4 = self.fire4(f2)
+        f4 = self.maxpool(f4)
+
+        # f5 = self.fire5(f4) + f4
+        f6 = self.fire6(f4)
+        # f7 = self.fire7(f6) + f6
+        f8 = self.fire8(f6)
+        f8 = self.maxpool(f8)
+
+        # f9 = self.fire9(f8)
+        c10 = self.conv10(f8)
+
+        x = self.avg(c10)
+        x = x.view(x.size(0), -1)
+
+        return x
+
+def sqnetc3579(class_num=952):
+    return SqNetC3579(class_num=class_num)

cangjie_models/sqnetC9.py

@@ -0,0 +1,97 @@
+"""squeezenet in pytorch
+
+
+
+[1] Song Han, Jeff Pool, John Tran, William J. Dally
+
+    squeezenet: Learning both Weights and Connections for Efficient Neural Networks
+    https://arxiv.org/abs/1506.02626
+"""
+
+import torch
+import torch.nn as nn
+
+
+class Fire(nn.Module):
+
+    def __init__(self, in_channel, out_channel, squzee_channel):
+
+        super().__init__()
+        self.squeeze = nn.Sequential(
+            nn.Conv2d(in_channel, squzee_channel, 1),
+            nn.BatchNorm2d(squzee_channel),
+            nn.ReLU(inplace=True)
+        )
+
+        self.expand_1x1 = nn.Sequential(
+            nn.Conv2d(squzee_channel, int(out_channel / 2), 1),
+            nn.BatchNorm2d(int(out_channel / 2)),
+            nn.ReLU(inplace=True)
+        )
+
+        self.expand_3x3 = nn.Sequential(
+            nn.Conv2d(squzee_channel, int(out_channel / 2), 3, padding=1),
+            nn.BatchNorm2d(int(out_channel / 2)),
+            nn.ReLU(inplace=True)
+        )
+
+    def forward(self, x):
+
+        x = self.squeeze(x)
+        x = torch.cat([
+            self.expand_1x1(x),
+            self.expand_3x3(x)
+        ], 1)
+
+        return x
+
+class SqNetC9(nn.Module):
+
+    """mobile net with simple bypass"""
+    def __init__(self, class_num=952):
+
+        super().__init__()
+        self.stem = nn.Sequential(
+            # nn.MaxPool2d(2, 2),
+            nn.Conv2d(3, 96, 3, padding=1),
+            nn.BatchNorm2d(96),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2)
+        )
+
+        self.fire2 = Fire(96, 128, 16)
+        self.fire3 = Fire(128, 128, 16)
+        self.fire4 = Fire(128, 256, 32)
+        self.fire5 = Fire(256, 256, 32)
+        self.fire6 = Fire(256, 384, 48)
+        self.fire7 = Fire(384, 384, 48)
+        self.fire8 = Fire(384, 512, 64)
+        # self.fire9 = Fire(512, 512, 64)
+        self.conv10 = nn.Conv2d(512, class_num, 1)
+        self.avg = nn.AdaptiveAvgPool2d(1)
+        self.maxpool = nn.MaxPool2d(2, 2)
+
+    def forward(self, x):
+        x = self.stem(x)       
+
+        f2 = self.fire2(x)
+        f3 = self.fire3(f2) + f2
+        f4 = self.fire4(f3)
+        f4 = self.maxpool(f4)
+
+        f5 = self.fire5(f4) + f4
+        f6 = self.fire6(f5)
+        f7 = self.fire7(f6) + f6
+        f8 = self.fire8(f7)
+        f8 = self.maxpool(f8)
+
+        # f9 = self.fire9(f8)
+        c10 = self.conv10(f8)
+
+        x = self.avg(c10)
+        x = x.view(x.size(0), -1)
+
+        return x
+
+def sqnetc9(class_num=952):
+    return SqNetC9(class_num=class_num)