add sdae

Author: eelxpeng

.DS_Store

@@ -19,4 +19,4 @@ Note:
 
 * The pretrained weights is important to initialize the weights of VaDE.
 * Unlike the original code using combined training and test data for training and evaluation, I split the training and test data, and only use training data for training and test data for evaluation. I think it is a more appropriate way to evaluate the method for generalization.
-* As found, with above evaluation scheme and training for 3000 epochs, the clustering accuracy achieved is 93.65\%.
+* As found, with above evaluation scheme and training for 3000 epochs, the clustering accuracy achieved is 94\%.

README.md

@@ -1,36 +1,31 @@
+import sys
+sys.path.append("..")
 import torch
+import torch.utils.data
+from torchvision import datasets, transforms
 import numpy as np
+import argparse
 from udlp.autoencoder.denoisingAutoencoder import DenoisingAutoencoder
-from utils import readData
 
 if __name__ == "__main__":
-    # from lib.Tox21_Data import read
-    # x_tr_t, y_tr_t, x_valid_t, y_valid_t, x_te_t, y_te_t = read("./dataset/tox21/", target=0)
+    parser = argparse.ArgumentParser(description='VAE MNIST Example')
+    parser.add_argument('--lr', type=float, default=0.002, metavar='N',
+                        help='learning rate for training (default: 0.001)')
+    parser.add_argument('--batch-size', type=int, default=128, metavar='N',
+                        help='input batch size for training (default: 128)')
+    parser.add_argument('--epochs', type=int, default=10, metavar='N',
+                        help='number of epochs to train (default: 10)')
+    args = parser.parse_args()
+    
+    train_loader = torch.utils.data.DataLoader(
+        datasets.MNIST('../dataset/mnist', train=True, download=True,
+                       transform=transforms.ToTensor()),
+        batch_size=args.batch_size, shuffle=True, num_workers=2)
+    test_loader = torch.utils.data.DataLoader(
+        datasets.MNIST('../dataset/mnist', train=False, transform=transforms.ToTensor()),
+        batch_size=args.batch_size, shuffle=False, num_workers=2)
 
-    label_name = ['World', 'Sports', 'Business', 'Sci/Tech']
-    training_num, valid_num, test_num, vocab_size = 110000, 10000, 7600, 10000
-    training_file = 'dataset/agnews_training_110K_10K-TFIDF-words.txt'
-    valid_file = 'dataset/agnews_valid_10K_10K-TFIDF-words.txt'
-    test_file = 'dataset/agnews_test_7600_10K-TFIDF-words.txt'
-
-    randgen = np.random.RandomState(13)
-    trainX, trainY = readData(training_file, training_num, vocab_size, randgen)
-    validX, validY = readData(valid_file, valid_num, vocab_size)
-    testX, testY = readData(test_file, test_num, vocab_size)
-
-    # preprocess, normalize each dimension to be [0, 1] for cross-entropy loss
-    train_max = torch.max(trainX, dim=0, keepdim=True)[0]
-    valid_max = torch.max(validX, dim=0, keepdim=True)[0]
-    test_max = torch.max(testX, dim=0, keepdim=True)[0]
-    print(train_max.size())
-    print(valid_max.size())
-    print(test_max.size())
-    x_max = torch.max(torch.cat((train_max, valid_max, test_max), 0), dim=0, keepdim=True)[0]
-    trainX.div_(x_max)
-    validX.div_(x_max)
-    testX.div_(x_max)
-
-    in_features = trainX.size()[1]
+    in_features = 784
     out_features = 500
     dae = DenoisingAutoencoder(in_features, out_features)
-    dae.fit(trainX, validX, lr=1e-3, num_epochs=10, loss_type="cross-entropy")
+    dae.fit(train_loader, test_loader, lr=args.lr, num_epochs=args.epochs, loss_type="cross-entropy")

dataset/.DS_Store

@@ -0,0 +1,37 @@
+import sys
+sys.path.append("..")
+import torch
+import torch.utils.data
+from torchvision import datasets, transforms
+import numpy as np
+import argparse
+from udlp.autoencoder.stackedDAE import StackedDAE
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='VAE MNIST Example')
+    parser.add_argument('--lr', type=float, default=0.002, metavar='N',
+                        help='learning rate for training (default: 0.001)')
+    parser.add_argument('--batch-size', type=int, default=128, metavar='N',
+                        help='input batch size for training (default: 128)')
+    parser.add_argument('--pretrainepochs', type=int, default=10, metavar='N',
+                        help='number of epochs to train (default: 10)')
+    parser.add_argument('--epochs', type=int, default=10, metavar='N',
+                        help='number of epochs to train (default: 10)')
+    args = parser.parse_args()
+    
+    train_loader = torch.utils.data.DataLoader(
+        datasets.MNIST('../dataset/mnist', train=True, download=True,
+                       transform=transforms.ToTensor()),
+        batch_size=args.batch_size, shuffle=True, num_workers=2)
+    test_loader = torch.utils.data.DataLoader(
+        datasets.MNIST('../dataset/mnist', train=False, transform=transforms.ToTensor()),
+        batch_size=args.batch_size, shuffle=False, num_workers=2)
+
+    in_features = 784
+    out_features = 500
+    sdae = StackedDAE(input_dim=784, z_dim=10, binary=True,
+        encodeLayer=[500,500,2000], decodeLayer=[2000,500,500], activation="relu", 
+        dropout=0)
+    sdae.pretrain(train_loader, test_loader, lr=args.lr, batch_size=args.batch_size, 
+        num_epochs=args.pretrainepochs, corrupt=0.3, loss_type="cross-entropy")
+    sdae.fit(train_loader, test_loader, lr=args.lr, num_epochs=args.epochs, corrupt=0.3, loss_type="cross-entropy")

test/test_dae.py

@@ -13,25 +13,31 @@
 from udlp.ops import MSELoss, BCELoss
 
 class DenoisingAutoencoder(nn.Module):
-    def __init__(self, in_features, out_features, activation="relu"):
+    def __init__(self, in_features, out_features, activation="relu", 
+        dropout=0.2, tied=False):
         super(self.__class__, self).__init__()
         self.weight = Parameter(torch.Tensor(out_features, in_features))
+        if tied:
+            self.deweight = self.weight.t()
+        else:
+            self.deweight = Parameter(torch.Tensor(in_features, out_features))
         self.bias = Parameter(torch.Tensor(out_features))
         self.vbias = Parameter(torch.Tensor(in_features))
 
         if activation=="relu":
             self.enc_act_func = nn.ReLU()
         elif activation=="sigmoid":
             self.enc_act_func = nn.Sigmoid()
-        self.dropout = nn.Dropout(p=0.2)
+        self.dropout = nn.Dropout(p=dropout)
 
         self.reset_parameters()
 
     def reset_parameters(self):
         stdv = 1. / math.sqrt(self.weight.size(1))
         self.weight.data.uniform_(-stdv, stdv)
         self.bias.data.uniform_(-stdv, stdv)
-        stdv = 1. / math.sqrt(self.vbias.size(0))
+        stdv = 1. / math.sqrt(self.deweight.size(1))
+        self.deweight.data.uniform_(-stdv, stdv)
         self.vbias.data.uniform_(-stdv, stdv)
 
     def forward(self, x):
@@ -44,13 +50,26 @@ def encode(self, x, train=True):
             self.dropout.eval()
         return self.dropout(self.enc_act_func(F.linear(x, self.weight, self.bias)))
 
+    def encodeBatch(self, dataloader):
+        encoded = []
+        for batch_idx, (inputs, _) in enumerate(dataloader):
+            inputs = inputs.view(inputs.size(0), -1).float()
+            if use_cuda:
+                inputs = inputs.cuda()
+            inputs = Variable(inputs)
+            hidden = self.encode(inputs, train=False)
+            encoded.append(hidden.data.cpu())
+
+        encoded = torch.cat(encoded, dim=0)
+        return encoded
+
     def decode(self, x, binary=False):
         if not binary:
-            return F.linear(x, self.weight.t(), self.vbias)
+            return F.linear(x, self.deweight, self.vbias)
         else:
-            return F.sigmoid(F.linear(x, self.weight.t(), self.vbias))
+            return F.sigmoid(F.linear(x, self.deweight, self.vbias))
 
-    def fit(self, data_x, valid_x, lr=0.001, batch_size=128, num_epochs=10, corrupt=0.5,
+    def fit(self, trainloader, validloader, lr=0.001, batch_size=128, num_epochs=10, corrupt=0.3,
         loss_type="mse"):
         """
         data_x: FloatTensor
@@ -60,17 +79,11 @@ def fit(self, data_x, valid_x, lr=0.001, batch_size=128, num_epochs=10, corrupt=
         if use_cuda:
             self.cuda()
         print("=====Denoising Autoencoding layer=======")
-        optimizer = optim.Adam(filter(lambda p: p.requires_grad, self.parameters()), lr=lr, betas=(0.9, 0.9))
+        optimizer = optim.Adam(filter(lambda p: p.requires_grad, self.parameters()), lr=lr)
         if loss_type=="mse":
             criterion = MSELoss()
         elif loss_type=="cross-entropy":
             criterion = BCELoss()
-        trainset = Dataset(data_x, data_x)
-        trainloader = torch.utils.data.DataLoader(
-            trainset, batch_size=batch_size, shuffle=True, num_workers=2)
-        validset = Dataset(valid_x, valid_x)
-        validloader = torch.utils.data.DataLoader(
-            validset, batch_size=1000, shuffle=False, num_workers=2)
 
         # validate
         total_loss = 0.0
@@ -87,14 +100,15 @@ def fit(self, data_x, valid_x, lr=0.001, batch_size=128, num_epochs=10, corrupt=
                 outputs = self.decode(hidden)
 
             valid_recon_loss = criterion(outputs, inputs)
-            total_loss += valid_recon_loss.data[0] * inputs.size()[0]
+            total_loss += valid_recon_loss.data[0] * len(inputs)
             total_num += inputs.size()[0]
 
         valid_loss = total_loss / total_num
         print("#Epoch 0: Valid Reconstruct Loss: %.3f" % (valid_loss))
 
         for epoch in range(num_epochs):
             # train 1 epoch
+            train_loss = 0.0
             for batch_idx, (inputs, _) in enumerate(trainloader):
                 inputs = inputs.view(inputs.size(0), -1).float()
                 inputs_corr = masking_noise(inputs, corrupt)
@@ -111,12 +125,12 @@ def fit(self, data_x, valid_x, lr=0.001, batch_size=128, num_epochs=10, corrupt=
                 else:
                     outputs = self.decode(hidden)
                 recon_loss = criterion(outputs, inputs)
+                train_loss += recon_loss.data[0]*len(inputs)
                 recon_loss.backward()
                 optimizer.step()
 
             # validate
-            total_loss = 0.0
-            total_num = 0
+            valid_loss = 0.0
             for batch_idx, (inputs, _) in enumerate(validloader):
                 inputs = inputs.view(inputs.size(0), -1).float()
                 if use_cuda:
@@ -129,10 +143,8 @@ def fit(self, data_x, valid_x, lr=0.001, batch_size=128, num_epochs=10, corrupt=
                     outputs = self.decode(hidden)
 
                 valid_recon_loss = criterion(outputs, inputs)
-                total_loss += valid_recon_loss.data[0] * inputs.size()[0]
-                total_num += inputs.size()[0]
+                valid_loss += valid_recon_loss.data[0] * len(inputs)
 
-            valid_loss = total_loss / total_num
             print("#Epoch %3d: Reconstruct Loss: %.3f, Valid Reconstruct Loss: %.3f" % (
-                epoch+1, recon_loss.data[0], valid_loss))
+                epoch+1, train_loss / len(trainloader.dataset), valid_loss / len(validloader.dataset)))