CS6910-Deep-Learning-Assignment-1/Question-4.py at main · sowmyamanojna/CS6910-Deep-Learning-Assignment-1 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
print("Importing Libraries ...")
####################################################################
import wandb
import numpy as np
import os
from activations import Sigmoid, Tanh, Relu, Softmax
from layers import Input, Dense
from optimizers import Normal, Momentum, Nesterov, AdaGrad, RMSProp, Adam, Nadam
from network import NeuralNetwork
from loss import CrossEntropy, SquaredError
from helper import OneHotEncoder, MinMaxScaler
from sklearn.model_selection import train_test_split

from keras.datasets import fashion_mnist
import matplotlib.pyplot as plt
print("Done!")
#################################################################### [markdown]
# # Loss on Training Data
####################################################################

print("Loading data ... ", end="")
[(x_train, y_train), (x_test, y_test)] = fashion_mnist.load_data()
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)
print("Done!")

print("Size of Training data:", x_train.shape)
print("Size of Validation data:", x_val.shape)

print("Performing Scaling and Encoding transformations on the data ... ", end="")
X_scaled = x_train/255
X_val_scaled = x_val/255
X_test_scaled = x_test/255

X_scaled = X_scaled.reshape(X_scaled.shape[0], X_scaled.shape[1]*X_scaled.shape[2]).T
X_val_scaled = X_val_scaled.reshape(X_val_scaled.shape[0], X_val_scaled.shape[1]*X_val_scaled.shape[2]).T
X_test_scaled = X_test_scaled.reshape(X_test_scaled.shape[0], X_test_scaled.shape[1]*X_test_scaled.shape[2]).T

encoder = OneHotEncoder()
t = encoder.fit_transform(y_train, 10)
t_val = encoder.fit_transform(y_val, 10)
t_test = encoder.fit_transform(y_test, 10)
print("Done!")

X_scaled = X_scaled[:, :21000]
X_test_scaled = X_test_scaled[:, :9000]
t = t[:, :21000]
t_test = t_test[:, :9000]
####################################################################
# # Preparing small dataset to test the code
# [(X_train, y_train), (X_test, y_test)] = fashion_mnist.load_data()
# scaler = MinMaxScaler()
# X_scaled = scaler.fit_transform(X_train)
# X_scaled = X_scaled.reshape(X_scaled.shape[0], X_scaled.shape[1]*X_scaled.shape[2]).T

# encoder = OneHotEncoder()
# t = encoder.fit_transform(y_train, 10)


####################################################################
sweep_config = {"name": "complete-sweep", "method": "grid"}
sweep_config["metric"] = {"name": "loss", "goal": "minimize"}
parameters_dict = {
                "num_epochs": {"values": [10, 50]}, \
                # "num_hidden_layers": {"values": [3, 4, 5]}, \
                "size_hidden_layer": {"values": [32, 64, 128]}, \
                # "learning_rate": {"values": [1e-3, 1e-4]}, \
                "optimizer": {"values": ["Normal","Momentum","AdaGrad","RMSProp","Adam","Nadam"]}, \
                "batch_size": {"values": [128, 1024, 60000]}, \
                "weight_init": {"values": ["RandomNormal", "XavierUniform"]} , \
                "activation": {"values": ["Sigmoid", "Tanh", "Relu"]}, \
                "loss": {"values": ["CrossEntropy", "SquaredError"]}, \
                  }
sweep_config["parameters"] = parameters_dict
# for i in sweep_config:
#     print(i, sweep_config[i])

####################################################################
def train_nn(config = sweep_config):
    with wandb.init(config = config):
        config = wandb.init().config
        wandb.run.name = "e_{}_hl_{}_opt_{}_bs_{}_init_{}_ac_{}_loss_{}".format(config.num_epochs,\
                                                                      config.size_hidden_layer,\
                                                                      config.optimizer,\
                                                                      config.batch_size,\
                                                                      config.weight_init,\
                                                                      config.activation,\
                                                                      config.loss)

        layers = [Input(data=X_scaled),\
                  Dense(size=config.size_hidden_layer, activation=config.activation, name="HL1"),\
                  Dense(size=10, activation=config.activation, name="OL")]

        nn_model = NeuralNetwork(layers=layers, batch_size=config.batch_size, \
                                 optimizer=config.optimizer, initialization=config.weight_init, \
                                 epochs=config.num_epochs, t=t, X_val=X_val_scaled, \
                                 t_val=t_val, loss=config.loss, use_wandb=True)#, \
                                #  optim_params={"eta":config.learning_rate})

        nn_model.forward_propogation()
        nn_model.backward_propogation()
        acc_val, loss_val, _ = nn_model.check_test(X_val_scaled, t_val)
        acc_test, loss_test, _ = nn_model.check_test(X_test_scaled, t_test)

        wandb.log({"val_loss_end": loss_val/t_val.shape[1], \
                   "val_acc_end": acc_val/t_val.shape[1], \
                   "test_loss_end": loss_test/t_test.shape[1], \
                   "test_acc_end": acc_test/t_test.shape[1], \
                   "epoch":config.num_epochs})

####################################################################
sweep_id = wandb.sweep(sweep_config, project = "trail-1")
wandb.agent(sweep_id, function = train_nn)
#################################################################### [markdown]