clpsych2019/GloveModels.py at master · ddenz/clpsych2019 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
from keras.models import Sequential
from keras.layers import Conv1D, Dense, Dropout, Embedding, Flatten, MaxPooling1D, SimpleRNN, Bidirectional, GRU, LSTM
from keras.optimizers import Adam
from utils import prepare_sequential, MAX_LENGTH


class GloveCNN(Sequential):
    def __init__(self, embedding_matrix):
        super().__init__()
        self.emb_matrix = embedding_matrix

    def build_model_(self, lr=0.001, optimizer='adam', loss='categorical_crossentropy'):
        self.add(Embedding(input_dim=self.emb_matrix.shape[0], output_dim=self.emb_matrix[0].shape[0],
                           input_length=MAX_LENGTH, weights=[self.emb_matrix], trainable=False))
        self.add(Conv1D(64, 7, activation='relu', padding='same'))
        self.add(MaxPooling1D())
        self.add(Conv1D(128, 5, activation='relu', padding='same'))
        self.add(MaxPooling1D())
        self.add(Conv1D(256, 3, activation='relu', padding='same'))
        self.add(MaxPooling1D())
        self.add(Flatten())
        self.add(Dense(64, activation='relu'))
        self.add(Dropout(0.5))
        self.add(Dense(4, activation='softmax'))
        self.compile(loss=loss, optimizer=optimizer, lr=lr, metrics=['accuracy'])
        self.summary()

    def build_model(self, optimizer=Adam(lr=0.001), loss='categorical_crossentropy'):
        self.add(Embedding(input_dim=self.emb_matrix.shape[0], output_dim=self.emb_matrix[0].shape[0],
                           input_length=MAX_LENGTH, weights=[self.emb_matrix], trainable=False))
        self.add(Conv1D(2, 300, activation='relu', padding='same'))
        self.add(MaxPooling1D())
        self.add(Flatten())
        self.add(Dropout(0.5))
        self.add(Dense(4, activation='softmax'))
        self.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
        self.summary()


class GloveBiRNN(Sequential):
    def __init__(self, embedding_matrix):
        super().__init__()
        self.emb_matrix = embedding_matrix

    def build_model(self, optimizer=Adam(lr=0.001), loss='categorical_crossentropy'):
        self.add(Embedding(input_dim=self.emb_matrix.shape[0], output_dim=self.emb_matrix[0].shape[0],
                           input_length=MAX_LENGTH, weights=[self.emb_matrix], trainable=False))
        self.add(Bidirectional(SimpleRNN(64, return_sequences=True)))
        self.add(Dropout(0.5))
        self.add(Bidirectional(SimpleRNN(64)))
        self.add(Dropout(0.5))
        self.add(Dense(4, activation='softmax'))
        self.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
        self.summary()


class GloveGRU(Sequential):
    def __init__(self, embedding_matrix):
        super().__init__()
        self.emb_matrix = embedding_matrix

    def build_model(self, optimizer=Adam(lr=0.001), loss='categorical_crossentropy'):
        self.add(Embedding(input_dim=self.emb_matrix.shape[0], output_dim=self.emb_matrix[0].shape[0],
                           input_length=MAX_LENGTH, weights=[self.emb_matrix], trainable=False))
        self.add(GRU(64, return_sequences=True))
        self.add(Dropout(0.5))
        self.add(GRU(64))
        self.add(Dropout(0.5))
        self.add(Dense(4, activation='softmax'))
        self.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
        self.summary()


class GloveBiLSTM(Sequential):
    def __init__(self, embedding_matrix):
        super().__init__()
        self.emb_matrix = embedding_matrix

    def build_model(self, optimizer=Adam(lr=0.001), loss='categorical_crossentropy'):
        self.add(Embedding(input_dim=self.emb_matrix.shape[0], output_dim=self.emb_matrix[0].shape[0],
                           input_length=MAX_LENGTH, weights=[self.emb_matrix], trainable=False))
        self.add(Bidirectional(LSTM(32, activation='sigmoid', recurrent_dropout=0.2, recurrent_activation='sigmoid',
                                    return_sequences=True)))
        self.add(Dropout(0.5))
        self.add(Bidirectional(LSTM(32, activation='sigmoid', recurrent_dropout=0.2, recurrent_activation='sigmoid',
                                    return_sequences=False)))
        self.add(Dropout(0.5))
        self.add(Dense(4, activation='softmax'))
        self.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
        self.summary()


if __name__ == '__main__':
    X_train, y_train, X_test, y_test, emb_matrix = prepare_sequential(merge=False, emb_name='glove-wiki-gigaword-300')

    #glove_cnn = GloveCNN(emb_matrix)
    #glove_cnn.build_model()
    #history = glove_cnn.fit(X_train, y_train, batch_size=32, epochs=20)

    #glove_rnn = GloveBiRNN(emb_matrix)
    #glove_rnn.build_model()
    #history = glove_rnn.fit(X_train, y_train, batch_size=32, epochs=20)

    #glove_gru = GloveGRU(emb_matrix)
    #glove_gru.build_model()
    #history = glove_gru.fit(X_train, y_train, batch_size=32, epochs=20)

    glove_lstm = GloveBiLSTM(emb_matrix)
    glove_lstm.build_model()
    history = glove_lstm.fit(X_train, y_train, batch_size=32, epochs=20)