REE NN

Author: catherinelee274

Fall20/NeuralNetworks4/rnn.py

@@ -2,9 +2,12 @@
 import tensorflow as tf
 from tensorflow import keras
 # from tensorflow.keras import layers
-# TODO: error : ValueError: Error when checking input: expected embedding_1_input to have 2 dimensions, but got array with shape (196024, 40, 55)
-# from keras.optimizers import RMSprop
+# from keras.callbacks import LambdaCallback
+import random
+import sys
+import io
 
+# one hot encoding / vectorizing 
 with open('1984.txt', 'r') as file:
     text = file.read().lower()
 print('text length', len(text))
@@ -13,6 +16,7 @@
 char_indices = dict((c, i) for i, c in enumerate(chars))
 indices_char = dict((i, c) for i, c in enumerate(chars))
 
+# create our time steps
 maxlen = 40
 step = 3
 sentences = []
@@ -27,23 +31,60 @@
         x[i, t, char_indices[char]] = 1
     y[i, char_indices[next_chars[i]]] = 1
 
-model = keras.Sequential()
+# model = keras.Sequential()
 # Add an Embedding layer expecting input vocab of size 1000, and
 # output embedding dimension of size 64.
-model.add(keras.layers.Embedding(input_dim=1000, output_dim=64))
-
-# Add a RNN layer with 128 internal units.
-model.add(keras.layers.SimpleRNN(128))
-
-# Add a Dense layer with 10 units.
-model.add(keras.layers.Dense(10))
+# model.add(keras.layers.Embedding(input_dim=1000, output_dim=64))
 
-model.summary()
+# Add a lstm layer with 128 internal units.
+model = keras.Sequential()
+model.add(keras.layers.LSTM(128, input_shape=(maxlen, len(chars))))
+model.add(keras.layers.Dense(len(chars), activation='softmax'))
 
 # predict some text 
 
 optimizer = keras.optimizers.RMSprop(lr=0.01)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
+def sample(preds, temperature=1.0):
+    # helper function to sample an index from a probability array
+    preds = np.asarray(preds).astype('float64')
+    preds = np.log(preds) / temperature
+    exp_preds = np.exp(preds)
+    preds = exp_preds / np.sum(exp_preds)
+    probas = np.random.multinomial(1, preds, 1)
+    return np.argmax(probas)
+
+def on_epoch_end(epoch, _):
+    # Function invoked at end of each epoch. Prints generated text.
+    print()
+    print('----- Generating text after Epoch: %d' % epoch)
+
+    start_index = random.randint(0, len(text) - maxlen - 1)
+    for diversity in [0.2, 0.5, 1.0, 1.2]:
+        print('----- diversity:', diversity)
+
+        generated = ''
+        sentence = text[start_index: start_index + maxlen]
+        generated += sentence
+        print('----- Generating with seed: "' + sentence + '"')
+        sys.stdout.write(generated)
+
+        for i in range(400):
+            x_pred = np.zeros((1, maxlen, len(chars)))
+            for t, char in enumerate(sentence):
+                x_pred[0, t, char_indices[char]] = 1.
+
+            preds = model.predict(x_pred, verbose=0)[0]
+            next_index = sample(preds, diversity)
+            next_char = indices_char[next_index]
+
+            generated += next_char
+            sentence = sentence[1:] + next_char
 
+            sys.stdout.write(next_char)
+            sys.stdout.flush()
+        print()
 
-model.fit(x, y, batch_size=128, epochs=5)
+print_callback = keras.callbacks.LambdaCallback(on_epoch_end=on_epoch_end)
+model.fit(x, y, batch_size=128, epochs=5,
+          callbacks=[print_callback])