Update phishing_email_detection_gpt2.py

Test with default params with AdamW.

Author: david-thrower

phishing_email_detection_gpt2.py

@@ -85,6 +85,7 @@
 
 """### A custom GPT2 encoder layer for text embedding"""
 
+"""
 
 class GPT2Layer(tf.keras.layers.Layer):
 
@@ -183,6 +184,7 @@ def from_config(cls, config):
 hy_df = pd.DataFrame(history.history)
 print(hy_df)
 
+"""
 
 ### Cerebros model:
 
@@ -248,17 +250,6 @@ def from_config(cls, config):
 )
 
 
-
-
-
-# dropout_embedded = tf.keras.layers.Dropout(0.6)(embedded)
-# flattened = tf.keras.layers.Flatten()(dropout_embedded)
-
-# cerebros_base_model =\
-#     tf.keras.Model(
-#         inputs=inp,
-#         outputs=flattened)
-
 """### Cerebros search for the best model"""
 
 #
@@ -352,20 +343,11 @@ def from_config(cls, config):
 cerebros_time_per_model = cerebros_time_all_models_min / models_tried
 
 print(f"Cerebros trained {models_tried} models FROM A COLD START in ONLY {cerebros_time_all_models_min} min. Cerebros took only {cerebros_time_per_model} minutes on average per model.")
-print(f"GPT2 took {gpt_time_on_one_model_min} just to FINE TUNE one PRE - TRAINED model. Although this is a small scale test, this shows the advantage of scaling in ON timing VS ON**2 timing.")
+# print(f"GPT2 took {gpt_time_on_one_model_min} just to FINE TUNE one PRE - TRAINED model. Although this is a small scale test, this shows the advantage of scaling in ON timing VS ON**2 timing.")
 
 
 print(f'Cerebros best accuracy achieved is {result}')
 print(f'val set accuracy')
 
 # """### Testing the best model found"""
 
-# #
-# # Load the best model (taking into account that it has a custom layer)
-# #
-# best_model_found =\
-# tf.keras.models.load_model(cerebros_automl.best_model_path,\
-# custom_objects={'GPT2Layer': GPT2Layer(max_seq_length)})
-
-# print('Evaluating on the test dataset')
-# best_model_found.evaluate(X_test, y_test)