Added some logging functionality from the P1Combo model into the

generic_utils and then used them in the P2B1 model to display training
progress.

Author: bvanessen

Pilot2/P2B1/p2b1.py

@@ -328,15 +328,11 @@ def train_ac(self):
                 #for frame in random.sample(range(len(xt_all)), int(self.sampling_density*len(xt_all))):
                 for frame in range(len(xt_all)):
                     history = self.molecular_model.fit(xt_all[frame], yt_all[frame], epochs=1,
-                                                       batch_size=self.batch_size, callbacks=self.callbacks[:2],
-                                                       verbose=0)
+                                                       batch_size=self.batch_size, callbacks=self.callbacks[:2])
                     frame_loss.append(history.history['loss'])
                     frame_mse.append(history.history['mean_squared_error'])
 
                     if not frame % 20 or self.sampling_density != 1.0:
-                        print ("Frame: {0:d}, Current history:\nLoss: {1:3.5f}\tMSE: {2:3.5f}\n"
-                               .format(frame, history.history['loss'][0], history.history['mean_squared_error'][0]))
-
                         # Update weights filed every few frames
                         self.molecular_model.save_weights(model_weight_file)
                         self.molecular_encoder.save_weights(encoder_weight_file)

Pilot2/P2B1/p2b1_baseline_keras2.py

@@ -4,6 +4,7 @@
 import sys, os, json
 import argparse
 import h5py
+import logging
 try:
     reload  # Python 2.7
 except NameError:
@@ -28,6 +29,9 @@
 
 HOME = os.environ['HOME']
 
+logger = logging.getLogger(__name__)
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
 def parse_list(option, opt, value, parser):
     setattr(parser.values, option.dest, value.split(','))
 
@@ -86,6 +90,16 @@ def run(GP):
         sys.exit(0)
     sys.path.append(GP['home_dir'])
 
+    # Setup loggin
+    args = candle.ArgumentStruct(**GP)
+#    set_seed(args.rng_seed)
+#    ext = extension_from_parameters(args)
+    candle.verify_path(args.save_path)
+    prefix = args.save_path # + ext
+    logfile = args.logfile if args.logfile else prefix+'.log'
+    candle.set_up_logger(logger, logfile, False) #args.verbose
+    logger.info('Params: {}'.format(GP))
+
     import p2b1 as hf
     reload(hf)
 
@@ -238,9 +252,10 @@ def step_decay(epoch):
     history = callbacks.History()
     # callbacks=[history,lr_scheduler]
 
+    history_logger = candle.LoggingCallback(logger.debug)
     candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP)
     timeoutMonitor = candle.TerminateOnTimeOut(TIMEOUT)
-    callbacks = [history, candleRemoteMonitor, timeoutMonitor]
+    callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor]
     loss = 0.
 
 #### Save the Model to disk

common/candle_keras/__init__.py

@@ -28,5 +28,10 @@
 from keras_utils import set_seed
 from keras_utils import set_parallelism_threads
 
+from generic_utils import Progbar
+from generic_utils import verify_path
+from generic_utils import set_up_logger
+from generic_utils import LoggingCallback
+
 from solr_keras import CandleRemoteMonitor, compute_trainable_params, TerminateOnTimeOut
 

common/generic_utils.py

@@ -2,10 +2,12 @@
 import numpy as np
 import time
 import sys
+import os
 import six
 import marshal
 import types as python_types
-
+import logging
+from keras.callbacks import Callback
 
 def get_from_module(identifier, module_params, module_name,
                     instantiate=False, kwargs=None):
@@ -191,3 +193,32 @@ def display_row(objects, positions):
 
     for objects in rows:
         display_row(objects, positions)
+
+
+def verify_path(path):
+    folder = os.path.dirname(path)
+    if folder and not os.path.exists(folder):
+        os.makedirs(folder)
+
+def set_up_logger(logger, logfile, verbose):
+    verify_path(logfile)
+    fh = logging.FileHandler(logfile)
+    fh.setFormatter(logging.Formatter("[%(asctime)s %(process)d] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
+    fh.setLevel(logging.DEBUG)
+
+    sh = logging.StreamHandler()
+    sh.setFormatter(logging.Formatter(''))
+    sh.setLevel(logging.DEBUG if verbose else logging.INFO)
+
+    logger.setLevel(logging.DEBUG)
+    logger.addHandler(fh)
+    logger.addHandler(sh)
+
+class LoggingCallback(Callback):
+    def __init__(self, print_fcn=print):
+        Callback.__init__(self)
+        self.print_fcn = print_fcn
+
+    def on_epoch_end(self, epoch, logs={}):
+        msg = "[Epoch: %i] %s" % (epoch, ", ".join("%s: %f" % (k, v) for k, v in sorted(logs.items())))
+        self.print_fcn(msg)