From 92bef120c9373082edf37d9baa7ccec2250b7e15 Mon Sep 17 00:00:00 2001
From: Tanmoy Bhattacharya <tanmoy@mindspring.com>
Date: Wed, 22 Aug 2018 11:02:22 -0500
Subject: [PATCH 1/4] Data directory should not be committed

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d646835b..3253a2d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 *.pyc
 __pycache__/
+Data

From 5ef9cb0b5dcd40bc78649be8af0f1527d74139c9 Mon Sep 17 00:00:00 2001
From: Tanmoy Bhattacharya <tanmoy@mindspring.com>
Date: Wed, 22 Aug 2018 11:06:06 -0500
Subject: [PATCH 2/4] Needed to import print_function from future to allow
 Python 2 usage

---
 common/generic_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/generic_utils.py b/common/generic_utils.py
index 27db31e2..afb6d18f 100644
--- a/common/generic_utils.py
+++ b/common/generic_utils.py
@@ -1,4 +1,4 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 import numpy as np
 import time
 import sys

From 95fb1eed45b8d4c052dfabfa424cdbc5f30968b5 Mon Sep 17 00:00:00 2001
From: Tanmoy Bhattacharya <tanmoy@mindspring.com>
Date: Wed, 22 Aug 2018 11:06:49 -0500
Subject: [PATCH 3/4] Allow logging of metrics

---
 common/solr_keras.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/common/solr_keras.py b/common/solr_keras.py
index 4d8bb8fe..319f772f 100644
--- a/common/solr_keras.py
+++ b/common/solr_keras.py
@@ -23,13 +23,20 @@ def compute_trainable_params(model):
 
 class CandleRemoteMonitor(Callback):
     """Capture Run level output and store/send for monitoring
+
+       Arguments:
+          params: the gParams global parameters dictionary
+          monitor: an array of strings specifying the metrics to be monitored
+                   For every string s, both s and 'val_'+s will be monitored
     """
 
     def __init__(self,
-                 params=None):
+                 params=None,
+                 monitor=None):
         super(CandleRemoteMonitor, self).__init__()
 
         self.global_params = params
+        self.monitor = monitor or []
         self.has_solr_config = False
         if 'solr_root' in params and params['solr_root'] != '':
             self.has_solr_config = True
@@ -77,13 +84,17 @@ def on_epoch_end(self, epoch, logs=None):
         epoch_in_sec = epoch_duration.total_seconds()
         epoch_line = "epoch: {}/{}, duration: {}s, loss: {}, val_loss: {}".format(
             (epoch + 1), epoch_total, epoch_in_sec, loss, val_loss)
-
-        send = {'run_id': self.run_id,
-                'status': {'set': 'Running'},
-                'training_loss': {'set': loss},
-                'validation_loss': {'set': val_loss},
-                'run_progress': {'add': [epoch_line]}
-               }
+        send = {}
+        for s in self.monitor:
+            send[s] = {'set': logs.get(s)}
+            send['val_'+s] = {'set': logs.get('val_' + s)}
+
+        send.update({'run_id': self.run_id,
+                     'status': {'set': 'Running'},
+                     'training_loss': {'set': loss},
+                     'validation_loss': {'set': val_loss},
+                     'run_progress': {'add': [epoch_line]}
+                     })
         # print("on_epoch_end", send)
         self.log_messages.append(send)
         if self.has_solr_config:

From 519deb26d7e5855cc735cfcc6a5e687d000d085d Mon Sep 17 00:00:00 2001
From: Jamal <jamal@lanl.gov>
Date: Wed, 22 Aug 2018 15:30:29 -0600
Subject: [PATCH 4/4] Jamal fixed upstream; cherry-picking to resolve conflict

---
 common/generic_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common/generic_utils.py b/common/generic_utils.py
index afb6d18f..2809a246 100644
--- a/common/generic_utils.py
+++ b/common/generic_utils.py
@@ -1,4 +1,5 @@
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import
+from __future__ import print_function
 import numpy as np
 import time
 import sys