Merge branch 'develop' of https://github.com/ECP-Candle/Benchmarks into develop

Author: brettinanl

Pilot1/Uno/test.sh

@@ -3,12 +3,13 @@
 set -x
 
 # AUC prediction model
-if [ ! -f "top_21_auc_1fold.uno.h5" ]; then
-  curl -o top_21_auc_1fold.uno.h5 http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/uno/top_21_auc_1fold.uno.h5
+if [ ! -f "$CANDLE_DATA_DIR/Pilot1/Uno/top_21_auc_1fold.uno.h5" ]; then
+  # download from http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/uno/top_21_auc_1fold.uno.h5
+  python -c 'from candle import get_file;get_file(fname="top_21_auc_1fold.uno.h5", origin="https://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/uno/top_21_auc_1fold.uno.h5", cache_subdir="Pilot1/Uno")'
 fi
 
-python uno_baseline_keras2.py --config_file uno_auc_model.txt --use_exported_data top_21_auc_1fold.uno.h5 -e 3 --save_weights save/saved.model.weights.h5
-python uno_infer.py --data top_21_auc_1fold.uno.h5 \
+python uno_baseline_keras2.py --config_file uno_auc_model.txt --use_exported_data "$CANDLE_DATA_DIR/Pilot1/Uno/top_21_auc_1fold.uno.h5" -e 3 --save_weights save/saved.model.weights.h5
+python uno_infer.py --data "$CANDLE_DATA_DIR/Pilot1/Uno/top_21_auc_1fold.uno.h5" \
   --model_file save/'uno.A=relu.B=32.E=3.O=adamax.LR=0.0001.CF=r.DF=d.DR=0.1.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.D4=1000.D5=1000.FD1=1000.FD2=1000.FD3=1000.FD4=1000.FD5=1000.model.json' \
   --weights_file save/saved.model.weights.h5 \
   --partition val \
@@ -17,5 +18,5 @@ python uno_infer.py --data top_21_auc_1fold.uno.h5 \
   --agg_dose AUC
 
 # CLR model
-python uno_clr_keras2.py --config_file uno_auc_clr_model.txt --use_exported_data top_21_auc_1fold.uno.h5 -e 3
+python uno_clr_keras2.py --config_file uno_auc_clr_model.txt --use_exported_data ../../Data/Pilot1/Uno/top_21_auc_1fold.uno.h5 -e 3
 

Pilot3/P3B7/data.py

@@ -200,7 +200,7 @@ def _download(self):
         self._preprocess(raw_data)
 
     def _preprocess(self, raw_data):
-        print(f"Preprocessing data...")
+        print(f'{"Preprocessing data..."}')
         self._make_processed_dirs()
 
         with open(raw_data, 'rb') as f:
@@ -215,7 +215,7 @@ def _preprocess(self, raw_data):
         self._save_split('train', corpus.train, y_train)
         self._save_split('valid', corpus.valid, y_valid)
         self._save_vocab(corpus.vocab)
-        print(f"Done!")
+        print(f'{"Done!"}')
 
     def _save_split(self, split, data, target):
         target = self._create_target(target)

Pilot3/P3B7/p3b7_baseline_pytorch.py

@@ -2,9 +2,9 @@
 import p3b7 as bmk
 import candle
 
-import numpy as np
+import pandas as pd
+from pathlib import Path
 
-import torch.nn as nn
 from torch.utils.data import DataLoader
 
 from data import P3B3, Egress
@@ -13,10 +13,7 @@
 from meters import AccuracyMeter
 from metrics import F1Meter
 
-from prune import (
-    negative_prune, min_max_prune,
-    create_prune_masks, remove_prune_masks
-)
+from prune import create_prune_masks, remove_prune_masks
 
 
 TASKS = {
@@ -134,47 +131,35 @@ def evaluate(model, loader, device):
 
     accmeter.update_accuracy()
 
-    print(f'Validation accuracy:')
+    print(f'{"Validation accuracy:"}')
     accmeter.print_task_accuracies()
 
     loss /= len(loader.dataset)
 
     return loss
 
 
-def save_dataframe(metrics, filename):
+def save_dataframe(metrics, filename, args):
     """Save F1 metrics"""
     df = pd.DataFrame(metrics, index=[0])
-    path = Path(ARGS.savepath).joinpath(f'f1/{filename}.csv')
+    path = Path(args.savepath).joinpath(f'f1/{filename}.csv')
     df.to_csv(path, index=False)
 
 
 def run(args):
     args = candle.ArgumentStruct(**args)
     args.cuda = torch.cuda.is_available()
-    args.device = torch.device(f"cuda" if args.cuda else "cpu")
-
-    if args.use_synthetic_data:
-        train_data, valid_data = get_synthetic_data(args)
-
-        hparams = Hparams(
-            kernel1=args.kernel1,
-            kernel2=args.kernel2,
-            kernel3=args.kernel3,
-            embed_dim=args.embed_dim,
-            n_filters=args.n_filters,
-        )
-    else:
-        train_data, valid_data = get_egress_data(tasks)
-
-        hparams = Hparams(
-            kernel1=args.kernel1,
-            kernel2=args.kernel2,
-            kernel3=args.kernel3,
-            embed_dim=args.embed_dim,
-            n_filters=args.n_filters,
-            vocab_size=len(train_data.vocab)
-        )
+    args.device = torch.device(f'{"cuda"}' if args.cuda else "cpu")
+
+    train_data, valid_data = get_synthetic_data(args)
+
+    hparams = Hparams(
+        kernel1=args.kernel1,
+        kernel2=args.kernel2,
+        kernel3=args.kernel3,
+        embed_dim=args.embed_dim,
+        n_filters=args.n_filters,
+    )
 
     train_loader = DataLoader(train_data, batch_size=args.batch_size)
     valid_loader = DataLoader(valid_data, batch_size=args.batch_size)

Pilot3/P3B7/prune.py

@@ -128,7 +128,6 @@ def compute_mask(self, t, default_mask):
         # Check that the amount of units to prune is not > than the number of
         # parameters in t
         tensor_size = t.nelement()
-        tNP = t.detach().cpu().numpy()
         # Compute number of units to prune: amount if int,
         # else amount * tensor_size
         nparams_toprune = _compute_nparams_toprune(self.amount, tensor_size)

README.md

@@ -17,8 +17,6 @@ At this point, we will refer to a benchmark by it's problem area and benchmark n
 
 Over time, we will be adding implementations that make use of different tensor frameworks. The primary (baseline) benchmarks are implemented using keras, and are named with '_baseline' in the name, for example p3b1_baseline_keras2.py. 
 
-Implementations that use alternative tensor frameworks, such as mxnet or neon, will have the name of the framework in the name. Examples can be seen in the P1B3 benchmark contribs/ directory, for example:
-        p1b3_mxnet.py
-        p1b3_neon.py
+Implementations that use alternative tensor frameworks, such as pytorch, will have the name of the framework in the name. Examples can be seen in the Pilot1 benchmark UnoMT directory, for example: `unoMT_baseline_pytorch.py`
 
-Documentation: https://ecp-candle.github.io/Candle/html/readme.html
+Documentation: https://ecp-candle.github.io/Candle/

common/uq_keras_utils.py

@@ -759,15 +759,15 @@ def loss(y_true, y_pred):
 
         y_shape = K.shape(y_true)
         if nout > 1:
-            y_out0 = K.reshape(y_pred[:, 0::nout], y_shape)
-            y_out1 = K.reshape(y_pred[:, 1::nout], y_shape)
-            y_out2 = K.reshape(y_pred[:, 2::nout], y_shape)
+            y_qtl0 = K.reshape(y_pred[:, 0::3], y_shape)
+            y_qtl1 = K.reshape(y_pred[:, 1::3], y_shape)
+            y_qtl2 = K.reshape(y_pred[:, 2::3], y_shape)
         else:
-            y_out0 = K.reshape(y_pred[:, 0], y_shape)
-            y_out1 = K.reshape(y_pred[:, 1], y_shape)
-            y_out2 = K.reshape(y_pred[:, 2], y_shape)
+            y_qtl0 = K.reshape(y_pred[:, 0], y_shape)
+            y_qtl1 = K.reshape(y_pred[:, 1], y_shape)
+            y_qtl2 = K.reshape(y_pred[:, 2], y_shape)
 
-        return quantile_loss(lowquantile, y_true, y_out1) + quantile_loss(highquantile, y_true, y_out2) + 2. * quantile_loss(0.5, y_true, y_out0)
+        return quantile_loss(lowquantile, y_true, y_qtl1) + quantile_loss(highquantile, y_true, y_qtl2) + 2. * quantile_loss(0.5, y_true, y_qtl0)
 
     return loss
 
@@ -795,10 +795,10 @@ def metric(y_true, y_pred):
         """
         y_shape = K.shape(y_true)
         if nout > 1:
-            y_out = K.reshape(y_pred[:, index::nout], y_shape)
+            y_qtl = K.reshape(y_pred[:, index::3], y_shape)
         else:
-            y_out = K.reshape(y_pred[:, index], y_shape)
-        return quantile_loss(quantile, y_true, y_out)
+            y_qtl = K.reshape(y_pred[:, index], y_shape)
+        return quantile_loss(quantile, y_true, y_qtl)
 
     metric.__name__ = 'quantile_{}'.format(quantile)
     return metric
@@ -819,7 +819,11 @@ def add_index_to_output(y_train):
     """
     # Add indices to y
     y_train_index = range(y_train.shape[0])
-    y_train_augmented = np.vstack([y_train, y_train_index]).T
+    if y_train.ndim > 1:
+        shp = (y_train.shape[0], 1)
+        y_train_augmented = np.hstack([y_train, np.reshape(y_train_index, shp)])
+    else:
+        y_train_augmented = np.vstack([y_train, y_train_index]).T
 
     return y_train_augmented
 
@@ -884,6 +888,11 @@ def __init__(self, x, y, a_max=0.99):
             Maximum value of a variable to allow
         """
         super(Contamination_Callback, self).__init__()
+        if y.ndim > 1:
+            if y.shape[1] > 1:
+                raise Exception(
+                    'ERROR ! Contamination model can be applied to one-output regression, but provided training data has: '
+                    + str(y.ndim) + 'outpus... Exiting')
 
         self.x = x                              # Features of training set
         self.y = y                              # Output of training set