Update steering (#652)

* -d option is required now when running do_entire_analysis.py since it
  has become impossible for a long time now that a database could be
  found via a default path

* overwrite database parameters on the fly passing
  --database-overwrite <path/to/overwrite.yaml>
  where parts of the nominal database structure can be replayed in
  <path/to/overwrite.yaml>. These supersede the nominal. E.g. an
  overwrite.yml containing

  ml:
    opt:
      filename_fonll: "<new/path/to/fonll>"

  would introduce another FONLL file for this run.
  NOTE: There is NO "case" key as it is present at the very top of the
        nominal databases

Co-authored-by: Benedikt Volkel <[email protected]>

Author: benedikt-voelkel

machine_learning_hep/config.py

@@ -18,6 +18,7 @@
 
 from itertools import product
 from machine_learning_hep.logger import get_logger
+from machine_learning_hep.do_variations import modify_dictionary
 
 
 # disable pylint unused-argument because this is done already in view of updating the
@@ -41,21 +42,28 @@ def update_config(database: dict, run_config: dict, database_overwrite=None): #
 
     logger = get_logger()
 
+    # Extract the case
+    case = list(database.keys())[0]
+    database = database[case]
+
     # First overwrite as required by the user
     # To be implemented
     if database_overwrite:
         logger.info("Updating database fields with custom user input")
+        modify_dictionary(database, database_overwrite)
 
-
-    # Extract the case
-    case = list(database.keys())[0]
-    database = database[case]
-
-    # If not an ML analysis, append "_std" to paths where necessary
+    # If not an ML analysis...
     if not database["doml"]:
-        logger.info("Not an ML analysis, adjust output paths")
+        logger.info("Not an ML analysis, adjust paths and settings accordingly")
+        # ...append "_std" to paths where necessary
         data_mc = ("data", "mc")
         pkl_keys = ("pkl_skimmed_dec", "pkl_skimmed_decmerged")
         for keys in product(data_mc, pkl_keys):
             database["mlapplication"][keys[0]][keys[1]][:] = \
                     [f"{path}_std" for path in database["mlapplication"][keys[0]][keys[1]]]
+        # ...set the ML working point all to 0
+        for k in data_mc:
+            database["mlapplication"]["probcutpresel"][k][:] = \
+                    [0] * len(database["mlapplication"]["probcutpresel"][k])
+        database["mlapplication"]["probcutoptimal"][:] \
+                = [0] * len(database["mlapplication"]["probcutoptimal"])

machine_learning_hep/processer.py

@@ -151,9 +151,6 @@ def __init__(self, case, datap, run_param, mcordata, p_maxfiles,
         self.lpt_model = datap["mlapplication"]["modelsperptbin"]
         self.dirmodel = datap["ml"]["mlout"]
         self.lpt_model = appendmainfoldertolist(self.dirmodel, self.lpt_model)
-        if not self.doml:
-            datap["mlapplication"]["probcutpresel"][self.mcordata] = [0 for _ in self.lpt_anbinmin]
-            datap["mlapplication"]["probcutoptimal"] = [0 for _ in self.lpt_anbinmin]
 
         self.lpt_probcutpre = datap["mlapplication"]["probcutpresel"][self.mcordata]
         self.lpt_probcutfin = datap["mlapplication"]["probcutoptimal"]

machine_learning_hep/steer_analysis.py

@@ -73,7 +73,8 @@
     print("##############################")
 
 
-def do_entire_analysis(data_config: dict, data_param: dict, data_model: dict, run_param: dict): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
+def do_entire_analysis(data_config: dict, data_param: dict, data_param_overwrite: dict, # pylint: disable=too-many-locals, too-many-statements, too-many-branches
+                       data_model: dict, run_param: dict):
 
     # Disable any graphical stuff. No TCanvases opened and shown by default
     gROOT.SetBatch(True)
@@ -85,7 +86,7 @@ def do_entire_analysis(data_config: dict, data_param: dict, data_model: dict, ru
     case = list(data_param.keys())[0]
 
     # Update database accordingly if needed
-    update_config(data_param, data_config)
+    update_config(data_param, data_config, data_param_overwrite)
 
     dodownloadalice = data_config["download"]["alice"]["activate"]
     doconversionmc = data_config["conversion"]["mc"]["activate"]
@@ -138,8 +139,8 @@ def do_entire_analysis(data_config: dict, data_param: dict, data_model: dict, ru
     do_syst_prob_cross = data_config["systematics"]["cutvar"]["probvariationcross"]
     dosystptshape = data_config["systematics"]["mcptshape"]["activate"]
     doanaperperiod = data_config["analysis"]["doperperiod"]
-
     typean = data_config["analysis"]["type"]
+
     dojetstudies = data_config["analysis"]["dojetstudies"]
 
     dirpklmc = data_param[case]["multi"]["mc"]["pkl"]
@@ -441,7 +442,7 @@ def do_entire_analysis(data_config: dict, data_param: dict, data_model: dict, ru
     syst_mgr.analyze(*ml_syst_steps)
 
 
-def load_config(user_path: str, default_path: tuple) -> dict:
+def load_config(user_path: str, default_path=None) -> dict:
     """
     Quickly extract either configuration given by user and fall back to package default if no user
     config given.
@@ -451,15 +452,16 @@ def load_config(user_path: str, default_path: tuple) -> dict:
     Returns:
         dictionary built from YAML
     """
-    logger = get_logger()
+    if not user_path and not default_path:
+        return None
+
     stream = None
-    if user_path is None:
-        stream = resource_stream(default_path[0], default_path[1])
-    else:
+    if user_path:
         if not exists(user_path):
-            logger_string = f"The file {user_path} does not exist."
-            logger.fatal(logger_string)
+            get_logger().fatal("The file %s does not exist", user_path)
         stream = open(user_path)
+    else:
+        stream = resource_stream(default_path[0], default_path[1])
     return yaml.safe_load(stream)
 
 def main():
@@ -474,7 +476,9 @@ def main():
     parser.add_argument("--run-config", "-r", dest="run_config",
                         help="the run configuration to be used")
     parser.add_argument("--database-analysis", "-d", dest="database_analysis",
-                        help="analysis database to be used")
+                        help="analysis database to be used", required=True)
+    parser.add_argument("--database-overwrite", dest="database_overwrite",
+                        help="overwrite fields in analysis database")
     parser.add_argument("--database-ml-models", dest="database_ml_models",
                         help="ml model database to be used")
     parser.add_argument("--database-run-list", dest="database_run_list",
@@ -490,15 +494,13 @@ def main():
     pkg_data = "machine_learning_hep.data"
     pkg_data_run_config = "machine_learning_hep.submission"
     run_config = load_config(args.run_config, (pkg_data_run_config, "default_complete.yml"))
-    case = run_config["case"]
     if args.type_ana is not None:
         run_config["analysis"]["type"] = args.type_ana
 
-    db_analysis_default_name = f"database_ml_parameters_{case}.yml"
-    print(args.database_analysis)
-    db_analysis = load_config(args.database_analysis, (pkg_data, db_analysis_default_name))
+    db_analysis = load_config(args.database_analysis)
+    db_analysis_overwrite = load_config(args.database_overwrite)
     db_ml_models = load_config(args.database_ml_models, (pkg_data, "config_model_parameters.yml"))
     db_run_list = load_config(args.database_run_list, (pkg_data, "database_run_list.yml"))
 
     # Run the chain
-    do_entire_analysis(run_config, db_analysis, db_ml_models, db_run_list)
+    do_entire_analysis(run_config, db_analysis, db_analysis_overwrite, db_ml_models, db_run_list)

machine_learning_hep/submission/default_ana.yml

@@ -1,4 +1,3 @@
-case: XXXX
 download:
   alice:
     activate: false

machine_learning_hep/submission/default_analyzer.yml

@@ -1,4 +1,3 @@
-case: XXXX # used to find the database file unless specified explicitly as do_entire_analysis -d database_analysis
 download:
   alice:
     activate: false

machine_learning_hep/submission/default_apply.yml

@@ -1,4 +1,3 @@
-case: XXXX
 download:
   alice:
     activate: false

machine_learning_hep/submission/default_pre.yml

@@ -1,4 +1,3 @@
-case: XXXX
 download:
   alice:
     activate: false

machine_learning_hep/submission/default_systematics.yml

@@ -1,4 +1,3 @@
-case: XXXX # used to find the database file unless specified explicitly as do_entire_analysis -d database_analysis
 download:
   alice:
     activate: false

machine_learning_hep/submission/default_train.yml

@@ -1,4 +1,3 @@
-case: XXXX
 download:
   alice:
     activate: false