More parsers (#181)

* start adding more subparsers

* start adding more subparsers

* remove early parsing

* wip: add more parsers

* add attribute error

* check before bugfx

* bugfix

* add more parsing options to instructions

* add attribute error for final plots

* add attribute error for final plots

* remove run-

Author: clementhelsens

README.md

@@ -170,8 +170,7 @@ variables needs extra fields like `title`, number of bins and range for the
 histogram creation. In the example analysis it can be run like this:
 
 ```shell
-fccanalysis run examples/FCCee/higgs/mH-recoil/mumu/analysis_final.py \
-       --final
+fccanalysis final examples/FCCee/higgs/mH-recoil/mumu/analysis_final.py
 ```
 
 This will create 2 files per selection `SAMPLENAME_SELECTIONNAME.root` for the
@@ -187,8 +186,7 @@ the rendering of the plots but also ways of combining samples for plotting.
 In the example analysis it can be run in the following manner:
 
 ```shell
-fccanalysis run examples/FCCee/higgs/mH-recoil/mumu/analysis_plots.py \
-       --plots
+fccanalysis plots examples/FCCee/higgs/mH-recoil/mumu/analysis_plots.py
 ```
 
 Resulting plots will be located the `outdir` defined in the analysis file.

bin/fccanalysis

@@ -4,11 +4,25 @@
 if __name__ == "__main__":
   import argparse
   import sys
-  parser = argparse.ArgumentParser()
-  subparsers = parser.add_subparsers() 
-  parser_run = subparsers.add_parser('run', help="run a RDataFrame based FCC analysis")
-  if len(sys.argv)<2:
-     print("for usage run fccanalyses --help")
+  parser = argparse.ArgumentParser('FCCAnalyses parser')
+  subparsers = parser.add_subparsers(help='types of running modes', dest='command')
+  parser_run       = subparsers.add_parser('run', help="run a RDataFrame based FCC analysis")
+  parser_run_final = subparsers.add_parser('final', help="run a RDataFrame based FCC analysis final configuration")
+  parser_run_plots = subparsers.add_parser('plots', help="run a RDataFrame based FCC analysis plot configuration")
+
+  from config.Parsers import *
+  setup_run_parser(parser_run)
+  setup_run_parser_final(parser_run_final)
+  setup_run_parser_plots(parser_run_plots)
+
+  args = parser.parse_args()
+  print('===============args bin ',args)
+
+  if len(sys.argv)<3:
+     print("minimal running requirements :   fccanalysis <runoption> <analysisfile>")
+     print("running example              :   fccanalysis run examples/FCCee/higgs/mH-recoil/mumu/analysis_stage1.py")
+     print("for running options, try     :   fccanalysis --help  and   fccanalysis <runoption> --help")
      sys.exit(3)
-  from config.FCCAnalysisRun import * #such that the dictionary is loaded only if the configuration is ok
-  run(parser, parser_run)
+
+  from config.FCCAnalysisRun import run
+  run(parser)

config/FCCAnalysisRun.py

@@ -596,7 +596,8 @@ def runStages(args, rdfModule, preprocess):
             #run locally
             if runBatch == False:
                 print ('----> Running Locally')
-                runLocal(rdfModule, chunkList[ch], outputchunk, args)
+                args.output = outputchunk
+                runLocal(rdfModule, chunkList[ch], args)
 
             #run on batch
         if runBatch == True:
@@ -851,16 +852,19 @@ def setup_run_parser(parser):
 
 
 #__________________________________________________________
-def run(mainparser, subparser):
+def run(mainparser, subparser=None):
     """
     Set things in motion.
-    The two parser arguments are a hack to allow running this 
+    The two parser arguments are a hack to allow running this
     both as `fccanalysis run` and `python config/FCCAnalysisRun.py`
     For the latter case, both are the same (see below).
     """
-    setup_run_parser(subparser)
-    args, _ = mainparser.parse_known_args()
 
+    if subparser:
+        print("===================setup subparser")
+        setup_run_parser(subparser)
+    args, _ = mainparser.parse_known_args()
+    print("args in mains code==============================",args)
     #check that the analysis file exists
     analysisFile = args.pathToAnalysisScript
     if not os.path.isfile(analysisFile):
@@ -869,14 +873,28 @@ def run(mainparser, subparser):
         sys.exit(3)
 
     #set the RDF ELogLevel
-    verbosity = ROOT.Experimental.RLogScopedVerbosity(ROOT.Detail.RDF.RDFLogChannel(), getattr(ROOT.Experimental.ELogLevel,args.eloglevel))
-
+    try:
+        verbosity = ROOT.Experimental.RLogScopedVerbosity(ROOT.Detail.RDF.RDFLogChannel(), getattr(ROOT.Experimental.ELogLevel,args.eloglevel))
+    except AttributeError:
+        pass
     #load the analysis
     analysisFile=os.path.abspath(analysisFile)
+    print ("--------------loading analysis file  ",analysisFile)
     rdfSpec   = importlib.util.spec_from_file_location("rdfanalysis", analysisFile)
     rdfModule = importlib.util.module_from_spec(rdfSpec)
     rdfSpec.loader.exec_module(rdfModule)
 
+    try:
+        args.command
+        if args.command == "run":          runStages(args, rdfModule, args.preprocess)
+        elif args.command == "final":  runFinal(rdfModule)
+        elif args.command == "plots":  runPlots(analysisFile)
+        return
+    except AttributeError:
+        print("============running the old way")
+
+
+    #below is legacy using the old way of runnig with options in "python config/FCCAnalysisRun.py analysis.py --options
     #check if this is final analysis
     if args.final:
         if args.plots:
@@ -909,12 +927,12 @@ def run(mainparser, subparser):
                 sys.exit(3)
         runStages(args, rdfModule, args.preprocess)
 
-    
+
 #__________________________________________________________
 if __name__ == "__main__":
     print("Running this script directly is deprecated, use `fccanalysis run` instead.")
-    # legacy behavior: allow running this script directly 
-    # with python config/FCCAnalysis.py 
+    # legacy behavior: allow running this script directly
+    # with python config/FCCAnalysis.py
     # and the same behavior as `fccanalysis run`
     import argparse
     parser = argparse.ArgumentParser()

config/Parsers.py

@@ -0,0 +1,27 @@
+def setup_run_parser(parser):
+    publicOptions = parser.add_argument_group('User options')
+    publicOptions.add_argument("pathToAnalysisScript", help="path to analysis script")
+    publicOptions.add_argument("--files-list", help="Specify input file to bypass the processList", default=[], nargs='+')
+    publicOptions.add_argument("--output", help="Specify output file name to bypass the processList and or outputList, default output.root", type=str, default="output.root")
+    publicOptions.add_argument("--nevents", help="Specify max number of events to process", type=int, default=-1)
+    publicOptions.add_argument("--test", action='store_true', help="Run over the test file", default=False)
+    publicOptions.add_argument('--bench', action='store_true', help='Output benchmark results to a JSON file', default=False)
+    #publicOptions.add_argument("--final", action='store_true', help="Run final analysis (produces final histograms and trees)", default=False)
+    #publicOptions.add_argument("--plots", action='store_true', help="Run analysis plots", default=False)
+    publicOptions.add_argument("--preprocess", action='store_true', help="Run preprocessing", default=False)
+    publicOptions.add_argument("--validate", action='store_true', help="Validate a given production", default=False)
+    publicOptions.add_argument("--rerunfailed", action='store_true', help="Rerun failed jobs", default=False)
+    publicOptions.add_argument("--jobdir", help="Specify the batch job directory", type=str, default="output.root")
+    publicOptions.add_argument("--eloglevel", help="Specify the RDataFrame ELogLevel", type=str, default="kUnset", choices = ['kUnset','kFatal','kError','kWarning','kInfo','kDebug'])
+
+    internalOptions = parser.add_argument_group('\033[4m\033[1m\033[91m Internal options, NOT FOR USERS\033[0m')
+    internalOptions.add_argument("--batch", action='store_true', help="Submit on batch", default=False)
+
+def setup_run_parser_final(parser):
+    publicOptions = parser.add_argument_group('User final options')
+    publicOptions.add_argument("pathToAnalysisScript", help="path to analysis_final script")
+    publicOptions.add_argument("--eloglevel", help="Specify the RDataFrame ELogLevel", type=str, default="kUnset", choices = ['kUnset','kFatal','kError','kWarning','kInfo','kDebug'])
+
+def setup_run_parser_plots(parser):
+    publicOptions = parser.add_argument_group('User plots options')
+    publicOptions.add_argument("pathToAnalysisScript", help="path to analysis_plots script")