tools: a few additional usability tweaks

Author: torbjoernk

tools/generate_coverage.py

@@ -52,7 +52,9 @@ class Options(object):
     coverage_dir = ""
     build_dir = ""
     base_dir = ""
+    with_examples = True
     tests = []
+    example_tests = []
     tracefiles = []
     final_tracefile = ""
 
@@ -98,13 +100,21 @@ def setup_and_init_options():
                   "    -Dpfasst_WITH_GCC_PROF=ON -Dpfasst_BUILD_TESTS=ON"
 
     parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=help_string)
-    parser.add_argument('-d', '--build_dir', required=True,
+    parser.add_argument('-d', '--build-dir', required=True,
                         help="name of build directory containing a debug build with GCC and enabled profiling")
     parser.add_argument('-o', '--output', default='coverage',
                         help="output directory for generated coverage report")
+    parser.add_argument('--no-examples', default=False, action='store_true',
+                        help="whether to not run and include tests from the examples")
+    parser.add_argument('--debug', default=False, action='store_true',
+                        help="enables more verbose debugging output")
 
     _args = parser.parse_args()
 
+    if _args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+        logging.debug("Debug mode enabled.")
+
     get_project_root()
 
     if not os.access(_args.build_dir, os.W_OK):
@@ -113,52 +123,63 @@ def setup_and_init_options():
     options.build_dir = os.path.abspath(_args.build_dir)
 
     if not os.access(_args.output, os.W_OK):
-        logging.info("output directory not found. creating: %s" % _args.output)
+        logging.info("Output directory not found. Creating: %s" % _args.output)
         os.mkdir(_args.output)
     else:
-        logging.warning("clearing out output directory: %s" % _args.output)
+        logging.warning("Clearing out output directory: %s" % _args.output)
         shutil.rmtree(_args.output)
         os.mkdir(_args.output)
     options.coverage_dir = os.path.abspath(_args.output)
 
+    options.with_examples = not _args.no_examples
+    if not options.with_examples:
+        logging.debug("Not running and tracing tests from examples.")
+
 
 def get_test_directories():
-    logging.info("looking for tests ...")
+    logging.info("Looking for tests ...")
     for root, dirs, files in os.walk(options.build_dir + '/tests'):
         match_name = re.search('^.*/(?P<test_name>test_[a-zA-Z\-_]+)\.dir$', root)
         match_is_example = re.search('^.*/tests/examples/.*$', root)
         is_example = match_is_example is not None
         if match_name is not None:
             testname = match_name.groupdict()['test_name']
-            options.tests.append({'path': root, 'name': testname, 'is_example': is_example})
-    logging.info("%d tests found" % len(options.tests))
+            if is_example:
+                options.example_tests.append({'path': root, 'name': testname, 'is_example': is_example})
+            else:
+                options.tests.append({'path': root, 'name': testname, 'is_example': is_example})
+    logging.info("%d tests found" % (len(options.tests) + len(options.example_tests)))
+    logging.info("  %d general tests" % len(options.tests))
+    if options.with_examples:
+        logging.info("  %d tests for examples" % len(options.example_tests))
 
 
 def run_test(path, name, is_example):
-    logging.info("running %s" % name)
-    logging.debug("in %s" % path)
+    logging.info("- %s" % name)
+    logging.debug("Found in %s" % path)
     output_file = open('%s/%s.log' % (options.coverage_dir, name), mode='a')
+    logging.debug("Output log: %s" % output_file.name)
 
     os.chdir(os.path.abspath(path))
-    logging.debug("deleting old tracing data ...")
+    logging.debug("Deleting old tracing data ...")
     print('### deleting old tracing data ...', file=output_file, flush=True)
     sp.check_call('lcov --zerocounters --directory .', shell=True, stdout=output_file, stderr=output_file)
     print('### done.', file=output_file, flush=True)
 
     os.chdir(options.build_dir)
-    logging.debug("running test ...")
+    logging.debug("Running test ...")
     print('### running test ...', file=output_file, flush=True)
     sp.check_call('ctest -R %s' % name, shell=True, stdout=output_file, stderr=output_file)
     print('### done.', file=output_file, flush=True)
 
     os.chdir(os.path.abspath(path))
-    logging.debug("capturing all tracing data ...")
+    logging.debug("Capturing all tracing data ...")
     print('### capturing all tracing data ...', file=output_file, flush=True)
     sp.check_call('lcov --capture --directory . --output-file "%s.info.complete"' % name,
                   shell=True, stdout=output_file, stderr=output_file)
     print('### done.', file=output_file, flush=True)
 
-    logging.debug("removing unnecessary data ...")
+    logging.debug("Removing unnecessary data ...")
     print('### removing unnecessary data ...', file=output_file, flush=True)
     try:
         sp.check_call('lcov --remove "%s.info.complete" "%s/include/pfasst/easylogging++.h" --output-file %s.info.prelim'
@@ -168,7 +189,7 @@ def run_test(path, name, is_example):
         logging.warning(e)
     print('### done.', file=output_file, flush=True)
 
-    logging.debug("extracting interesting tracing data ...")
+    logging.debug("Extracting interesting tracing data ...")
     print('### extracting interesting tracing data ...', file=output_file, flush=True)
     try:
         sp.check_call('lcov --extract "%s.info.prelim" "*%s/include/**/*" --output-file %s.info'
@@ -178,7 +199,7 @@ def run_test(path, name, is_example):
     except sp.CalledProcessError as e:
         logging.warning(e)
     if is_example:
-        logging.debug("this test belongs to an example, thus also covering examples code")
+        logging.debug("This test belongs to an example, thus also covering examples code")
         try:
             sp.check_call('lcov --extract "%s.info.prelim" "*%s/examples/**/*" --output-file %s.info.example'
                           % (name, options.base_dir, name),
@@ -192,12 +213,23 @@ def run_test(path, name, is_example):
     output_file.close()
 
 
+def run_tests():
+    logging.info("Running general tests ...")
+    for test in options.tests:
+        run_test(**test)
+    if options.with_examples:
+        logging.info("Running tests for examples ...")
+        for example in options.example_tests:
+            run_test(**example)
+
+
 def aggregate_tracefiles():
-    logging.info("aggregating %d tracefiles ..." % len(options.tracefiles))
+    logging.info("Aggregating %d tracefiles ..." % len(options.tracefiles))
     output_file = open('%s/aggegrating.log' % (options.coverage_dir,), mode='a')
+    logging.debug("Output log: %s" % output_file.name)
     options.final_tracefile = "%s/all_tests.info" % options.coverage_dir
     for tracefile in options.tracefiles:
-        logging.debug("adding tracefile: %s" % (tracefile))
+        logging.debug("- %s" % (tracefile))
         print("### adding tracefile: %s" % (tracefile,), file=output_file, flush=True)
         if os.access(options.final_tracefile, os.W_OK):
             sp.check_call('lcov --add-tracefile "%s" --add-tracefile "%s" --output-file "%s"'
@@ -212,21 +244,21 @@ def aggregate_tracefiles():
 
 
 def generate_html():
-    logging.info("generating HTML report ...")
+    logging.info("Generating HTML report ...")
     output_file = open('%s/generate_html.log' % (options.coverage_dir,), mode='a')
     sp.check_call('genhtml --output-directory %s --demangle-cpp --num-spaces 2 --sort '
                   '--title "PFASST++ Test Coverage" --prefix "%s" --function-coverage --legend "%s"'
                   % (options.coverage_dir, options.base_dir, options.final_tracefile),
                   shell=True, stdout=output_file, stderr=output_file)
     output_file.close()
-    logging.info("coverage report can be found in: %s" % options.coverage_dir)
+    logging.info("Coverage report can be found in: file://%s/index.html" % options.coverage_dir)
 
 
 if __name__ == "__main__":
-    assert(is_lcov_available())
+    if not is_lcov_available():
+        raise RuntimeError("Required commands could not be found.")
     setup_and_init_options()
     get_test_directories()
-    for test in options.tests:
-        run_test(**test)
+    run_tests()
     aggregate_tracefiles()
     generate_html()