Merge pull request #8154 from lorjala/unittests-coveragefiltering

Unit tests: add code coverage filtering

Author: 0xc0170

UNITTESTS/mbed_unittest.py

@@ -23,12 +23,14 @@
 from __future__ import print_function
 import os
 import logging
+import re
 
 from unit_test.options import get_options_parser, \
                               pretty_print_test_options
 from unit_test.settings import DEFAULT_CMAKE_GENERATORS
 from unit_test.test import UnitTestTool
 from unit_test.new import UnitTestGeneratorTool
+from unit_test.coverage import CoverageAPI
 
 def _mbed_unittest_test(options, cwd, pwd):
     if options is None:
@@ -80,14 +82,28 @@ def _mbed_unittest_test(options, cwd, pwd):
         tool.build_tests()
 
     if options.run_only:
+        tool.run_tests(filter_regex=options.test_regex)
+
         # If code coverage generation:
         if options.coverage:
-            # Run tests and generate reports
-            tool.generate_coverage_report(coverage_output_type=options.coverage,
-                                          excludes=[pwd, options.build],
-                                          build_path=options.build)
-        else:
-            tool.run_tests(filter_regex=options.test_regex) # Only run tests
+            cov_api = CoverageAPI(
+                mbed_os_root=os.path.normpath(os.path.join(pwd, "..")),
+                build_dir=options.build)
+
+            # Generate reports
+            outputs = [options.coverage]
+            if options.coverage == "both":
+                outputs = ["html", "xml"]
+
+            excludes = [pwd, options.build]
+
+            if not options.include_headers:
+                excludes.append(re.compile(".*\\.h"))
+
+            cov_api.generate_reports(outputs=outputs,
+                                     excludes=excludes,
+                                     filter_regex=options.test_regex,
+                                     build_path=options.build)
 
 def _mbed_unittest_new(options, pwd):
     if options is None:

UNITTESTS/unit_test/coverage.py

@@ -0,0 +1,148 @@
+"""
+Copyright (c) 2018, Arm Limited
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+GENERATE TEST CODE COVERAGE
+"""
+
+import os
+import logging
+import posixpath
+import re
+
+from .utils import execute_program
+from .get_tools import get_gcov_program, \
+    get_gcovr_program
+from .settings import COVERAGE_OUTPUT_TYPES
+
+
+class CoverageAPI(object):
+    """
+    Generate code coverage reports for unit tests.
+    """
+
+    def __init__(self, mbed_os_root=None, build_dir=None):
+        self.root = mbed_os_root
+
+        if not self.root:
+            self.root = os.path.normpath(os.path.join(
+                os.path.dirname(os.path.realpath(__file__)),
+                "../.."))
+
+        self.build_dir = build_dir
+
+        if not self.build_dir:
+            logging.error("No build directory given for CoverageAPI.")
+
+    def _gen_cmd(self, coverage_type, excludes, filter_regex):
+        # Generate coverage generation command:
+        args = [get_gcovr_program(),
+                "--gcov-executable",
+                get_gcov_program(),
+                "-r",
+                os.path.relpath(self.root, self.build_dir),
+                "."]
+
+        if coverage_type == "html":
+            args.extend(["--html",
+                         "--html-detail",
+                         "-o",
+                         "./coverage/index.html"])
+        elif coverage_type == "xml":
+            args.extend(["-x",
+                         "-o",
+                         "./coverage.xml"])
+        else:
+            logging.error("Invalid coverage output type: %s" % coverage_type)
+
+        # Add exclude filters:
+        for path in excludes:
+            # Use posix separators if path is string
+            if isinstance(path, ("".__class__, u"".__class__)):
+                path = path.replace("\\", "/")
+                args.extend(["-e", path])
+            # Use regular expressions as is
+            elif isinstance(path, type(re.compile(""))):
+                args.extend(["-e", path.pattern])
+
+        # Add include filters:
+        if filter_regex:
+            filters = filter_regex.split(",")
+
+            for filt in filters:
+                regex = "(.+/)?%s" % filt.replace("-", "/")
+                args.extend(["-f", regex])
+
+        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+            args.append("-v")
+
+        return args
+
+    def generate_reports(self,
+                         outputs,
+                         excludes=None,
+                         filter_regex=None,
+                         build_path=None):
+        """
+        Run tests to generate coverage data, and generate coverage reports.
+
+        Positional arguments:
+        outputs - list of types to generate
+
+        Keyword arguments:
+        excludes - list of paths to exclude from the report
+        filter_regex - comma-separated string to use for test filtering
+        build_path - build path
+        """
+
+        # Check for the tool
+        if get_gcovr_program() is None:
+            logging.error(
+                "No gcovr tool found in path. " +
+                "Cannot generate coverage reports.")
+            return
+
+        if build_path is None:
+            build_path = os.getcwd()
+
+        if outputs is None:
+            outputs = []
+
+        if excludes is None:
+            excludes = []
+
+        for output in outputs:
+            # Skip if invalid/unsupported output type
+            if output not in COVERAGE_OUTPUT_TYPES:
+                logging.warning(
+                    "Invalid output type. " +
+                    "Skip coverage report for type: %s." % output.upper())
+                continue
+
+            if output == "html":
+                # Create a build directory if not exist
+                coverage_path = os.path.join(build_path, "coverage")
+                if not os.path.exists(coverage_path):
+                    os.mkdir(coverage_path)
+
+            # Generate the command
+            args = self._gen_cmd(output, excludes, filter_regex)
+
+            # Run the coverage tool
+            execute_program(
+                args,
+                "%s code coverage report generation failed." % output.upper(),
+                "%s code coverage report created." % output.upper())

UNITTESTS/unit_test/options.py

@@ -21,7 +21,7 @@
 import argparse
 import logging
 
-from .settings import CMAKE_GENERATORS, MAKE_PROGRAMS, COVERAGE_TYPES
+from .settings import CMAKE_GENERATORS, MAKE_PROGRAMS, COVERAGE_ARGS
 from .get_tools import get_make_tool
 
 def get_options_parser():
@@ -71,10 +71,15 @@ def get_options_parser():
                         dest="debug_build")
 
     parser.add_argument("--coverage",
-                        choices=COVERAGE_TYPES,
+                        choices=COVERAGE_ARGS,
                         help="Generate code coverage report",
                         dest="coverage")
 
+    parser.add_argument("--include-headers",
+                        action="store_true",
+                        help="Include headers to coverage reports, defaults to false.",
+                        dest="include_headers")
+
     parser.add_argument("-m",
                         "--make-program",
                         default=get_make_tool(),
@@ -140,3 +145,4 @@ def pretty_print_test_options(options=None):
     if options.coverage:
         logging.info(" [%s]  \tGenerate coverage reports", "SET")
         logging.info(" \t\t - Output: %s", options.coverage)
+        logging.info(" \t\t - Include headers: %s", options.include_headers)

UNITTESTS/unit_test/settings.py

@@ -29,10 +29,12 @@
     "ninja": "Ninja"
 }
 
-COVERAGE_TYPES = ["html",
+COVERAGE_ARGS = ["html",
                   "xml",
                   "both"]
 
+COVERAGE_OUTPUT_TYPES = ["html", "xml"]
+
 CXX_COMPILERS = ["g++-6", "g++-8", "g++-7", "g++-5", "g++-4.9", "g++"]
 
 C_COMPILERS = ["gcc-6", "gcc-8", "gcc-7", "gcc-5", "gcc-4.9", "gcc"]

UNITTESTS/unit_test/test.py

@@ -27,9 +27,7 @@
 from .get_tools import get_make_tool, \
                        get_cmake_tool, \
                        get_cxx_tool, \
-                       get_c_tool, \
-                       get_gcov_program, \
-                       get_gcovr_program
+                       get_c_tool
 from .settings import DEFAULT_CMAKE_GENERATORS
 
 class UnitTestTool(object):
@@ -115,80 +113,6 @@ def build_tests(self):
                         "Building unit tests failed.",
                         "Unit tests built successfully.")
 
-    def _get_coverage_script(self, coverage_type, excludes):
-        args = [get_gcovr_program(),
-                "--gcov-executable",
-                get_gcov_program(),
-                "-r",
-                "../..",
-                "."]
-
-        if coverage_type == "html":
-            args.extend(["--html",
-                         "--html-detail",
-                         "-o",
-                         "./coverage/index.html"])
-        elif coverage_type == "xml":
-            args.extend(["-x",
-                         "-o",
-                         "./coverage.xml"])
-
-        for path in excludes:
-            args.extend(["-e", path.replace("\\", "/")])
-
-        #Exclude header files from report
-        args.extend(["-e", ".*\.h"])
-
-        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
-            args.append("-v")
-
-        return args
-
-    def generate_coverage_report(self,
-                                 coverage_output_type=None,
-                                 excludes=None,
-                                 build_path=None):
-        """
-        Run tests to generate coverage data, and generate coverage reports.
-        """
-
-        self.run_tests()
-
-        if get_gcovr_program() is None:
-            logging.error("No gcovr tool found in path. \
-            Cannot generate coverage report.")
-            return
-
-        if build_path is None:
-            build_path = os.getcwd()
-
-        if coverage_output_type is None:
-            logging.warning("No coverage output type give. \
-                            Cannot generate coverage reports.")
-            return
-
-        if excludes is None:
-            excludes = []
-
-        if coverage_output_type == "html" or coverage_output_type == "both":
-            # Create build directory if not exist.
-            coverage_path = os.path.join(build_path, "coverage")
-            if not os.path.exists(coverage_path):
-                os.mkdir(coverage_path)
-
-            args = self._get_coverage_script("html", excludes)
-
-            execute_program(args,
-                            "HTML code coverage report generation failed.",
-                            "HTML code coverage report created.")
-
-        if coverage_output_type == "xml" or coverage_output_type == "both":
-            args = self._get_coverage_script("xml", excludes)
-
-            execute_program(args,
-                            "XML code coverage report generation failed.",
-                            "XML code coverage report created.")
-
     def run_tests(self, filter_regex=None):
         """
         Run unit tests.