SCANPY-80 Read ".coveragerc" configuration to exclude files from coverage (#234)

Author: maksim-grebeniuk-sonarsource

CLI_ARGS.md

@@ -51,6 +51,7 @@
 | `--skip-jre-provisioning`, `-Dsonar.scanner.skipJreProvisioning` | If provided, the provisioning of the JRE will be skipped |
 | `--sonar-branch-name`, `-Dsonar.branch.name` | Name of the branch being analyzed |
 | `--sonar-build-string`, `-Dsonar.buildString` | The string passed with this property will be stored with the analysis and available in the results of api/project_analyses/search, thus allowing you to later identify a specific analysis and obtain its key for use with api/new_code_periods/set on the SPECIFIC_ANALYSIS type |
+| `--sonar-coverage-exclusions`, `--sonar.coverage.exclusions`, `-Dsonar.coverage.exclusions` | Defines the source files to be excluded from the code coverage analysis. |
 | `--sonar-cpd-python-minimum-lines`, `-Dsonar.cpd.python.minimumLines` | Minimum number of tokens to be considered as a duplicated block of code |
 | `--sonar-cpd-python-minimum-tokens`, `-Dsonar.cpd.python.minimumTokens` | Minimum number of tokens to be considered as a duplicated block of code |
 | `--sonar-links-ci`, `-Dsonar.links.ci` | The URL of the continuous integration system used |

src/pysonar_scanner/configuration/cli.py

@@ -469,6 +469,13 @@ def __create_parser(cls):
             type=str,
             help="Comma-delimited list of paths to coverage reports in the Cobertura XML format.",
         )
+        reports_group.add_argument(
+            "--sonar-coverage-exclusions",
+            "--sonar.coverage.exclusions",
+            "-Dsonar.coverage.exclusions",
+            type=str,
+            help="Defines the source files to be excluded from the code coverage analysis.",
+        )
         reports_group.add_argument(
             "-Dsonar.python.skipUnchanged",
             type=bool,

src/pysonar_scanner/configuration/configuration_loader.py

@@ -22,6 +22,7 @@
 from typing import Any
 
 from pysonar_scanner.configuration.cli import CliConfigurationLoader
+from pysonar_scanner.configuration.coveragerc_loader import CoverageRCConfigurationLoader
 from pysonar_scanner.configuration.pyproject_toml import TomlConfigurationLoader
 from pysonar_scanner.configuration.properties import SONAR_PROJECT_KEY, SONAR_TOKEN, SONAR_PROJECT_BASE_DIR, Key
 from pysonar_scanner.configuration.properties import PROPERTIES
@@ -50,9 +51,11 @@ def load() -> dict[Key, Any]:
         toml_path_property = cli_properties.get("toml-path", ".")
         toml_dir = Path(toml_path_property) if "toml-path" in cli_properties else base_dir
         toml_properties = TomlConfigurationLoader.load(toml_dir)
+        coverage_properties = CoverageRCConfigurationLoader.load_exclusion_properties(base_dir)
 
         resolved_properties = get_static_default_properties()
         resolved_properties.update(dynamic_defaults_loader.load())
+        resolved_properties.update(coverage_properties)
         resolved_properties.update(toml_properties.project_properties)
         resolved_properties.update(sonar_project_properties.load(base_dir))
         resolved_properties.update(toml_properties.sonar_properties)

src/pysonar_scanner/configuration/coveragerc_loader.py

@@ -0,0 +1,70 @@
+#
+# Sonar Scanner Python
+# Copyright (C) 2011-2024 SonarSource SA.
+# mailto:info AT sonarsource DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+import configparser
+import logging
+import pathlib
+from typing import Any
+
+
+class CoverageRCConfigurationLoader:
+
+    @staticmethod
+    def load_exclusion_properties(base_dir: pathlib.Path) -> dict[str, str]:
+        config_file_path = base_dir / ".coveragerc"
+        coverage_properties = CoverageRCConfigurationLoader.__read_config(config_file_path)
+        if len(coverage_properties) == 0:
+            return {}
+        translated_exclusions = CoverageRCConfigurationLoader.__extract_coverage_exclusion_patterns(
+            config_file_path, coverage_properties
+        )
+        if len(translated_exclusions) == 0:
+            return {}
+
+        return {"sonar.coverage.exclusions": ", ".join(translated_exclusions)}
+
+    @staticmethod
+    def __read_config(config_file_path: pathlib.Path) -> dict[str, Any]:
+        config_dict: dict[str, Any] = {}
+        if not config_file_path.exists():
+            logging.debug(f"Coverage file not found: {config_file_path}")
+            return config_dict
+
+        try:
+            config_parser = configparser.ConfigParser()
+            config_parser.read(config_file_path)
+            for section in config_parser.sections():
+                section_values = {}
+                for key, value in config_parser.items(section):
+                    section_values[key] = value
+                config_dict[section] = section_values
+        except Exception as e:
+            logging.debug(f"Error decoding coverage file {config_file_path}: {e}")
+        return config_dict
+
+    @staticmethod
+    def __extract_coverage_exclusion_patterns(
+        config_file_path: pathlib.Path, coverage_properties: dict[str, Any]
+    ) -> list[str]:
+        if "run" not in coverage_properties or "omit" not in coverage_properties["run"]:
+            logging.debug(f"Coverage file {config_file_path} has no exclusion properties")
+            return []
+
+        omit_exclusions = coverage_properties["run"]["omit"]
+        return [patterns.strip() for patterns in omit_exclusions.splitlines() if patterns.strip()]

src/pysonar_scanner/configuration/properties.py

@@ -85,6 +85,7 @@
 SONAR_PYTHON_VERSION: Key = "sonar.python.version"
 SONAR_PYTHON_PYLINT_REPORT_PATH: Key = "sonar.python.pylint.reportPath"
 SONAR_PYTHON_COVERAGE_REPORT_PATHS: Key = "sonar.python.coverage.reportPaths"
+SONAR_COVERAGE_EXCLUSIONS: Key = "sonar.coverage.exclusions"
 SONAR_PYTHON_SKIP_UNCHANGED: Key = "sonar.python.skipUnchanged"
 SONAR_NEWCODE_REFERENCE_BRANCH: Key = "sonar.newCode.referenceBranch"
 SONAR_SCM_REVISION: Key = "sonar.scm.revision"
@@ -490,6 +491,11 @@ def env_variable_name(self) -> str:
         default_value=None,
         cli_getter=lambda args: args.sonar_python_coverage_report_paths
     ),
+    Property(
+        name=SONAR_COVERAGE_EXCLUSIONS,
+        default_value=None,
+        cli_getter=lambda args: args.sonar_coverage_exclusions
+    ),
     Property(
         name=SONAR_PYTHON_SKIP_UNCHANGED,
         default_value=None,

tests/unit/test_configuration_cli.py

@@ -88,6 +88,7 @@
     SONAR_SCM_FORCE_RELOAD_ALL,
     SONAR_PYTHON_PYLINT_REPORT_PATH,
     SONAR_PYTHON_COVERAGE_REPORT_PATHS,
+    SONAR_COVERAGE_EXCLUSIONS,
     SONAR_PYTHON_SKIP_UNCHANGED,
     SONAR_PYTHON_XUNIT_REPORT_PATH,
     SONAR_PYTHON_XUNIT_SKIP_DETAILS,
@@ -156,6 +157,7 @@
     SONAR_SCM_FORCE_RELOAD_ALL: True,
     SONAR_PYTHON_PYLINT_REPORT_PATH: "path/to/pylint/report",
     SONAR_PYTHON_COVERAGE_REPORT_PATHS: "path/to/coverage1,path/to/coverage2",
+    SONAR_COVERAGE_EXCLUSIONS: "*/.local/*,/usr/*,utils/tirefire.py",
     SONAR_PYTHON_SKIP_UNCHANGED: True,
     SONAR_PYTHON_XUNIT_REPORT_PATH: "path/to/xunit/report",
     SONAR_PYTHON_XUNIT_SKIP_DETAILS: True,
@@ -384,6 +386,8 @@ def test_impossible_os_choice(self):
             "path/to/pylint/report",
             "--sonar-python-coverage-report-paths",
             "path/to/coverage1,path/to/coverage2",
+            "--sonar-coverage-exclusions",
+            "*/.local/*,/usr/*,utils/tirefire.py",
             "--sonar-python-skip-unchanged",
             "--sonar-python-xunit-report-path",
             "path/to/xunit/report",
@@ -469,6 +473,7 @@ def test_all_cli_args(self):
             "-Dsonar.log.level=INFO",
             "-Dsonar.python.pylint.reportPath=path/to/pylint/report",
             "-Dsonar.python.coverage.reportPaths=path/to/coverage1,path/to/coverage2",
+            "-Dsonar.coverage.exclusions=*/.local/*,/usr/*,utils/tirefire.py",
             "-Dsonar.python.skipUnchanged=true",
             "-Dsonar.python.xunit.reportPath=path/to/xunit/report",
             "-Dsonar.python.xunit.skipDetails=true",

tests/unit/test_configuration_loader.py

@@ -51,6 +51,7 @@
     SONAR_SCANNER_JAVA_OPTS,
     SONAR_SCANNER_ARCH,
     SONAR_SCANNER_OS,
+    SONAR_COVERAGE_EXCLUSIONS,
 )
 from pysonar_scanner.utils import Arch, Os
 from pysonar_scanner.configuration.configuration_loader import ConfigurationLoader, SONAR_PROJECT_BASE_DIR
@@ -307,6 +308,45 @@ def test_load_pyproject_toml_from_toml_path(self, mock_get_os, mock_get_arch):
         }
         self.assertDictEqual(configuration, expected_configuration)
 
+    @patch(
+        "sys.argv",
+        [
+            "myscript.py",
+        ],
+    )
+    def test_load_coveragerc_properties(self, mock_get_os, mock_get_arch):
+        self.fs.create_dir("custom/path")
+        self.fs.create_file(
+            ".coveragerc",
+            contents=(
+                """
+                [run]
+                omit =
+                    */.local/*
+                    /usr/*
+                    utils/tirefire.py
+                """
+            ),
+        )
+        configuration = ConfigurationLoader.load()
+        expected_configuration = {
+            SONAR_SCANNER_APP: "python",
+            SONAR_SCANNER_APP_VERSION: "1.0",
+            SONAR_SCANNER_BOOTSTRAP_START_TIME: configuration[SONAR_SCANNER_BOOTSTRAP_START_TIME],
+            SONAR_VERBOSE: False,
+            SONAR_SCANNER_SKIP_JRE_PROVISIONING: False,
+            SONAR_PROJECT_BASE_DIR: os.getcwd(),
+            SONAR_SCANNER_CONNECT_TIMEOUT: 5,
+            SONAR_SCANNER_SOCKET_TIMEOUT: 60,
+            SONAR_SCANNER_RESPONSE_TIMEOUT: 0,
+            SONAR_SCANNER_KEYSTORE_PASSWORD: "changeit",
+            SONAR_SCANNER_TRUSTSTORE_PASSWORD: "changeit",
+            SONAR_SCANNER_OS: Os.LINUX.value,
+            SONAR_SCANNER_ARCH: Arch.X64.value,
+            SONAR_COVERAGE_EXCLUSIONS: "*/.local/*, /usr/*, utils/tirefire.py",
+        }
+        self.assertDictEqual(configuration, expected_configuration)
+
     @patch("sys.argv", ["myscript.py"])
     @patch.dict("os.environ", {"SONAR_TOKEN": "TokenFromEnv", "SONAR_PROJECT_KEY": "KeyFromEnv"}, clear=True)
     def test_load_from_env_variables_only(self, mock_get_os, mock_get_arch):

tests/unit/test_coveragerc_loader.py

@@ -0,0 +1,87 @@
+#
+# Sonar Scanner Python
+# Copyright (C) 2011-2024 SonarSource SA.
+# mailto:info AT sonarsource DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+from pathlib import Path
+from unittest.mock import patch
+
+from pyfakefs.fake_filesystem_unittest import TestCase
+
+from pysonar_scanner.configuration.coveragerc_loader import CoverageRCConfigurationLoader
+
+
+class TestCoverageRcFile(TestCase):
+    def setUp(self):
+        self.setUpPyfakefs()
+
+    def test_load_coverage_file(self):
+        self.fs.create_file(
+            ".coveragerc",
+            contents="""
+            [run]
+            omit =
+                */.local/*
+                /usr/*
+                utils/tirefire.py
+            """,
+        )
+        properties = CoverageRCConfigurationLoader.load_exclusion_properties(Path("."))
+
+        self.assertEqual(properties["sonar.coverage.exclusions"], "*/.local/*, /usr/*, utils/tirefire.py")
+
+    @patch("pysonar_scanner.configuration.coveragerc_loader.logging")
+    def test_load_missing_file(self, mock_logging):
+        properties = CoverageRCConfigurationLoader.load_exclusion_properties(Path("."))
+        self.assertEqual(len(properties), 0)
+        mock_logging.debug.assert_called_with("Coverage file not found: .coveragerc")
+
+    @patch("pysonar_scanner.configuration.coveragerc_loader.logging")
+    def test_load_without_run_section(self, mock_logging):
+        self.fs.create_file(
+            ".coveragerc",
+            contents="""
+                    [something_else]
+                    """,
+        )
+        properties = CoverageRCConfigurationLoader.load_exclusion_properties(Path("."))
+        self.assertEqual(len(properties), 0)
+        mock_logging.debug.assert_called_with("Coverage file .coveragerc has no exclusion properties")
+
+    @patch("pysonar_scanner.configuration.coveragerc_loader.logging")
+    def test_load_without_exclusions_property(self, mock_logging):
+        self.fs.create_file(
+            ".coveragerc",
+            contents="""
+                    [run]
+                    """,
+        )
+        properties = CoverageRCConfigurationLoader.load_exclusion_properties(Path("."))
+        self.assertEqual(len(properties), 0)
+        mock_logging.debug.assert_called_with("Coverage file .coveragerc has no exclusion properties")
+
+    @patch("pysonar_scanner.configuration.coveragerc_loader.logging")
+    def test_load_malformed_file(self, mock_logging):
+        self.fs.create_file(
+            ".coveragerc",
+            contents="""
+                    [run
+                    omit = 
+                    """,
+        )
+        properties = CoverageRCConfigurationLoader.load_exclusion_properties(Path("."))
+        self.assertEqual(len(properties), 0)