Finalize how we will handle no coverage for new projects in the PTB

* Switch COVERAGE_FILE to COVERAGE_DB

Author: ArBridgeman

doc/user_guide/getting_started.rst

@@ -204,7 +204,14 @@ We also need to configure settings for github-pages environment:
 8. Set up for Sonar
 +++++++++++++++++++
 PTB supports using SonarQube Cloud to analyze, visualize, & track linting, security, &
-coverage. In order to set it up, you'll need to do the following instructions.
+coverage. All of our Python projects are evaluated against the
+`Exasol Way <https://sonarcloud.io/organizations/exasol/quality_gates/show/AXxvLH-3BdtLlpiYmZhh>`__
+and subscribe to the
+`Clean as You Code <https://docs.sonarsource.com/sonarqube-server/9.8/user-guide/clean-as-you-code/>`__
+methodology, which means that SonarQube analysis will fail and, if its included in the branch protections, block a PR
+if code modified in that PR does not meet the standards of the Exasol Way.
+
+In order to set up Sonar, you will need to perform the following instructions.
 
 For a **public** project
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -223,6 +230,11 @@ For a **public** project
         hostUrl = "https://sonarcloud.io"
         organization = "exasol"
 6. Post-merge, update the branch protections to include SonarQube analysis
+  * This should only be done when tests exist for the project, & that the project is
+    at a state in which enforced code coverage would not be a burden. For new projects,
+    we recommend creating an issue to add the SonarQube analysis to the branch protections
+    at a later point. In such scenarios, SonarQube analysis will still report its analysis
+    results to the PR, but it will not prevent the PR from being merged.
 
 For a **private** project
 ^^^^^^^^^^^^^^^^^^^^^^^^^

exasol/toolbox/nox/_artifacts.py

@@ -18,13 +18,13 @@
     Config,
 )
 
-COVERAGE_FILE = ".coverage"
+COVERAGE_DB = ".coverage"
 COVERAGE_XML = "ci-coverage.xml"
 LINT_JSON = ".lint.json"
 LINT_TXT = ".lint.txt"
 SECURITY_JSON = ".security.json"
 
-ALL_LINT_FILES = {COVERAGE_FILE, LINT_JSON, LINT_TXT, SECURITY_JSON}
+ALL_LINT_FILES = {COVERAGE_DB, LINT_JSON, LINT_TXT, SECURITY_JSON}
 COVERAGE_TABLES = {"coverage_schema", "meta", "file", "line_bits"}
 LINT_JSON_ATTRIBUTES = {
     "type",
@@ -55,7 +55,7 @@ def check_artifacts(session: Session) -> None:
         _is_valid_lint_txt(Path(PROJECT_CONFIG.root, LINT_TXT)),
         _is_valid_lint_json(Path(PROJECT_CONFIG.root, LINT_JSON)),
         _is_valid_security_json(Path(PROJECT_CONFIG.root, SECURITY_JSON)),
-        _is_valid_coverage(Path(PROJECT_CONFIG.root, COVERAGE_FILE)),
+        _is_valid_coverage(Path(PROJECT_CONFIG.root, COVERAGE_DB)),
     ]
     if not all(all_is_valid_checks):
         sys.exit(1)
@@ -146,7 +146,7 @@ def copy_artifacts(session: Session) -> None:
 
     artifact_dir = Path(session.posargs[0])
     suffix = _python_version_suffix()
-    _combine_coverage(session, artifact_dir, f"coverage{suffix}*/{COVERAGE_FILE}")
+    _combine_coverage(session, artifact_dir, f"coverage{suffix}*/{COVERAGE_DB}")
     _copy_artifacts(
         artifact_dir,
         artifact_dir.parent,
@@ -189,14 +189,15 @@ def _prepare_coverage_xml(session: Session, source: Path) -> None:
     Prepare the coverage XML for input into Sonar
 
     The coverage XML is used within Sonar to determine the overall coverage in our
-    project, as well as, the coverage associated with changes made in a PR. If the
-    coverage in our PR does not meet the set Sonar guidelines for that project,
-    then when the sonarqubecloud bot writes in the PR, it will indicate that the
-    coverage constraint was not met & block the PR from being merged. Thus, in the
-    preparation of the coverage XML, we set `--fail-under=0` so that this sub-step
-    will not fail prior to sending the data to Sonar. Otherwise, this sub-step would
-    fail whenever the `fail_under` in `[tool.coverage.report]` of the `pyproject.toml`
-    was not met.
+    project and the coverage associated with source code changes made in a PR. If the
+    code coverage associated with changes made in our PR does not meet the set Sonar
+    guidelines for that project, then when the sonarqubecloud bot writes in the PR, it
+    will indicate that the coverage constraint was not met, &, if the bot is required in
+    the branch protections for that project, then it will block the PR from being
+    merged. Thus, in the preparation of the coverage XML, we set `--fail-under=0` so
+    that this sub-step will not fail prior to sending the data to Sonar. Otherwise, this
+    sub-step would fail whenever the `fail_under` in `[tool.coverage.report]` of the
+    `pyproject.toml` was not met.
     """
     command = [
         "coverage",
@@ -207,7 +208,19 @@ def _prepare_coverage_xml(session: Session, source: Path) -> None:
         f"{source}/*",
         "--fail-under=0",
     ]
-    session.run(*command)
+    output = subprocess.run(command, capture_output=True, text=True, cwd=source)
+    if output.returncode != 0:
+        if output.stdout.strip() == "No data to report.":
+            # Assuming that previous steps passed in the CI, this indicates — as
+            # is in the case for newly created projects — that no coverage over the
+            # `source` files was found. In this scenario, we do not need the CI to
+            # break from a non-zero exit code. This is because without a coverage file
+            # the sonarqubecloud bot will still report that the coverage is 0%, but as
+            # long as the bot is not required in the project's branch protections, then
+            # the PR can still be merged, provided that the other required constraints
+            # have been met.
+            return
+        session.error(output.returncode, output.stdout, output.stderr)
 
 
 def _upload_to_sonar(
@@ -226,9 +239,8 @@ def _upload_to_sonar(
         "--sonar-sources",
         config.source,
     ]
-    if Path("dummy").exists():
+    if Path(COVERAGE_XML).exists():
         command.extend(["--sonar-python-coverage-report-paths", COVERAGE_XML])
-
     session.run(*command)  # type: ignore
 
 

test/unit/nox/_artifacts_test.py

@@ -5,6 +5,7 @@
 from dataclasses import dataclass
 from inspect import cleandoc
 from pathlib import Path
+from typing import Any
 from unittest import mock
 from unittest.mock import (
     Mock,
@@ -13,10 +14,21 @@
 )
 
 import pytest
+from nox import (
+    Session,
+    _options,
+    manifest,
+    virtualenv,
+)
+from nox.sessions import (
+    SessionRunner,
+    _SessionQuit,
+)
 
+from exasol.toolbox.nox import _artifacts
 from exasol.toolbox.nox._artifacts import (
     ALL_LINT_FILES,
-    COVERAGE_FILE,
+    COVERAGE_DB,
     COVERAGE_TABLES,
     COVERAGE_XML,
     LINT_JSON,
@@ -32,7 +44,6 @@
     check_artifacts,
     copy_artifacts,
 )
-from noxconfig import PROJECT_CONFIG
 
 
 @contextlib.contextmanager
@@ -229,7 +240,7 @@ def test_missing_attributes(self, tmp_path, capsys, missing_attributes):
 
 class TestIsValidCoverage:
     def test_passes_when_as_expected(self, tmp_path):
-        path = Path(tmp_path, COVERAGE_FILE)
+        path = Path(tmp_path, COVERAGE_DB)
         _create_coverage_file(path, COVERAGE_TABLES)
 
         result = _is_valid_coverage(path)
@@ -244,7 +255,7 @@ def test_passes_when_as_expected(self, tmp_path):
     )
     def test_database_missing_tables(self, tmp_path, capsys, missing_table):
         tables = COVERAGE_TABLES - missing_table
-        path = Path(tmp_path, COVERAGE_FILE)
+        path = Path(tmp_path, COVERAGE_DB)
         _create_coverage_file(path, tables)
 
         result = _is_valid_coverage(path)
@@ -308,21 +319,66 @@ def test_all_files(tmp_path, capsys):
             assert (tmp_path / f).exists()
 
 
-# class TestPrepareCoverageXml:
-#     @staticmethod
-#     def test_no_coverage_file_creates_dummy():
-#         _prepare_coverage_xml(Mock(), PROJECT_CONFIG.source)
-#
-#         assert Path(COVERAGE_XML).exists()
-#         assert Path(COVERAGE_XML).read_text() == ""
-#         assert not Path(COVERAGE_FILE).exists()
-#
-#     @staticmethod
-#     def test_that_bad_coverage_file_still_raises_error(capsys):
-#         _create_coverage_file(Path(COVERAGE_FILE), COVERAGE_TABLES)
-#         session_mock = Mock()
-#
-#         _prepare_coverage_xml(session_mock, PROJECT_CONFIG.source)
-#
-#         assert Path(COVERAGE_FILE).exists()
-#         assert not Path(COVERAGE_XML).exists()
+class FakeEnv(mock.MagicMock):
+    # Extracted from nox testing
+    _get_env = virtualenv.VirtualEnv._get_env
+
+
+def make_fake_env(venv_backend: str = "venv", **kwargs: Any) -> FakeEnv:
+    # Extracted from nox testing
+    return FakeEnv(
+        spec=virtualenv.VirtualEnv,
+        env={},
+        venv_backend=venv_backend,
+        **kwargs,
+    )
+
+
+@pytest.fixture
+def nox_session(tmp_path):
+    # Extracted from nox testing
+    session_runner = SessionRunner(
+        name="test",
+        signatures=["test"],
+        func=mock.Mock(spec=["python"], python="3.10"),
+        global_config=_options.options.namespace(
+            posargs=[],
+            error_on_external_run=False,
+            install_only=False,
+            invoked_from=tmp_path,
+        ),
+        manifest=mock.create_autospec(manifest.Manifest),
+    )
+    session_runner.venv = make_fake_env(bin_paths=["/no/bin/for/you"])
+    session = Session(session_runner)
+    yield session
+
+
+class TestPrepareCoverageXml:
+    @staticmethod
+    def test_no_coverage_file_silently_passes(monkeypatch, tmp_path, nox_session):
+        coverage_xml = tmp_path / COVERAGE_XML
+        coverage_db = tmp_path / COVERAGE_DB
+        monkeypatch.setattr(_artifacts, "COVERAGE_XML", coverage_xml)
+
+        _prepare_coverage_xml(nox_session, tmp_path)
+
+        assert not Path(coverage_db).exists()
+        assert not Path(coverage_xml).exists()
+
+    @staticmethod
+    def test_that_bad_coverage_file_still_raises_error(
+        monkeypatch, tmp_path, nox_session
+    ):
+        coverage_xml = tmp_path / COVERAGE_XML
+        coverage_db = tmp_path / COVERAGE_DB
+        monkeypatch.setattr(_artifacts, "COVERAGE_XML", coverage_xml)
+        _create_coverage_file(coverage_db, COVERAGE_TABLES)
+
+        with pytest.raises(
+            _SessionQuit, match="doesn't seem to be a coverage data file"
+        ):
+            _prepare_coverage_xml(nox_session, tmp_path)
+
+        assert coverage_db.exists()
+        assert not coverage_xml.exists()