Merge pull request #2749 from jku/test-fixes

Unit test infrastructure fixes

Author: kairoaraujo

.github/workflows/_test.yml

@@ -74,14 +74,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_FLAG_NAME: ${{ runner.os }} / Python ${{ matrix.python-version }}
           COVERALLS_PARALLEL: true
-        # Use cp workaround to publish coverage reports with relative paths
-        # FIXME: Consider refactoring the tests to not require the test
-        # aggregation script being invoked from the `tests` directory, so
-        # that `.coverage` is written to and .coveragrc can also reside in
-        # the project root directory as is the convention.
         run: |
-          cp tests/.coverage .
-          coveralls --service=github --rcfile=tests/.coveragerc
+          coveralls --service=github
 
   coveralls-fin:
     # Always run when all 'tests' jobs have finished even if they failed

.github/workflows/specification-version-check.yml

@@ -20,7 +20,6 @@ jobs:
           python-version: "3.x"
       - id: get-version
         run: |
-          python3 -m pip install -e .
           script="from tuf.api.metadata import SPECIFICATION_VERSION; \
                   print(f\"v{'.'.join(SPECIFICATION_VERSION)}\")"
           ver=$(python3 -c "$script")

docs/CONTRIBUTING.rst

@@ -39,21 +39,18 @@ you need debug/run outside ``tox``.
 Unit tests
 ----------
 
-More specifically, the Update Framework's test suite can be executed by invoking
-the test aggregation script inside the *tests* subdirectory. ``tuf`` and its
-dependencies must already be installed.
+test suite can be executed directly as well (in this case the environment managed by tox is
+not used):
 ::
 
-     cd tests/
-     python3 aggregate_tests.py
+     python3 -m unittest
 
 
 Individual tests can also be executed. Optional ``-v`` flags can be added to
 increase log level up to DEBUG (``-vvvv``).
 ::
 
-     cd tests/
-     python3 test_updater_ng.py -v
+     python3 tests/test_updater_ng.py -v
 
 
 Coverage
@@ -64,8 +61,7 @@ invoked with the ``coverage`` tool (requires installation of ``coverage``, e.g.
 via PyPI).
 ::
 
-     cd tests/
-     coverage run aggregate_tests.py && coverage report
+     coverage run -m unittest
 
 
 Auto-formatting

pyproject.toml

@@ -70,11 +70,6 @@ include = [
   "/setup.py",
 ]
 
-[tool.hatch.build.targets.wheel]
-# The testing phase changes the current working directory to `tests` but the test scripts import
-# from `tests` so the root directory must be added to Python's path for editable installations
-dev-mode-dirs = ["."]
-
 # Ruff section
 # Read more here: https://docs.astral.sh/ruff/linter/#rule-selection
 [tool.ruff]
@@ -158,4 +153,7 @@ exclude_also = [
     "raise AssertionError",
     # imports for mypy only
     "if TYPE_CHECKING",
-]
+]
+[tool.coverage.run]
+branch = true
+omit = [ "tests/*" ]

requirements/dev.txt

@@ -1,8 +1,3 @@
-# Install tuf in editable mode and requirements for local testing with tox,
-# and also for running test suite or individual tests manually.
-# The build and tox versions specified here are also used as constraints
-# during CI and CD Github workflows
 -r build.txt
 -r test.txt
 -r lint.txt
--e .

tests/.coveragerc

@@ -56,9 +56,6 @@
     signers.append(CryptoSigner(private_key, key))
 
 EXPIRY = datetime(2050, 1, 1, tzinfo=timezone.utc)
-OUT_DIR = "generated_data/ed25519_metadata"
-if not os.path.exists(OUT_DIR):
-    os.mkdir(OUT_DIR)
 
 SERIALIZER = JSONSerializer()
 
@@ -80,15 +77,13 @@ def verify_generation(md: Metadata, path: str) -> None:
             )
 
 
-def generate_all_files(
-    dump: bool | None = False, verify: bool | None = False
-) -> None:
-    """Generate a new repository and optionally verify it.
+def generate_all_files(dump: bool = False) -> None:
+    """Generate a new repository or verify that output has not changed.
 
     Args:
-        dump: Wheter to dump the newly generated files.
-        verify: Whether to verify the newly generated files with the
-            local staored.
+        dump: If True, new files are generated. If False, existing files
+           are compared to generated files and an exception is raised if
+           there are differences.
     """
     md_root = Metadata(Root(expires=EXPIRY))
     md_timestamp = Metadata(Timestamp(expires=EXPIRY))
@@ -103,12 +98,16 @@ def generate_all_files(
     for i, md in enumerate([md_root, md_timestamp, md_snapshot, md_targets]):
         assert isinstance(md, Metadata)
         md.sign(signers[i])
-        path = os.path.join(OUT_DIR, f"{md.signed.type}_with_ed25519.json")
-        if verify:
-            verify_generation(md, path)
-
+        path = os.path.join(
+            utils.TESTS_DIR,
+            "generated_data",
+            "ed25519_metadata",
+            f"{md.signed.type}_with_ed25519.json",
+        )
         if dump:
             md.to_file(path, SERIALIZER)
+        else:
+            verify_generation(md, path)
 
 
 if __name__ == "__main__":

tests/aggregate_tests.py

@@ -10,8 +10,7 @@
 import sys
 import tempfile
 import unittest
-from collections.abc import Iterator
-from typing import Any, ClassVar
+from typing import ClassVar
 from unittest.mock import Mock, patch
 
 import requests
@@ -163,11 +162,11 @@ def test_download_file_upper_length(self) -> None:
             self.assertEqual(self.file_length, temp_file.tell())
 
     # Download a file bigger than expected
-    def test_download_file_length_mismatch(self) -> Iterator[Any]:
-        with self.assertRaises(exceptions.DownloadLengthMismatchError):
-            # Force download_file to execute and raise the error since it is a
-            # context manager and returns Iterator[IO]
-            yield self.fetcher.download_file(self.url, self.file_length - 4)
+    def test_download_file_length_mismatch(self) -> None:
+        with self.assertRaises(
+            exceptions.DownloadLengthMismatchError
+        ), self.fetcher.download_file(self.url, self.file_length - 4):
+            pass  # we never get here as download_file() raises
 
 
 # Run unit test.