diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index ba125e3124..76a8afbb01 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -74,14 +74,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: ${{ runner.os }} / Python ${{ matrix.python-version }} COVERALLS_PARALLEL: true - # Use cp workaround to publish coverage reports with relative paths - # FIXME: Consider refactoring the tests to not require the test - # aggregation script being invoked from the `tests` directory, so - # that `.coverage` is written to and .coveragrc can also reside in - # the project root directory as is the convention. run: | - cp tests/.coverage . - coveralls --service=github --rcfile=tests/.coveragerc + coveralls --service=github coveralls-fin: # Always run when all 'tests' jobs have finished even if they failed diff --git a/.github/workflows/specification-version-check.yml b/.github/workflows/specification-version-check.yml index 09bb87b0da..58da796824 100644 --- a/.github/workflows/specification-version-check.yml +++ b/.github/workflows/specification-version-check.yml @@ -20,7 +20,6 @@ jobs: python-version: "3.x" - id: get-version run: | - python3 -m pip install -e . script="from tuf.api.metadata import SPECIFICATION_VERSION; \ print(f\"v{'.'.join(SPECIFICATION_VERSION)}\")" ver=$(python3 -c "$script") diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index be6830fb42..bf571950c3 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -39,21 +39,18 @@ you need debug/run outside ``tox``. Unit tests ---------- -More specifically, the Update Framework's test suite can be executed by invoking -the test aggregation script inside the *tests* subdirectory. ``tuf`` and its -dependencies must already be installed. +test suite can be executed directly as well (in this case the environment managed by tox is +not used): :: - cd tests/ - python3 aggregate_tests.py + python3 -m unittest Individual tests can also be executed. Optional ``-v`` flags can be added to increase log level up to DEBUG (``-vvvv``). :: - cd tests/ - python3 test_updater_ng.py -v + python3 tests/test_updater_ng.py -v Coverage @@ -64,8 +61,7 @@ invoked with the ``coverage`` tool (requires installation of ``coverage``, e.g. via PyPI). :: - cd tests/ - coverage run aggregate_tests.py && coverage report + coverage run -m unittest Auto-formatting diff --git a/pyproject.toml b/pyproject.toml index f4e19f4236..fae68878a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,11 +70,6 @@ include = [ "/setup.py", ] -[tool.hatch.build.targets.wheel] -# The testing phase changes the current working directory to `tests` but the test scripts import -# from `tests` so the root directory must be added to Python's path for editable installations -dev-mode-dirs = ["."] - # Ruff section # Read more here: https://docs.astral.sh/ruff/linter/#rule-selection [tool.ruff] @@ -158,4 +153,7 @@ exclude_also = [ "raise AssertionError", # imports for mypy only "if TYPE_CHECKING", -] \ No newline at end of file +] +[tool.coverage.run] +branch = true +omit = [ "tests/*" ] diff --git a/requirements/dev.txt b/requirements/dev.txt index dae95c1439..6b81f9bc22 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,8 +1,3 @@ -# Install tuf in editable mode and requirements for local testing with tox, -# and also for running test suite or individual tests manually. -# The build and tox versions specified here are also used as constraints -# during CI and CD Github workflows -r build.txt -r test.txt -r lint.txt --e . diff --git a/tests/.coveragerc b/tests/.coveragerc deleted file mode 100644 index 1fa2203580..0000000000 --- a/tests/.coveragerc +++ /dev/null @@ -1,13 +0,0 @@ -[run] -branch = True - -omit = - */tests/* - */site-packages/* - -[report] -exclude_lines = - pragma: no cover - def __str__ - if __name__ == .__main__.: - @abstractmethod diff --git a/tests/aggregate_tests.py b/tests/aggregate_tests.py deleted file mode 100755 index 835ffd10ba..0000000000 --- a/tests/aggregate_tests.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2013 - 2017, New York University and the TUF contributors -# SPDX-License-Identifier: MIT OR Apache-2.0 - -""" - - aggregate_tests.py - - - Konstantin Andrianov. - Zane Fisher. - - - January 26, 2013. - - August 2013. - Modified previous behavior that explicitly imported individual - unit tests. -Zane Fisher - - - See LICENSE-MIT OR LICENSE for licensing information. - - - Run all the unit tests from every .py file beginning with "test_" in - 'tuf/tests'. Use --random to run the tests in random order. -""" - -import sys -import unittest - -if __name__ == "__main__": - suite = unittest.TestLoader().discover(".") - all_tests_passed = ( - unittest.TextTestRunner(verbosity=1, buffer=True) - .run(suite) - .wasSuccessful() - ) - - if not all_tests_passed: - sys.exit(1) - - else: - sys.exit(0) diff --git a/tests/generated_data/generate_md.py b/tests/generated_data/generate_md.py index 6a820fa154..c7cabeec78 100644 --- a/tests/generated_data/generate_md.py +++ b/tests/generated_data/generate_md.py @@ -56,9 +56,6 @@ signers.append(CryptoSigner(private_key, key)) EXPIRY = datetime(2050, 1, 1, tzinfo=timezone.utc) -OUT_DIR = "generated_data/ed25519_metadata" -if not os.path.exists(OUT_DIR): - os.mkdir(OUT_DIR) SERIALIZER = JSONSerializer() @@ -80,15 +77,13 @@ def verify_generation(md: Metadata, path: str) -> None: ) -def generate_all_files( - dump: bool | None = False, verify: bool | None = False -) -> None: - """Generate a new repository and optionally verify it. +def generate_all_files(dump: bool = False) -> None: + """Generate a new repository or verify that output has not changed. Args: - dump: Wheter to dump the newly generated files. - verify: Whether to verify the newly generated files with the - local staored. + dump: If True, new files are generated. If False, existing files + are compared to generated files and an exception is raised if + there are differences. """ md_root = Metadata(Root(expires=EXPIRY)) md_timestamp = Metadata(Timestamp(expires=EXPIRY)) @@ -103,12 +98,16 @@ def generate_all_files( for i, md in enumerate([md_root, md_timestamp, md_snapshot, md_targets]): assert isinstance(md, Metadata) md.sign(signers[i]) - path = os.path.join(OUT_DIR, f"{md.signed.type}_with_ed25519.json") - if verify: - verify_generation(md, path) - + path = os.path.join( + utils.TESTS_DIR, + "generated_data", + "ed25519_metadata", + f"{md.signed.type}_with_ed25519.json", + ) if dump: md.to_file(path, SERIALIZER) + else: + verify_generation(md, path) if __name__ == "__main__": diff --git a/tests/repository_data/README.md b/tests/repository_data/README.md deleted file mode 100644 index 9819e1c318..0000000000 --- a/tests/repository_data/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Unit and integration testing - -## Running the tests -The unit and integration tests can be executed by invoking `tox` from any -path under the project directory. - -``` -$ tox -``` - -Or by invoking `aggregate_tests.py` from the -[tests](https://github.com/theupdateframework/python-tuf/tree/develop/tests) -directory. - -``` -$ python3 aggregate_tests.py -``` - -Note: integration tests end in `_integration.py`. - -If you wish to run a particular unit test, navigate to the tests directory and -run that specific unit test. For example: - -``` -$ python3 test_updater.py -``` - -It it also possible to run the test cases of a unit test. For instance: - -``` -$ python3 -m unittest test_updater.TestMultiRepoUpdater.test_get_one_valid_targetinfo -``` - -## Setup -The unit and integration tests operate on static metadata available in the -[repository_data -directory](https://github.com/theupdateframework/python-tuf/tree/develop/tests/repository_data/). -Before running the tests, static metadata is first copied to temporary -directories and modified, as needed, by the tests. - -The test modules typically spawn HTTP(S) servers that serve metadata and target -files for the unit tests. The [map -file](https://github.com/theupdateframework/python-tuf/tree/develop/tests/repository_data) -specifies the location of the test repositories and other properties. For -specific targets and metadata provided by the tests repositories, please -inspect their [respective -metadata](https://github.com/theupdateframework/python-tuf/tree/develop/tests/repository_data/repository). - diff --git a/tests/test_fetcher_ng.py b/tests/test_fetcher_ng.py index c4f924867e..33e2c9227f 100644 --- a/tests/test_fetcher_ng.py +++ b/tests/test_fetcher_ng.py @@ -10,8 +10,7 @@ import sys import tempfile import unittest -from collections.abc import Iterator -from typing import Any, ClassVar +from typing import ClassVar from unittest.mock import Mock, patch import requests @@ -163,11 +162,11 @@ def test_download_file_upper_length(self) -> None: self.assertEqual(self.file_length, temp_file.tell()) # Download a file bigger than expected - def test_download_file_length_mismatch(self) -> Iterator[Any]: - with self.assertRaises(exceptions.DownloadLengthMismatchError): - # Force download_file to execute and raise the error since it is a - # context manager and returns Iterator[IO] - yield self.fetcher.download_file(self.url, self.file_length - 4) + def test_download_file_length_mismatch(self) -> None: + with self.assertRaises( + exceptions.DownloadLengthMismatchError + ), self.fetcher.download_file(self.url, self.file_length - 4): + pass # we never get here as download_file() raises # Run unit test. diff --git a/tests/test_metadata_generation.py b/tests/test_metadata_generation.py index df99819f90..03cc5ab688 100644 --- a/tests/test_metadata_generation.py +++ b/tests/test_metadata_generation.py @@ -16,7 +16,7 @@ class TestMetadataGeneration(unittest.TestCase): @staticmethod def test_compare_static_md_to_generated() -> None: # md_generator = MetadataGenerator("generated_data/ed25519_metadata") - generate_all_files(dump=False, verify=True) + generate_all_files(dump=False) # Run unit test. diff --git a/tests/test_utils.py b/tests/test_utils.py index cdb6890509..fcdc3c449b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,6 +19,7 @@ """ import logging +import os import socket import sys import unittest @@ -56,7 +57,7 @@ def test_simple_server_startup(self) -> None: def test_cleanup(self) -> None: # Test normal case server_process_handler = utils.TestServerProcess( - log=logger, server="simple_server.py" + log=logger, server=os.path.join(utils.TESTS_DIR, "simple_server.py") ) server_process_handler.clean() diff --git a/tox.ini b/tox.ini index 03dd2324e8..e10627a874 100644 --- a/tox.ini +++ b/tox.ini @@ -9,21 +9,13 @@ envlist = lint,docs,py skipsdist = true [testenv] -# TODO: Consider refactoring the tests to not require the aggregation script -# being invoked from the `tests` directory. This seems to be the convention and -# would make use of other testing tools such as coverage/coveralls easier. -changedir = tests - commands = python3 --version - python3 -m coverage run aggregate_tests.py - python3 -m coverage report --rcfile {toxinidir}/pyproject.toml -m --fail-under 97 + python3 -m coverage run -m unittest + python3 -m coverage report -m --fail-under 97 deps = -r{toxinidir}/requirements/test.txt - # Install TUF in editable mode, instead of tox default virtual environment - # installation (see `skipsdist`), to get relative paths in coverage reports - --editable {toxinidir} install_command = python3 -m pip install {opts} {packages} @@ -37,14 +29,12 @@ commands_pre = python3 -m pip install --force-reinstall git+https://github.com/secure-systems-lab/securesystemslib.git@main#egg=securesystemslib[crypto,pynacl] commands = - python3 -m coverage run aggregate_tests.py - python3 -m coverage report --rcfile {toxinidir}/pyproject.toml -m + python3 -m coverage run -m unittest + python3 -m coverage report -m [testenv:lint] -changedir = {toxinidir} deps = -r{toxinidir}/requirements/lint.txt - --editable {toxinidir} lint_dirs = tuf examples tests verify_release .github/scripts passenv = RUFF_OUTPUT_FORMAT commands = @@ -54,7 +44,6 @@ commands = mypy {[testenv:lint]lint_dirs} [testenv:fix] -changedir = {toxinidir} deps = {[testenv:lint]deps} commands = ruff check --fix {[testenv:lint]lint_dirs} @@ -64,6 +53,5 @@ commands = deps = -r{toxinidir}/requirements/docs.txt -changedir = {toxinidir} commands = sphinx-build -b html docs docs/build/html -W