test: using pytest as a test runner, diversify the tests into unit, integration, and performance tests

.pre-commit-config.yaml

@@ -87,7 +87,7 @@ repos:
     language: python
     files: ^src/package/|^tests/
     types: [text, python]
-    args: [--config-file, pyproject.toml]
+    args: [--explicit-package-bases, --config-file, pyproject.toml]
 
 # Check for potential security issues.
 - repo: https://github.com/PyCQA/bandit
@@ -160,14 +160,14 @@ repos:
   hooks:
   - id: actionlint
 
-# On push to the remote, run the unit tests. Note that the `COVERAGE_CORE` variable is
-# required for Python 3.12+ to make sure Coverage uses the new Python monitoring module.
+# On push to the remote, run all tests. Note that the `COVERAGE_CORE` variable is required
+# for Python 3.12+ to make sure Coverage uses the new Python monitoring module.
 # See also: https://blog.trailofbits.com/2025/05/01/making-pypis-test-suite-81-faster/#optimizing-coverage-with-python-312s-sysmonitoring
 - repo: local
   hooks:
   - id: pytest
-    name: Run unit tests
-    entry: env COVERAGE_CORE=sysmon pytest -c pyproject.toml --cov-config pyproject.toml src/package/ tests/ docs/
+    name: Run all tests
+    entry: env COVERAGE_CORE=sysmon pytest --config-file pyproject.toml --cov-config pyproject.toml src/package/ tests/ docs/
     language: python
     verbose: true
     always_run: true

Makefile

@@ -168,9 +168,15 @@ check:
 
 # Run all unit tests. The --files option avoids stashing but passes files; however,
 # the hook setup itself does not pass files to pytest (see .pre-commit-config.yaml).
-.PHONY: test
+.PHONY: test test-unit test-integration test-performance
 test:
 	pre-commit run pytest --hook-stage push --files tests/
+test-unit:
+	PYTEST_ADDOPTS="-m 'not integration and not performance'" pre-commit run pytest --hook-stage push --files tests/
+test-integration:
+	PYTEST_ADDOPTS="-m integration" pre-commit run pytest --hook-stage push --files tests/
+test-performance:
+	PYTEST_ADDOPTS="-m performance" pre-commit run pytest --hook-stage push --files tests/
 
 # Build a source distribution package and a binary wheel distribution artifact.
 # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH

pyproject.toml

@@ -65,6 +65,7 @@ test = [
     "faker ==37.6.0",
     "hypothesis >=6.21.0,<6.138.17",
     "pytest >=7.2.0,<9.0.0",
+    "pytest-benchmark ==5.2.0",
     "pytest-cases ==3.9.1",
     "pytest-custom_exit_code ==0.3.0",
     "pytest-cov ==6.3.0",  # Uses: coverage[toml] >=7.5
@@ -258,7 +259,7 @@ max-line-length = 120
 # https://github.com/yashtodi94/pytest-custom_exit_code
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = """-vv -ra --tb native --durations 0 \
+addopts = """-vv -ra --tb native --durations 0 --strict-markers --import-mode importlib \
     --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose \
     --doctest-modules --doctest-continue-on-failure --doctest-glob '*.rst' --doctest-plus \
     --suppress-no-test-exit-code \
@@ -281,3 +282,7 @@ filterwarnings = [
     "error::pytest.PytestUnraisableExceptionWarning",
     "error::pytest.PytestUnhandledThreadExceptionWarning",
 ]
+markers = [
+    "integration",
+    "performance",
+]

tests/integration/test_something.py

@@ -0,0 +1,16 @@
+"""Test the Package as an integration."""
+
+# https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b404-import-subprocess
+import subprocess  # nosec B404
+
+import pytest
+
+
+@pytest.mark.integration
+def test_package() -> None:
+    """Test the Something command."""
+    # For testing we disable this warning here:
+    # https://bandit.readthedocs.io/en/latest/plugins/b603_subprocess_without_shell_equals_true.html
+    # https://bandit.readthedocs.io/en/latest/plugins/b607_start_process_with_partial_path.html
+    completed = subprocess.run(["something"], check=True, shell=False)  # nosec B603, B607
+    assert completed.returncode == 0

tests/performance/test_something.py

@@ -0,0 +1,12 @@
+"""Test the performance of various package parts, or the package as a while."""
+
+import pytest
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from package.something import Something
+
+
+@pytest.mark.performance
+def test_something(benchmark: BenchmarkFixture) -> None:
+    """Test performance of the function."""
+    benchmark.pedantic(Something.do_something, iterations=10, rounds=100)  # type: ignore[no-untyped-call]

tests/test_something.py → tests/unit/test_something.py

@@ -1,4 +1,4 @@
-"""Test the Something module. Add more tests here, as needed."""
+"""Test the Something module as a unit test. Add more tests here, as needed."""
 
 import faker
 from hypothesis import given, strategies