[HOTFIX] CI config changes

 * CI coverage is missing doctests
 * manual workaround implemented
 * related work

Author: reactive-firewall

.github/workflows/CI-MATs.yml

@@ -50,7 +50,7 @@ jobs:
           fi
       - id: get_env
         run: |
-          ENV_VALUE=$(gh api "${{ github.event.workflow_run.artifacts_url }}" --jq '.environment')
+          ENV_VALUE=$(gh api "${{ github.event.workflow_run.check_suite_url }}" --jq '.check_suite.pull_requests[0].head.repo.environment')
           if [[ -n "$ENV_VALUE" ]]; then
             echo "environment=$ENV_VALUE" >> "$GITHUB_OUTPUT"
           else

Makefile

@@ -95,6 +95,11 @@ ifndef PYTEST
 	PYTEST=$(PYTHON) -m pytest --cache-clear --junitxml=test-reports/junit.xml -v --rootdir=.
 endif
 
+ifndef DOCTEST_ARGS
+	# Define common doctest flags
+	DOCTEST_ARGS := --doctest-glob=**/*.py --doctest-modules
+endif
+
 ifndef PIP_COMMON_FLAGS
 	# Define common pip install flags
 	PIP_COMMON_FLAGS := --use-pep517 --exists-action s --upgrade --upgrade-strategy eager
@@ -252,9 +257,21 @@ test-mat: test-mat-build test-mat-bootstrap test-mat-basic test-mat-say test-mat
 	$(QUIET)$(WAIT) ;
 	$(QUIET)$(DO_FAIL) ;
 
+test-mat-doctests: MANIFEST.in ## Run doctests MAT category (doctests are special)
+	$(QUIET)if [ -n "$$TESTS_USE_PYTEST" ]; then \
+		$(ECHO) "SKIP: The target '$@' is not compatable with pytests;"; \
+		$(ECHO) "Try 'make test-mat-doctests' instead."; \
+	else \
+		$(COVERAGE) run -p --source=multicast -m tests.run_selective --group mat --category doctests || DO_FAIL="exit 2" ; \
+		$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; \
+		$(COVERAGE) xml --include=* 2>$(ERROR_LOG_PATH) || : ; \
+	fi
+	$(QUIET)$(WAIT) ;
+	$(QUIET)$(DO_FAIL) ;
+
 test-mat-%: MANIFEST.in ## Run specific MAT category (basic|doctests|say|hear|usage|build|bootstrap)
 	$(QUIET)if [ -n "$$TESTS_USE_PYTEST" ]; then \
-		$(PYTEST) tests/ --verbose $(COVERAGE_ARGS) --junitxml=test-reports/junit.xml -v --rootdir=. -m "mat and $*" || DO_FAIL="exit 2" ; \
+		$(PYTEST) $(COVERAGE_ARGS) -m "mat and $*" tests/ || DO_FAIL="exit 2" ; \
 	else \
 		$(COVERAGE) run -p --source=multicast -m tests.run_selective --group mat --category $*; \
 	fi
@@ -263,7 +280,7 @@ test-mat-%: MANIFEST.in ## Run specific MAT category (basic|doctests|say|hear|us
 
 test-extra: ## Run all extra tests
 	$(QUIET)if [ -n "$$TESTS_USE_PYTEST" ]; then \
-		$(PYTEST) tests/ --verbose $(COVERAGE_ARGS) -m "extra" || DO_FAIL="exit 2" ; \
+		$(PYTEST) $(COVERAGE_ARGS) -m "extra" tests/ || DO_FAIL="exit 2" ; \
 	else \
 		$(COVERAGE) run -p --source=multicast -m tests.run_selective --group extra; \
 	fi

pyproject.toml

@@ -24,21 +24,27 @@ docstring-convention = "google"
 testpaths = "tests"
 python_files = "test_*.py"
 
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = "tests"
+python_files = "test_*.py"
+addopts = "--cache-clear --doctest-glob=**/*.py --doctest-modules"
+
 [tool.pytest.markers]
 mat = "minimum acceptance tests"
-mat_basic = "basic component tests"
-mat_doctests = "documentation tests"
-mat_say = "send.py focused tests"
-mat_hear = "recv.py and hear.py focused tests"
-mat_usage = "__main__.py and API tests"
-mat_build = "build and packaging tests"
-mat_bootstrap = "init/exceptions/env/skt tests"
+basic = "basic component tests"
+doctests = "documentation tests"
+say = "send.py focused tests"
+hear = "recv.py and hear.py focused tests"
+usage = "__main__.py and API tests"
+build = "build and packaging tests"
+bootstrap = "init/exceptions/env/skt tests"
 extra = "additional important tests"
-extra_coverage = "coverage-focused tests"
-extra_linting = "linting tests"
-extra_security = "security tests"
+coverage = "coverage-focused tests"
+linting = "linting tests"
+security = "security tests"
 fuzzing = "fuzzing tests"
-fuzzing_basic = "smaller sub-set of fuzzing tests"
+slow = "smaller sub-set of fuzzing tests"
 performance = "performance and scalability tests"
 
 [tool.pytest.enabler.flake8]

pytest.ini

@@ -1,20 +1,23 @@
 [pytest]
-addopts = --doctest-modules --cov=multicast --cov-append --cov-report=xml
+addopts = --cache-clear --doctest-glob=**/*.py --doctest-modules --cov=multicast --cov-append --cov-report=xml --rootdir=.
 testpaths = tests
+pythonpath = multicast tests
 python_files = test_*.py
+python_classes = *TestSuite
 markers =
     mat: minimum acceptance tests
-    mat_basic: basic component tests
-    mat_doctests: documentation tests
-    mat_say: send.py focused tests
-    mat_hear: recv.py and hear.py focused tests
-    mat_usage: __main__.py and API tests
-    mat_build: build and packaging tests
-    mat_bootstrap: init/exceptions/env/skt tests
+    basic: basic component tests
+    doctests: documentation tests
+    say: send.py focused tests
+    hear: recv.py and hear.py focused tests
+    usage: __main__.py and API tests
+    build: build and packaging tests
+    bootstrap: init/exceptions/env/skt tests
     extra: additional important tests
-    extra_coverage: coverage-focused tests
-    extra_linting: linting tests
-    extra_security: security tests
+    coverage: coverage-focused tests
+    linting: linting tests
+    security: security tests
     fuzzing: fuzzing tests
-    fuzzing_basic: smaller sub-set of fuzzing tests
+    slow: smaller sub-set of fuzzing tests
     performance: performance and scalability tests
+junit_logging = all

tests/__init__.py

@@ -279,13 +279,13 @@ def loadDocstringsFromModule(module: types.ModuleType) -> TestSuite:
 
 try:
 	FUZZING_TESTS = {
-		"basic": [
+		"slow": [
 			test_fuzz.HypothesisTestSuite,  # Assuming this exists
 		],
 		# Future fuzzing test categories to be added
 	}
 except Exception:
-	FUZZING_TESTS = {"basic": []}
+	FUZZING_TESTS = {"slow": []}
 
 PERFORMANCE_TESTS = {
 	"scalability": [],  # Future implementation

tests/requirements.txt

@@ -53,6 +53,8 @@ pep8>=1.0
 pytest>=7.0, !=8.1.0
 # pytest-checkdocs - MIT license
 pytest-checkdocs>=1.2.5
+# pytest-doctestplus - BSD license
+pytest-doctestplus>=1.4.0
 # pytest-cov - MIT license
 pytest-cov>=3.0.0
 # pytest-enabler - MIT license

tests/test_basic.py

@@ -35,7 +35,7 @@
 	raise ImportError("[CWE-758] Failed to import test context") from _cause
 
 
-@context.markWithMetaTag("mat", "mat_basic")
+@context.markWithMetaTag("mat", "basic")
 class BasicTestSuite(context.BasicUsageTestSuite):
 	"""
 	A test suite containing basic test cases for the multicast module.

tests/test_build.py

@@ -56,7 +56,7 @@
 	raise ImportError("[CWE-758] Failed to import test context") from _cause
 
 
-@context.markWithMetaTag("mat", "mat_build")
+@context.markWithMetaTag("mat", "build")
 class BuildPEP517TestSuite(BasicUsageTestSuite):
 
 	__module__ = "tests.test_build"

tests/test_deps.py

@@ -46,7 +46,7 @@
 	raise ImportError("[CWE-440] venv Failed to import.") from _cause
 
 
-@context.markWithMetaTag("extra", "extra_coverage")
+@context.markWithMetaTag("extra", "coverage")
 class BuildRequirementsTxtTestSuite(context.BasicUsageTestSuite):
 	"""Test cases for 'tests/requirements.txt'."""
 

tests/test_exceptions.py

@@ -46,7 +46,7 @@
 	raise ImportError("[CWE-758] Failed to import test context") from _cause
 
 
-@context.markWithMetaTag("mat", "mat_bootstrap")
+@context.markWithMetaTag("mat", "bootstrap")
 class ExceptionsTestSuite(BasicUsageTestSuite):
 	"""
 	Test suite for validating the behavior of exception classes in the multicast package.