Fixing Workflow unit-test execution & Adding Test-Reports functionality (#343)

* Run unit-tests instead of consumer-tests

Currently consumer-tests are run when the documentation is build.
This is not needed as the consumer tests are executed separately as well,
and the normal build should execute the unit-tests instead.

This also enables the testlinker & xml parser to look for the 'tests-report' folder to enable testing being done outside of bazel.

Closes: #328

Author: MaximilianSoerenPollak

.github/workflows/test.yml

@@ -15,6 +15,7 @@ name: Run Bazel Tests
 on:
   pull_request:
     types: [opened, reopened, synchronize]
+  workflow_call:
 jobs:
   code:
     runs-on: ubuntu-latest
@@ -39,3 +40,19 @@ jobs:
         run: |
           bazel run //:ide_support
           bazel test //src/...
+
+      - name: Prepare bundled consumer report
+        if: always()
+        # Creating tests-report directory
+        # Follow Symlinks via '-L' to copy correctly
+        # Copy everything inside the 'test-reports' folder
+        run: |
+          mkdir -p tests-report
+          rsync -amL --include='*/' --include='test.xml' --include='test.log' --exclude='*' bazel-testlogs/ tests-report/
+
+      - name: Upload bundled consumer report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: tests-report
+          path: tests-report

.github/workflows/test_and_docs.yml

@@ -38,13 +38,13 @@ jobs:
       bazel-docs-verify-target: "//:docs_check"
 
   # This is the user configurable part of the workflow
-  consumer-tests:
-    uses: ./.github/workflows/consumer_test.yml
+  unit-tests:
+    uses: ./.github/workflows/test.yml
     secrets: inherit
 
   docs-build:
     # Waits for consumer-tests but run only when docs verification succeeded
-    needs: [docs-verify, consumer-tests]
+    needs: [docs-verify, unit-tests]
     if: ${{ always() && needs.docs-verify.result == 'success' }}
     uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@main
     permissions:

docs/internals/extensions/source_code_linker.md

@@ -64,7 +64,7 @@ These tags are extracted and matched to Sphinx needs via the `source_code_link`
 
 ### ✅ TestLink: Test Result Integration
 
-TestLink scans test result XMLs from Bazel and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements.
+TestLink scans test result XMLs from Bazel (bazel-testlogs) or in the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements.
 This depends on the `attribute_plugin` in our tooling repository, find it [here](https://github.com/eclipse-score/tooling/tree/main/python_basics/score_pytest)
 #### Test Tagging Options
 

src/extensions/score_source_code_linker/tests/test_xml_parser.py

@@ -17,6 +17,7 @@
 """
 
 import xml.etree.ElementTree as ET
+from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
@@ -30,85 +31,136 @@
 
 
 # Unsure if I should make these last a session or not
+def _write_test_xml(
+    path: Path,
+    name: str,
+    result: str = "",
+    props: dict[str, str] | None = None,
+    file: str = "",
+    line: int = 0,
+):
+    """Helper to create the XML structure for a test case."""
+    ts = ET.Element("testsuites")
+    suite = ET.SubElement(ts, "testsuite")
+
+    # Create testcase with attributes
+    tc_attrs = {"name": name}
+    if file:
+        tc_attrs["file"] = file
+    if line:
+        tc_attrs["line"] = str(line)
+    tc = ET.SubElement(suite, "testcase", tc_attrs)
+
+    # Add failure/skipped status
+    if result == "failed":
+        ET.SubElement(tc, "failure", {"message": "failmsg"})
+    elif result == "skipped":
+        ET.SubElement(tc, "skipped", {"message": "skipmsg"})
+
+    # Add properties if provided
+    if props:
+        props_el = ET.SubElement(tc, "properties")
+        for k, v in props.items():
+            ET.SubElement(props_el, "property", {"name": k, "value": v})
+
+    # Save to file
+    ET.ElementTree(ts).write(path, encoding="utf-8", xml_declaration=True)
+
+
 @pytest.fixture
-def tmp_xml_dirs(tmp_path: Path) -> tuple[Path, Path, Path]:
-    root: Path = tmp_path / "bazel-testlogs"
-    dir1: Path = root / "with_props"
-    dir2: Path = root / "no_props"
-    dir1.mkdir(parents=True)
-    dir2.mkdir(parents=True)
-
-    def write(file_path: Path, testcases: list[ET.Element]):
-        ts = ET.Element("testsuites")
-        suite = ET.SubElement(ts, "testsuite")
-        for tc in testcases:
-            suite.append(tc)
-        tree = ET.ElementTree(ts)
-        tree.write(file_path, encoding="utf-8", xml_declaration=True)
-
-    def make_tc(
-        name: str,
-        result: str = "",
-        props: dict[str, str] | None = None,
-        file: str = "",
-        line: int = 0,
-    ):
-        tc = ET.Element("testcase", {"name": name})
-        if file:
-            tc.set("file", file)
-        if line:
-            tc.set("line", str(line))
-        if result == "failed":
-            ET.SubElement(tc, "failure", {"message": "failmsg"})
-        elif result == "skipped":
-            ET.SubElement(tc, "skipped", {"message": "skipmsg"})
-        if props:
-            props_el = ET.SubElement(tc, "properties")
-            for k, v in props.items():
-                ET.SubElement(props_el, "property", {"name": k, "value": v})
-        return tc
-
-    # File with properties
-    tc1 = make_tc(
-        "tc_with_props",
-        result="failed",
-        props={
-            "PartiallyVerifies": "REQ1",
-            "FullyVerifies": "",
-            "TestType": "type",
-            "DerivationTechnique": "tech",
-            "Description": "desc",
-        },
-        file="path1",
-        line=10,
-    )
-    write(dir1 / "test.xml", [tc1])
-
-    # File without properties
-    # HINT: Once the assertions in xml_parser are back and active, this should allow us
-    #       to catch that the tests Need to be changed too.
-    tc2 = make_tc("tc_no_props", file="path2", line=20)
-    write(dir2 / "test.xml", [tc2])
-
-    return root, dir1, dir2
+def tmp_xml_dirs(tmp_path: Path) -> Callable[..., tuple[Path, Path, Path]]:
+    def _tmp_xml_dirs(test_folder: str = "bazel-testlogs") -> tuple[Path, Path, Path]:
+        root = tmp_path / test_folder
+        dir1, dir2 = root / "with_props", root / "no_props"
+
+        for d in (dir1, dir2):
+            d.mkdir(parents=True, exist_ok=True)
+
+        # File with properties
+        _write_test_xml(
+            dir1 / "test.xml",
+            name="tc_with_props",
+            result="failed",
+            file="path1",
+            line=10,
+            props={
+                "PartiallyVerifies": "REQ1",
+                "FullyVerifies": "",
+                "TestType": "type",
+                "DerivationTechnique": "tech",
+                "Description": "desc",
+            },
+        )
+
+        # File without properties
+        _write_test_xml(dir2 / "test.xml", name="tc_no_props", file="path2", line=20)
+
+        return root, dir1, dir2
+
+    return _tmp_xml_dirs
 
 
 @add_test_properties(
     partially_verifies=["tool_req__docs_test_link_testcase"],
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )
-def test_find_xml_files(tmp_xml_dirs: tuple[Path, Path, Path]):
-    """Ensure xml files are found as expected"""
+def test_find_xml_files(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]):
+    """Ensure xml files are found as expected if bazel-testlogs is used"""
     root: Path
     dir1: Path
     dir2: Path
-    root, dir1, dir2 = tmp_xml_dirs
+    root, dir1, dir2 = tmp_xml_dirs()
     found = xml_parser.find_xml_files(root)
     expected: set[Path] = {dir1 / "test.xml", dir2 / "test.xml"}
     assert set(found) == expected
 
 
+def test_find_xml_folder(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]):
+    """Ensure xml files are found as expected if bazel-testlogs is used"""
+    root: Path
+    root, _, _ = tmp_xml_dirs()
+    found = xml_parser.find_test_folder(base_path=root.parent)
+    assert found is not None
+    assert found == root
+
+
+def test_find_xml_folder_test_reports(
+    tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]],
+):
+    # root is the 'tests-report' folder inside tmp_path
+    root, _, _ = tmp_xml_dirs(test_folder="tests-report")
+    # We pass the PARENT of 'tests-report' as the workspace root
+    found = xml_parser.find_test_folder(base_path=root.parent)
+    assert found is not None
+    assert found == root
+
+
+def test_find_xml_files_test_reports(
+    tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]],
+):
+    """Ensure xml files are found as expected if tests-report is used"""
+    root: Path
+    dir1: Path
+    dir2: Path
+    root, dir1, dir2 = tmp_xml_dirs(test_folder="tests-report")
+    found = xml_parser.find_xml_files(dir=root)
+    assert found is not None
+    expected: set[Path] = {root / dir1 / "test.xml", root / dir2 / "test.xml"}
+    assert set(found) == expected
+
+
+def test_early_return(tmp_path: Path):
+    """
+    Ensure that if tests-report & bazel-testlogs is not found,
+    we return None for early return inside extension
+    """
+    # Move the test execution context to a 100% empty folder
+
+    found = xml_parser.find_test_folder(tmp_path)
+    assert found is None
+
+
 @add_test_properties(
     partially_verifies=["tool_req__docs_test_link_testcase"],
     test_type="requirements-based",
@@ -152,12 +204,12 @@ def test_parse_properties():
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )
-def test_read_test_xml_file(tmp_xml_dirs: tuple[Path, Path, Path]):
+def test_read_test_xml_file(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]):
     """Ensure a whole pre-defined xml file is parsed correctly"""
     _: Path
     dir1: Path
     dir2: Path
-    _, dir1, dir2 = tmp_xml_dirs
+    _, dir1, dir2 = tmp_xml_dirs()
 
     needs1, no_props1 = xml_parser.read_test_xml_file(dir1 / "test.xml")
     assert isinstance(needs1, list) and len(needs1) == 1

src/extensions/score_source_code_linker/xml_parser.py

@@ -171,16 +171,28 @@ def find_xml_files(dir: Path) -> list[Path]:
     return xml_paths
 
 
+def find_test_folder(base_path: Path | None = None) -> Path | None:
+    ws_root = base_path if base_path is not None else find_ws_root()
+    assert ws_root is not None
+    if os.path.isdir(ws_root / "tests-report"):
+        return ws_root / "tests-report"
+    if os.path.isdir(ws_root / "bazel-testlogs"):
+        return ws_root / "bazel-testlogs"
+    logger.info("could not find tests-report or bazel-testlogs to parse testcases")
+    return None
+
+
 def run_xml_parser(app: Sphinx, env: BuildEnvironment):
     """
     This is the 'main' function for parsing test.xml's and
     building testcase needs.
     It gets called from the source_code_linker __init__
     """
-    ws_root = find_ws_root()
-    assert ws_root is not None
-    bazel_testlogs = ws_root / "bazel-testlogs"
-    xml_file_paths = find_xml_files(bazel_testlogs)
+    testlogs_dir = find_test_folder()
+    # early return
+    if testlogs_dir is None:
+        return
+    xml_file_paths = find_xml_files(testlogs_dir)
     test_case_needs = build_test_needs_from_files(app, env, xml_file_paths)
     # Saving the test case needs for cache
     store_data_of_test_case_json(