Updated R tester to support MarkUs test metadata

Author: david-yz-liu

Changelog.md

@@ -4,6 +4,7 @@ All notable changes to this project will be documented here.
 ## [unreleased]
 - Update Python and Jupyter test names to follow `[file] class.funcname` format (#605)
 - Update r tester test_name formatting and add r tester tests (#606)
+- Update R tester to support MarkUs metadata (#615)
 
 ## [v2.7.0]
 - Update python, pyta and jupyter testers to allow a requirements file (#580)

server/autotest_server/testers/r/lib/r_tester.R

@@ -3,6 +3,10 @@ library(testthat)
 library(rjson)
 args <- commandArgs(TRUE)
 test_results <- testthat::test_file(args[1], reporter = testthat::ListReporter)
+annotations <- list()
+tags <- list()
+overall_comments <- list()
+
 for (i in 1:length(test_results)) {
   for (j in 1:length(test_results[[i]]$results)) {
     result <- test_results[[i]]$results[[j]]
@@ -15,8 +19,28 @@ for (i in 1:length(test_results)) {
     if (!is.null(test_results[[i]]$results[[j]]$trace)) {
       test_results[[i]]$results[[j]]$trace <- format(test_results[[i]]$results[[j]]$trace)
     }
+
+    # Check result for MarkUs metadata
+    if ("markus_tag" %in% names(attributes(result))) {
+      tags <- append(tags, attr(result, "markus_tag"))
+      test_results[[i]]$results[[j]]$type <- "metadata"
+    }
+    if ("markus_annotation" %in% names(attributes(result))) {
+      annotations <- append(annotations, list(attr(result, "markus_annotation")))
+      test_results[[i]]$results[[j]]$type <- "metadata"
+    }
+    if ("markus_overall_comments" %in% names(attributes(result))) {
+      overall_comments <- append(overall_comments, attr(result, "markus_overall_comments"))
+      test_results[[i]]$results[[j]]$type <- "metadata"
+    }
   }
 }
-json <- rjson::toJSON(test_results)
+
+json <- rjson::toJSON(list(
+  test_results = test_results,
+  tags = tags,
+  annotations = annotations,
+  overall_comments = overall_comments
+))
 sink()
 cat(json)

server/autotest_server/testers/r/r_tester.py

@@ -51,6 +51,10 @@ def run(self):
         successes = 0
         error = False
         for result in self.result:
+            # Skip results that were only used to specify MarkUs metadata
+            if result["type"] == "metadata":
+                continue
+
             # Only add message if not a success, as testthat reports failure messages only
             if result["type"] != "expectation_success":
                 messages.append(result["message"])
@@ -89,6 +93,9 @@ def __init__(
         This tester will create tests of type test_class.
         """
         super().__init__(specs, test_class, resource_settings=resource_settings)
+        self.annotations = []
+        self.overall_comments = []
+        self.tags = set()
 
     def run_r_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]:
         """
@@ -108,7 +115,11 @@ def run_r_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]:
             if not results.get(test_file):
                 results[test_file] = []
             if proc.returncode == 0:
-                results[test_file].extend(json.loads(proc.stdout))
+                test_data = json.loads(proc.stdout)
+                results[test_file].extend(test_data.get("test_results", []))
+                self.annotations.extend(test_data.get("annotations", []))
+                self.tags.update(test_data.get("tags", []))
+                self.overall_comments.extend(test_data.get("overall_comments", []))
             else:
                 raise TestError(proc.stderr)
         return results
@@ -122,7 +133,25 @@ def run(self) -> None:
             results = self.run_r_tests()
         except subprocess.CalledProcessError as e:
             raise TestError(e.stderr) from e
-        for test_file, result in results.items():
-            for res in result:
-                test = self.test_class(self, test_file, res)
+        for test_file, test_results in results.items():
+            for result in test_results:
+                test = self.test_class(self, test_file, result)
                 print(test.run(), flush=True)
+
+    def after_tester_run(self) -> None:
+        """Print all MarkUs metadata from the tests."""
+        if self.annotations:
+            import sys
+
+            print(self.annotations, file=sys.stderr)
+            print(self.test_class.format_annotations(self.annotations))
+        if self.tags:
+            import sys
+
+            print(self.tags, file=sys.stderr)
+            print(self.test_class.format_tags(self.tags))
+        if self.overall_comments:
+            import sys
+
+            print(self.overall_comments, file=sys.stderr)
+            print(self.test_class.format_overall_comment(self.overall_comments, separator="\n\n"))

server/autotest_server/tests/testers/r/test_r_tester.py

@@ -9,13 +9,15 @@ def test_success_with_context(request, monkeypatch):
     monkeypatch.chdir(request.fspath.dirname)
 
     # Mock R test results - simulates what R would return as JSON
-    mock_r_output = [
-        {
-            "context": "Basic arithmetic",
-            "test": "addition works correctly",
-            "results": [{"type": "expectation_success", "message": ""}],
-        }
-    ]
+    mock_r_output = {
+        "test_results": [
+            {
+                "context": "Basic arithmetic",
+                "test": "addition works correctly",
+                "results": [{"type": "expectation_success", "message": ""}],
+            }
+        ]
+    }
 
     # Mock subprocess.run to return our simulated R output
     mock_process = MagicMock()