diff --git a/Changelog.md b/Changelog.md index 0790a463..3c190132 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented here. ## [unreleased] - Update Python and Jupyter test names to follow `[file] class.funcname` format (#605) - Update r tester test_name formatting and add r tester tests (#606) +- Update R tester to support MarkUs metadata (#615) ## [v2.7.0] - Update python, pyta and jupyter testers to allow a requirements file (#580) diff --git a/server/autotest_server/testers/r/lib/r_tester.R b/server/autotest_server/testers/r/lib/r_tester.R index a3e80065..546f50d0 100644 --- a/server/autotest_server/testers/r/lib/r_tester.R +++ b/server/autotest_server/testers/r/lib/r_tester.R @@ -3,6 +3,10 @@ library(testthat) library(rjson) args <- commandArgs(TRUE) test_results <- testthat::test_file(args[1], reporter = testthat::ListReporter) +annotations <- list() +tags <- list() +overall_comments <- list() + for (i in 1:length(test_results)) { for (j in 1:length(test_results[[i]]$results)) { result <- test_results[[i]]$results[[j]] @@ -15,8 +19,28 @@ for (i in 1:length(test_results)) { if (!is.null(test_results[[i]]$results[[j]]$trace)) { test_results[[i]]$results[[j]]$trace <- format(test_results[[i]]$results[[j]]$trace) } + + # Check result for MarkUs metadata + if ("markus_tag" %in% names(attributes(result))) { + tags <- append(tags, attr(result, "markus_tag")) + test_results[[i]]$results[[j]]$type <- "metadata" + } + if ("markus_annotation" %in% names(attributes(result))) { + annotations <- append(annotations, list(attr(result, "markus_annotation"))) + test_results[[i]]$results[[j]]$type <- "metadata" + } + if ("markus_overall_comments" %in% names(attributes(result))) { + overall_comments <- append(overall_comments, attr(result, "markus_overall_comments")) + test_results[[i]]$results[[j]]$type <- "metadata" + } } } -json <- rjson::toJSON(test_results) + +json <- rjson::toJSON(list( + test_results = test_results, + tags = tags, + annotations = annotations, + overall_comments = overall_comments +)) sink() cat(json) diff --git a/server/autotest_server/testers/r/r_tester.py b/server/autotest_server/testers/r/r_tester.py index 3fa29bf5..cd45349b 100644 --- a/server/autotest_server/testers/r/r_tester.py +++ b/server/autotest_server/testers/r/r_tester.py @@ -51,6 +51,10 @@ def run(self): successes = 0 error = False for result in self.result: + # Skip results that were only used to specify MarkUs metadata + if result["type"] == "metadata": + continue + # Only add message if not a success, as testthat reports failure messages only if result["type"] != "expectation_success": messages.append(result["message"]) @@ -89,6 +93,9 @@ def __init__( This tester will create tests of type test_class. """ super().__init__(specs, test_class, resource_settings=resource_settings) + self.annotations = [] + self.overall_comments = [] + self.tags = set() def run_r_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]: """ @@ -108,7 +115,11 @@ def run_r_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]: if not results.get(test_file): results[test_file] = [] if proc.returncode == 0: - results[test_file].extend(json.loads(proc.stdout)) + test_data = json.loads(proc.stdout) + results[test_file].extend(test_data.get("test_results", [])) + self.annotations.extend(test_data.get("annotations", [])) + self.tags.update(test_data.get("tags", [])) + self.overall_comments.extend(test_data.get("overall_comments", [])) else: raise TestError(proc.stderr) return results @@ -126,3 +137,12 @@ def run(self) -> None: for res in result: test = self.test_class(self, test_file, res) print(test.run(), flush=True) + + def after_tester_run(self) -> None: + """Print all MarkUs metadata from the tests.""" + if self.annotations: + print(self.test_class.format_annotations(self.annotations)) + if self.tags: + print(self.test_class.format_tags(self.tags)) + if self.overall_comments: + print(self.test_class.format_overall_comment(self.overall_comments, separator="\n\n")) diff --git a/server/autotest_server/tests/testers/r/test_r_tester.py b/server/autotest_server/tests/testers/r/test_r_tester.py index 0aa4cb26..bec5e022 100644 --- a/server/autotest_server/tests/testers/r/test_r_tester.py +++ b/server/autotest_server/tests/testers/r/test_r_tester.py @@ -9,13 +9,15 @@ def test_success_with_context(request, monkeypatch): monkeypatch.chdir(request.fspath.dirname) # Mock R test results - simulates what R would return as JSON - mock_r_output = [ - { - "context": "Basic arithmetic", - "test": "addition works correctly", - "results": [{"type": "expectation_success", "message": ""}], - } - ] + mock_r_output = { + "test_results": [ + { + "context": "Basic arithmetic", + "test": "addition works correctly", + "results": [{"type": "expectation_success", "message": ""}], + } + ] + } # Mock subprocess.run to return our simulated R output mock_process = MagicMock()