Emit CTRF output on test when --json is passed

DEPENDENCIES

@@ -3,3 +3,4 @@ core https://github.com/sourcemeta/core e4d7ae9358710fc138d2afd3179db6d850e4190f
 jsonbinpack https://github.com/sourcemeta/jsonbinpack 8fae212dc7ec02af4bb0cd4e7fccd42a2471f1c1
 blaze https://github.com/sourcemeta/blaze 8dba65f8aebfe1ac976168b76e01c20dd406c517
 hydra https://github.com/sourcemeta/hydra af9f2c54709d620872ead0c3f8f683c15a0fa702
+ctrf https://github.com/ctrf-io/ctrf 93ea827d951390190171d37443bff169cf47c808

docs/test.markdown

@@ -21,6 +21,10 @@ suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite).
 **If you want to validate that a schema adheres to its metaschema, use the
 [`metaschema`](./metaschema.markdown) command instead.**
 
+Pass `--json` to output results in [CTRF (Common Test Report
+Format)](https://ctrf.io), a standardized JSON format for test results that
+integrates with CI/CD tools and test result dashboards.
+
 Writing tests
 -------------
 

src/command_test.cc

@@ -1,65 +1,90 @@
 #include <sourcemeta/blaze/output.h>
 #include <sourcemeta/blaze/test.h>
 
+#include <sourcemeta/core/json.h>
+
+#include <chrono>   // std::chrono
 #include <cstdlib>  // EXIT_FAILURE
 #include <iostream> // std::cerr, std::cout
+#include <sstream>  // std::ostringstream
 #include <string>   // std::string
+#include <thread>   // std::this_thread
+#include <vector>   // std::vector
 
 #include "command.h"
 #include "configuration.h"
+#include "configure.h"
 #include "error.h"
 #include "input.h"
 #include "logger.h"
 #include "resolver.h"
 #include "utils.h"
 
-auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
-    -> void {
-  bool result{true};
-
-  const auto verbose{options.contains("verbose")};
+namespace {
 
-  for (const auto &entry : for_each_json(options)) {
-    const auto configuration_path{find_configuration(entry.first)};
-    const auto &configuration{read_configuration(options, configuration_path)};
-    const auto dialect{default_dialect(options, configuration)};
-
-    const auto &schema_resolver{
-        resolver(options, options.contains("http"), dialect, configuration)};
-
-    std::optional<sourcemeta::blaze::TestSuite> test_suite;
-    try {
-      test_suite.emplace(sourcemeta::blaze::TestSuite::parse(
-          entry.second, entry.positions, entry.first.parent_path(),
-          schema_resolver, sourcemeta::core::schema_walker,
-          sourcemeta::blaze::default_schema_compiler, dialect));
-    } catch (const sourcemeta::blaze::TestParseError &error) {
+auto parse_test_suite(const sourcemeta::jsonschema::InputJSON &entry,
+                      const sourcemeta::core::SchemaResolver &schema_resolver,
+                      const std::optional<std::string> &dialect,
+                      const bool json_output) -> sourcemeta::blaze::TestSuite {
+  try {
+    return sourcemeta::blaze::TestSuite::parse(
+        entry.second, entry.positions, entry.first.parent_path(),
+        schema_resolver, sourcemeta::core::schema_walker,
+        sourcemeta::blaze::default_schema_compiler, dialect);
+  } catch (const sourcemeta::blaze::TestParseError &error) {
+    if (!json_output) {
       std::cout << entry.first.string() << ":\n";
-      throw FileError<sourcemeta::blaze::TestParseError>{
-          entry.first, error.what(), error.location(), error.line(),
-          error.column()};
-    } catch (const sourcemeta::core::SchemaRelativeMetaschemaResolutionError
-                 &error) {
+    }
+    throw sourcemeta::jsonschema::FileError<sourcemeta::blaze::TestParseError>{
+        entry.first, error.what(), error.location(), error.line(),
+        error.column()};
+  } catch (
+      const sourcemeta::core::SchemaRelativeMetaschemaResolutionError &error) {
+    if (!json_output) {
       std::cout << entry.first.string() << ":\n";
-      throw FileError<
-          sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{
-          entry.first, error};
-    } catch (const sourcemeta::core::SchemaResolutionError &error) {
+    }
+    throw sourcemeta::jsonschema::FileError<
+        sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{entry.first,
+                                                                   error};
+  } catch (const sourcemeta::core::SchemaResolutionError &error) {
+    if (!json_output) {
       std::cout << entry.first.string() << ":\n";
-      throw FileError<sourcemeta::core::SchemaResolutionError>{entry.first,
-                                                               error};
-    } catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) {
+    }
+    throw sourcemeta::jsonschema::FileError<
+        sourcemeta::core::SchemaResolutionError>{entry.first, error};
+  } catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) {
+    if (!json_output) {
       std::cout << entry.first.string() << ":\n";
-      throw FileError<sourcemeta::core::SchemaUnknownBaseDialectError>{
-          entry.first};
-    } catch (...) {
+    }
+    throw sourcemeta::jsonschema::FileError<
+        sourcemeta::core::SchemaUnknownBaseDialectError>{entry.first};
+  } catch (...) {
+    if (!json_output) {
       std::cout << entry.first.string() << ":\n";
-      throw;
     }
+    throw;
+  }
+}
+
+auto report_as_text(const sourcemeta::core::Options &options) -> void {
+  bool result{true};
+  const auto verbose{options.contains("verbose")};
+
+  for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) {
+    const auto configuration_path{
+        sourcemeta::jsonschema::find_configuration(entry.first)};
+    const auto &configuration{sourcemeta::jsonschema::read_configuration(
+        options, configuration_path)};
+    const auto dialect{
+        sourcemeta::jsonschema::default_dialect(options, configuration)};
+    const auto &schema_resolver{sourcemeta::jsonschema::resolver(
+        options, options.contains("http"), dialect, configuration)};
+
+    auto test_suite{parse_test_suite(entry, schema_resolver, dialect, false)};
 
     std::cout << entry.first.string() << ":";
 
-    const auto suite_result{test_suite->run(
+    const auto suite_result{test_suite.run(
         [&](const sourcemeta::core::JSON::String &, std::size_t index,
             std::size_t total, const sourcemeta::blaze::TestCase &test_case,
             bool actual, sourcemeta::blaze::TestTimestamp,
@@ -90,20 +115,19 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
               std::cout << "\n";
             }
           } else {
-            // Re-run with exhaustive mode to get detailed error output
             const std::string ref{"$ref"};
             sourcemeta::blaze::SimpleOutput output{test_case.data,
                                                    {std::cref(ref)}};
-            test_suite->evaluator.validate(test_suite->schema_exhaustive,
-                                           test_case.data, std::ref(output));
+            test_suite.evaluator.validate(test_suite.schema_exhaustive,
+                                          test_case.data, std::ref(output));
 
             if (!verbose) {
               std::cout << "\n";
             }
 
             std::cout << "  " << index << "/" << total << " FAIL "
                       << description << "\n\n";
-            print(output, test_case.tracker, std::cout);
+            sourcemeta::jsonschema::print(output, test_case.tracker, std::cout);
 
             if (index != total && verbose) {
               std::cout << "\n";
@@ -124,8 +148,174 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
   }
 
   if (!result) {
-    // Report a different exit code for test failures, to
-    // distinguish them from other errors
-    throw Fail{2};
+    throw sourcemeta::jsonschema::Fail{2};
+  }
+}
+
+auto timestamp_to_unix_ms(
+    const sourcemeta::blaze::TestTimestamp &timestamp,
+    const std::chrono::system_clock::time_point &system_ref,
+    const sourcemeta::blaze::TestTimestamp &steady_ref) -> std::int64_t {
+  const auto offset{timestamp - steady_ref};
+  const auto unix_time{system_ref + offset};
+  return std::chrono::duration_cast<std::chrono::milliseconds>(
+             unix_time.time_since_epoch())
+      .count();
+}
+
+auto duration_ms(const sourcemeta::blaze::TestTimestamp &start,
+                 const sourcemeta::blaze::TestTimestamp &end) -> std::int64_t {
+  return std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+      .count();
+}
+
+auto report_as_ctrf(const sourcemeta::core::Options &options) -> void {
+  bool result{true};
+
+  const auto system_ref{std::chrono::system_clock::now()};
+  const auto steady_ref{std::chrono::steady_clock::now()};
+
+  auto ctrf_tests{sourcemeta::core::JSON::make_array()};
+  std::size_t total_passed{0};
+  std::size_t total_failed{0};
+  sourcemeta::blaze::TestTimestamp global_start{};
+  sourcemeta::blaze::TestTimestamp global_end{};
+  bool first_suite{true};
+
+  for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) {
+    const auto configuration_path{
+        sourcemeta::jsonschema::find_configuration(entry.first)};
+    const auto &configuration{sourcemeta::jsonschema::read_configuration(
+        options, configuration_path)};
+    const auto dialect{
+        sourcemeta::jsonschema::default_dialect(options, configuration)};
+    const auto &schema_resolver{sourcemeta::jsonschema::resolver(
+        options, options.contains("http"), dialect, configuration)};
+
+    auto test_suite{parse_test_suite(entry, schema_resolver, dialect, true)};
+
+    const auto file_path{
+        sourcemeta::core::weakly_canonical(entry.first).string()};
+
+    const auto suite_result{test_suite.run(
+        [&](const sourcemeta::core::JSON::String &target, std::size_t,
+            std::size_t, const sourcemeta::blaze::TestCase &test_case,
+            bool actual, sourcemeta::blaze::TestTimestamp start,
+            sourcemeta::blaze::TestTimestamp end) {
+          auto test_object{sourcemeta::core::JSON::make_object()};
+
+          const auto &name{test_case.description.empty()
+                               ? "<no description>"
+                               : test_case.description};
+          test_object.assign("name", sourcemeta::core::JSON{name});
+
+          const bool passed{test_case.valid == actual};
+          test_object.assign(
+              "status", sourcemeta::core::JSON{passed ? "passed" : "failed"});
+
+          test_object.assign("duration",
+                             sourcemeta::core::JSON{duration_ms(start, end)});
+          auto suite{sourcemeta::core::JSON::make_array()};
+          suite.push_back(sourcemeta::core::JSON{target});
+          test_object.assign("suite", std::move(suite));
+          test_object.assign("type", sourcemeta::core::JSON{"unit"});
+          test_object.assign("filePath", sourcemeta::core::JSON{file_path});
+
+          test_object.assign("line",
+                             sourcemeta::core::JSON{static_cast<std::int64_t>(
+                                 std::get<0>(test_case.position))});
+          test_object.assign(
+              "retries", sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
+          test_object.assign("flaky", sourcemeta::core::JSON{false});
+          std::ostringstream thread_id_stream;
+          thread_id_stream << std::this_thread::get_id();
+          test_object.assign("threadId",
+                             sourcemeta::core::JSON{thread_id_stream.str()});
+
+          if (!passed) {
+            if (!test_case.valid && actual) {
+              test_object.assign("message",
+                                 sourcemeta::core::JSON{"Passed but was "
+                                                        "expected to fail"});
+            } else {
+              std::ostringstream trace_stream;
+              const std::string ref{"$ref"};
+              sourcemeta::blaze::SimpleOutput output{test_case.data,
+                                                     {std::cref(ref)}};
+              test_suite.evaluator.validate(test_suite.schema_exhaustive,
+                                            test_case.data, std::ref(output));
+              sourcemeta::jsonschema::print(output, test_case.tracker,
+                                            trace_stream);
+              test_object.assign("trace",
+                                 sourcemeta::core::JSON{trace_stream.str()});
+            }
+          }
+
+          ctrf_tests.push_back(test_object);
+        })};
+
+    if (first_suite) {
+      global_start = suite_result.start;
+      first_suite = false;
+    }
+    global_end = suite_result.end;
+
+    total_passed += suite_result.passed;
+    total_failed += suite_result.total - suite_result.passed;
+
+    if (suite_result.passed != suite_result.total) {
+      result = false;
+    }
+  }
+
+  // Build CTRF output
+  auto summary{sourcemeta::core::JSON::make_object()};
+  summary.assign("tests", sourcemeta::core::JSON{static_cast<std::int64_t>(
+                              total_passed + total_failed)});
+  summary.assign("passed", sourcemeta::core::JSON{
+                               static_cast<std::int64_t>(total_passed)});
+  summary.assign("failed", sourcemeta::core::JSON{
+                               static_cast<std::int64_t>(total_failed)});
+  summary.assign("pending",
+                 sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
+  summary.assign("skipped",
+                 sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
+  summary.assign("other", sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
+  summary.assign("start", sourcemeta::core::JSON{timestamp_to_unix_ms(
+                              global_start, system_ref, steady_ref)});
+  summary.assign("stop", sourcemeta::core::JSON{timestamp_to_unix_ms(
+                             global_end, system_ref, steady_ref)});
+
+  auto tool{sourcemeta::core::JSON::make_object()};
+  tool.assign("name", sourcemeta::core::JSON{"jsonschema"});
+  tool.assign("version", sourcemeta::core::JSON{std::string{
+                             sourcemeta::jsonschema::PROJECT_VERSION}});
+
+  auto results{sourcemeta::core::JSON::make_object()};
+  results.assign("tool", std::move(tool));
+  results.assign("summary", std::move(summary));
+  results.assign("tests", std::move(ctrf_tests));
+
+  auto ctrf{sourcemeta::core::JSON::make_object()};
+  ctrf.assign("reportFormat", sourcemeta::core::JSON{"CTRF"});
+  ctrf.assign("specVersion", sourcemeta::core::JSON{"0.0.0"});
+  ctrf.assign("results", std::move(results));
+
+  sourcemeta::core::prettify(ctrf, std::cout);
+  std::cout << "\n";
+
+  if (!result) {
+    throw sourcemeta::jsonschema::Fail{2};
+  }
+}
+
+} // namespace
+
+auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
+    -> void {
+  if (options.contains("json")) {
+    report_as_ctrf(options);
+  } else {
+    report_as_text(options);
   }
 }

src/main.cc

@@ -66,6 +66,7 @@ Global Options:
         [--ignore/-i <schemas-or-directories>]
 
        Run a set of unit tests against a schema.
+       Pass --json/-j to output results in CTRF format (https://ctrf.io).
 
    fmt [schemas-or-directories...] [--check/-c] [--extension/-e <extension>]
        [--ignore/-i <schemas-or-directories>] [--keep-ordering/-k]

test/CMakeLists.txt

@@ -271,6 +271,7 @@ add_jsonschema_test_unix(test/pass_file_target_with_resolve)
 add_jsonschema_test_unix(test/pass_file_target_with_resolve_verbose)
 add_jsonschema_test_unix(test/pass_file_target_without_resolve)
 add_jsonschema_test_unix(test/pass_file_target_without_resolve_verbose)
+add_jsonschema_test_unix(test/pass_file_target_without_resolve_json)
 add_jsonschema_test_unix(test/pass_file_target_fragment)
 add_jsonschema_test_unix(test/pass_single_resolve_remap)
 add_jsonschema_test_unix(test/pass_single_resolve_remap_relative)
@@ -283,6 +284,11 @@ add_jsonschema_test_unix(test/pass_target_no_extension_json)
 add_jsonschema_test_unix(test/pass_target_no_extension_yaml)
 add_jsonschema_test_unix(test/pass_custom_extension_json)
 add_jsonschema_test_unix(test/pass_custom_extension_yaml)
+add_jsonschema_test_unix(test/pass_single_resolve_json)
+add_jsonschema_test_unix(test/pass_single_no_description_json)
+add_jsonschema_test_unix(test/fail_true_single_resolve_json)
+add_jsonschema_test_unix(test/fail_false_single_resolve_json)
+add_jsonschema_test_unix(test/fail_unresolvable_json)
 
 # Bundle
 add_jsonschema_test_unix(bundle/pass_into_resolve_directory)