[WIP] Integrate against the new official annotations test suite

See: json-schema-org/JSON-Schema-Test-Suite#775
Fixes: #442
Signed-off-by: Juan Cruz Viotti <[email protected]>

Author: jviotti

DEPENDENCIES

@@ -1,3 +1,3 @@
 vendorpull https://github.com/sourcemeta/vendorpull dea311b5bfb53b6926a4140267959ae334d3ecf4
 core https://github.com/sourcemeta/core 375cdcacefe7f8bedb038c761fce1802e4954244
-jsonschema-test-suite https://github.com/json-schema-org/JSON-Schema-Test-Suite bc919bdb266a4949f8a2425f2f540371604d82b4
+jsonschema-test-suite https://github.com/json-schema-org/JSON-Schema-Test-Suite 9ad349be933f1e74810cb4fd3ad19780694dc77e

test/evaluator/CMakeLists.txt

@@ -33,3 +33,15 @@ target_link_libraries(sourcemeta_blaze_evaluator_official_suite_unit PRIVATE sou
 target_link_libraries(sourcemeta_blaze_evaluator_official_suite_unit PRIVATE sourcemeta::core::jsonschema)
 target_link_libraries(sourcemeta_blaze_evaluator_official_suite_unit PRIVATE sourcemeta::blaze::compiler)
 target_link_libraries(sourcemeta_blaze_evaluator_official_suite_unit PRIVATE sourcemeta::blaze::evaluator)
+
+# JSON Schema Test Suite (annotations)
+# See https://github.com/json-schema-org/JSON-Schema-Test-Suite/tree/main/annotations
+sourcemeta_googletest(NAMESPACE sourcemeta PROJECT blaze NAME evaluator_annotation_suite
+  FOLDER "Blaze/Evaluator" SOURCES annotationsuite.cc)
+target_compile_definitions(sourcemeta_blaze_evaluator_annotation_suite_unit
+  PRIVATE ANNOTATION_SUITE_PATH="${PROJECT_SOURCE_DIR}/vendor/jsonschema-test-suite/annotations/tests")
+target_link_libraries(sourcemeta_blaze_evaluator_annotation_suite_unit PRIVATE sourcemeta::core::json)
+target_link_libraries(sourcemeta_blaze_evaluator_annotation_suite_unit PRIVATE sourcemeta::core::jsonschema)
+target_link_libraries(sourcemeta_blaze_evaluator_annotation_suite_unit PRIVATE sourcemeta::core::jsonpointer)
+target_link_libraries(sourcemeta_blaze_evaluator_annotation_suite_unit PRIVATE sourcemeta::blaze::compiler)
+target_link_libraries(sourcemeta_blaze_evaluator_annotation_suite_unit PRIVATE sourcemeta::blaze::evaluator)

test/evaluator/annotationsuite.cc

@@ -0,0 +1,219 @@
+#include <gtest/gtest.h>
+
+#include <sourcemeta/blaze/compiler.h>
+#include <sourcemeta/blaze/evaluator.h>
+
+#include <sourcemeta/core/json.h>
+#include <sourcemeta/core/jsonpointer.h>
+#include <sourcemeta/core/jsonschema.h>
+
+#include <algorithm>
+#include <cassert>
+#include <filesystem>
+#include <functional>
+#include <iostream>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <unordered_set>
+#include <utility>
+
+static auto slugify(const std::string &input, std::ostream &output) -> void {
+  for (const auto character : input) {
+    output << (std::isalnum(character) ? character : '_');
+  }
+}
+
+static auto
+has_annotation(const sourcemeta::blaze::SimpleOutput &output,
+               const sourcemeta::core::WeakPointer &instance_location,
+               const std::string &schema_location,
+               const sourcemeta::core::JSON &value) -> bool {
+  std::cerr << "Looking for ";
+  sourcemeta::core::stringify(value, std::cerr);
+  std::cerr << " at instance location \""
+            << sourcemeta::core::to_string(instance_location)
+            << "\" and schema location \"" << schema_location << "\"" << "\n";
+
+  for (const auto &annotation : output.annotations()) {
+    std::cerr << "Checking against annotations for instance location \""
+              << sourcemeta::core::to_string(annotation.first.instance_location)
+              << "\" and schema location \"" << annotation.first.schema_location
+              << "\"" << "\n";
+
+    if (annotation.first.instance_location != instance_location ||
+        annotation.first.schema_location != schema_location) {
+      continue;
+    } else if (std::find(annotation.second.cbegin(), annotation.second.cend(),
+                         value) != annotation.second.cend()) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+class AnnotationTest : public testing::Test {
+public:
+  explicit AnnotationTest(sourcemeta::blaze::Template test_schema,
+                          sourcemeta::core::JSON test_instance,
+                          sourcemeta::core::JSON::Array test_assertions)
+      : schema{std::move(test_schema)}, instance{std::move(test_instance)},
+        assertions{std::move(test_assertions)} {}
+
+  auto TestBody() -> void override {
+    sourcemeta::blaze::SimpleOutput output{this->instance};
+    const auto result{this->evaluator.validate(this->schema, this->instance,
+                                               std::ref(output))};
+    EXPECT_TRUE(result);
+
+    for (const auto &assertion : this->assertions) {
+      assert(assertion.is_object());
+      assert(assertion.defines("location"));
+      assert(assertion.at("location").is_string());
+      const auto instance_location{
+          sourcemeta::core::to_pointer(assertion.at("location").to_string())};
+      assert(assertion.defines("expected"));
+      assert(assertion.at("expected").is_object());
+      for (const auto &entry : assertion.at("expected").as_object()) {
+        std::ostringstream schema_location;
+        schema_location << entry.first;
+        assert(assertion.defines("keyword"));
+        assert(assertion.at("keyword").is_string());
+        schema_location << "/" << assertion.at("keyword").to_string();
+
+        EXPECT_TRUE(has_annotation(
+            output, sourcemeta::core::to_weak_pointer(instance_location),
+            schema_location.str(), entry.second));
+      }
+    }
+  }
+
+private:
+  const sourcemeta::blaze::Template schema;
+  const sourcemeta::core::JSON instance;
+  const sourcemeta::core::JSON::Array assertions;
+  sourcemeta::blaze::Evaluator evaluator;
+};
+
+static auto register_tests(const std::string &suite_name,
+                           const std::string &default_dialect,
+                           const std::string &default_dialect_name,
+                           const std::unordered_set<std::string> &targets)
+    -> void {
+  auto suite_path{std::filesystem::path{ANNOTATION_SUITE_PATH} / suite_name};
+  suite_path.replace_extension(".json");
+  assert(std::filesystem::exists(suite_path));
+
+  std::cerr << "Registering suite: " << suite_path.string() << "\n";
+  std::cerr << "-- Dialect: " << default_dialect << "\n";
+  std::cerr << "-- Targets:";
+  assert(!targets.empty());
+  for (const auto &target : targets) {
+    std::cerr << " " << target;
+  }
+  std::cerr << "\n";
+
+  const auto suite{sourcemeta::core::read_json(suite_path)};
+  assert(suite.is_object());
+  assert(suite.defines("suite"));
+  assert(suite.at("suite").is_array());
+
+  for (const auto &entry : suite.at("suite").as_array()) {
+    assert(entry.is_object());
+    std::ostringstream category;
+    category << "JSONSchemaAnnotationSuite_";
+    slugify(default_dialect_name, category);
+    category << "_" << suite_name << "_";
+    assert(entry.defines("description"));
+    assert(entry.at("description").is_string());
+    slugify(entry.at("description").to_string(), category);
+
+    if (entry.defines("compatibility")) {
+      assert(entry.at("compatibility").is_string());
+      const auto &compatibility{entry.at("compatibility").to_string()};
+      if (!targets.contains(compatibility)) {
+        std::cerr << "-- Skipping " << category.str() << " for compatibility "
+                  << compatibility << "\n";
+        continue;
+      }
+    }
+
+    std::cerr << "-- Compiling " << category.str() << "\n";
+    assert(entry.defines("schema"));
+    const auto &schema{entry.at("schema")};
+    assert(sourcemeta::core::is_schema(schema));
+    const auto schema_template{sourcemeta::blaze::compile(
+        schema, sourcemeta::core::schema_official_walker,
+        sourcemeta::core::schema_official_resolver,
+        sourcemeta::blaze::default_schema_compiler,
+        sourcemeta::blaze::Mode::Exhaustive, default_dialect)};
+
+    assert(entry.defines("tests"));
+    assert(entry.at("tests").is_array());
+    for (std::size_t index = 0; index < entry.at("tests").size(); index++) {
+      std::ostringstream title;
+      title << index;
+      const auto &test{entry.at("tests").at(index)};
+      assert(test.is_object());
+      assert(test.defines("instance"));
+      const auto &instance{test.at("instance")};
+      assert(test.defines("assertions"));
+      assert(test.at("assertions").is_array());
+      const auto &assertions{test.at("assertions").as_array()};
+
+      testing::RegisterTest(
+          category.str().c_str(), title.str().c_str(), nullptr, nullptr,
+          __FILE__, __LINE__, [=]() -> AnnotationTest * {
+            return new AnnotationTest(schema_template, instance, assertions);
+          });
+    }
+  }
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  try {
+    // 2020-12
+    register_tests("applicators",
+                   "https://json-schema.org/draft/2020-12/schema", "2020-12",
+                   {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("content", "https://json-schema.org/draft/2020-12/schema",
+                   "2020-12", {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("core", "https://json-schema.org/draft/2020-12/schema",
+                   "2020-12", {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("format", "https://json-schema.org/draft/2020-12/schema",
+                   "2020-12", {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("meta-data", "https://json-schema.org/draft/2020-12/schema",
+                   "2020-12", {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("unevaluated",
+                   "https://json-schema.org/draft/2020-12/schema", "2020-12",
+                   {"3", "4", "6", "7", "2019", "2020"});
+    register_tests("unknown", "https://json-schema.org/draft/2020-12/schema",
+                   "2020-12", {"3", "4", "6", "7", "2019", "2020"});
+
+    // 2019-09
+    register_tests("applicators",
+                   "https://json-schema.org/draft/2019-09/schema", "2019-09",
+                   {"3", "4", "6", "7", "2019"});
+    register_tests("content", "https://json-schema.org/draft/2019-09/schema",
+                   "2019-09", {"3", "4", "6", "7", "2019"});
+    register_tests("core", "https://json-schema.org/draft/2019-09/schema",
+                   "2019-09", {"3", "4", "6", "7", "2019"});
+    register_tests("format", "https://json-schema.org/draft/2019-09/schema",
+                   "2019-09", {"3", "4", "6", "7", "2019"});
+    register_tests("meta-data", "https://json-schema.org/draft/2019-09/schema",
+                   "2019-09", {"3", "4", "6", "7", "2019"});
+    register_tests("unevaluated",
+                   "https://json-schema.org/draft/2019-09/schema", "2019-09",
+                   {"3", "4", "6", "7", "2019"});
+    register_tests("unknown", "https://json-schema.org/draft/2019-09/schema",
+                   "2019-09", {"3", "4", "6", "7", "2019"});
+  } catch (const std::exception &error) {
+    std::cerr << "Error: " << error.what() << "\n";
+    return EXIT_FAILURE;
+  }
+
+  return RUN_ALL_TESTS();
+}