diff --git a/DEPENDENCIES b/DEPENDENCIES index 614427c2..405e213c 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -3,3 +3,4 @@ core https://github.com/sourcemeta/core e4d7ae9358710fc138d2afd3179db6d850e4190f jsonbinpack https://github.com/sourcemeta/jsonbinpack 8fae212dc7ec02af4bb0cd4e7fccd42a2471f1c1 blaze https://github.com/sourcemeta/blaze 8dba65f8aebfe1ac976168b76e01c20dd406c517 hydra https://github.com/sourcemeta/hydra af9f2c54709d620872ead0c3f8f683c15a0fa702 +ctrf https://github.com/ctrf-io/ctrf 93ea827d951390190171d37443bff169cf47c808 diff --git a/docs/test.markdown b/docs/test.markdown index 6589a40f..266d8f2c 100644 --- a/docs/test.markdown +++ b/docs/test.markdown @@ -21,6 +21,10 @@ suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite). **If you want to validate that a schema adheres to its metaschema, use the [`metaschema`](./metaschema.markdown) command instead.** +Pass `--json` to output results in [CTRF (Common Test Report +Format)](https://ctrf.io), a standardized JSON format for test results that +integrates with CI/CD tools and test result dashboards. + Writing tests ------------- diff --git a/src/command_test.cc b/src/command_test.cc index d0a14c74..f40c17be 100644 --- a/src/command_test.cc +++ b/src/command_test.cc @@ -1,65 +1,90 @@ #include #include +#include + +#include // std::chrono #include // EXIT_FAILURE #include // std::cerr, std::cout +#include // std::ostringstream #include // std::string +#include // std::this_thread +#include // std::vector #include "command.h" #include "configuration.h" +#include "configure.h" #include "error.h" #include "input.h" #include "logger.h" #include "resolver.h" #include "utils.h" -auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options) - -> void { - bool result{true}; - - const auto verbose{options.contains("verbose")}; +namespace { - for (const auto &entry : for_each_json(options)) { - const auto configuration_path{find_configuration(entry.first)}; - const auto &configuration{read_configuration(options, configuration_path)}; - const auto dialect{default_dialect(options, configuration)}; - - const auto &schema_resolver{ - resolver(options, options.contains("http"), dialect, configuration)}; - - std::optional test_suite; - try { - test_suite.emplace(sourcemeta::blaze::TestSuite::parse( - entry.second, entry.positions, entry.first.parent_path(), - schema_resolver, sourcemeta::core::schema_walker, - sourcemeta::blaze::default_schema_compiler, dialect)); - } catch (const sourcemeta::blaze::TestParseError &error) { +auto parse_test_suite(const sourcemeta::jsonschema::InputJSON &entry, + const sourcemeta::core::SchemaResolver &schema_resolver, + const std::optional &dialect, + const bool json_output) -> sourcemeta::blaze::TestSuite { + try { + return sourcemeta::blaze::TestSuite::parse( + entry.second, entry.positions, entry.first.parent_path(), + schema_resolver, sourcemeta::core::schema_walker, + sourcemeta::blaze::default_schema_compiler, dialect); + } catch (const sourcemeta::blaze::TestParseError &error) { + if (!json_output) { std::cout << entry.first.string() << ":\n"; - throw FileError{ - entry.first, error.what(), error.location(), error.line(), - error.column()}; - } catch (const sourcemeta::core::SchemaRelativeMetaschemaResolutionError - &error) { + } + throw sourcemeta::jsonschema::FileError{ + entry.first, error.what(), error.location(), error.line(), + error.column()}; + } catch ( + const sourcemeta::core::SchemaRelativeMetaschemaResolutionError &error) { + if (!json_output) { std::cout << entry.first.string() << ":\n"; - throw FileError< - sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{ - entry.first, error}; - } catch (const sourcemeta::core::SchemaResolutionError &error) { + } + throw sourcemeta::jsonschema::FileError< + sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{entry.first, + error}; + } catch (const sourcemeta::core::SchemaResolutionError &error) { + if (!json_output) { std::cout << entry.first.string() << ":\n"; - throw FileError{entry.first, - error}; - } catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) { + } + throw sourcemeta::jsonschema::FileError< + sourcemeta::core::SchemaResolutionError>{entry.first, error}; + } catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) { + if (!json_output) { std::cout << entry.first.string() << ":\n"; - throw FileError{ - entry.first}; - } catch (...) { + } + throw sourcemeta::jsonschema::FileError< + sourcemeta::core::SchemaUnknownBaseDialectError>{entry.first}; + } catch (...) { + if (!json_output) { std::cout << entry.first.string() << ":\n"; - throw; } + throw; + } +} + +auto report_as_text(const sourcemeta::core::Options &options) -> void { + bool result{true}; + const auto verbose{options.contains("verbose")}; + + for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) { + const auto configuration_path{ + sourcemeta::jsonschema::find_configuration(entry.first)}; + const auto &configuration{sourcemeta::jsonschema::read_configuration( + options, configuration_path)}; + const auto dialect{ + sourcemeta::jsonschema::default_dialect(options, configuration)}; + const auto &schema_resolver{sourcemeta::jsonschema::resolver( + options, options.contains("http"), dialect, configuration)}; + + auto test_suite{parse_test_suite(entry, schema_resolver, dialect, false)}; std::cout << entry.first.string() << ":"; - const auto suite_result{test_suite->run( + const auto suite_result{test_suite.run( [&](const sourcemeta::core::JSON::String &, std::size_t index, std::size_t total, const sourcemeta::blaze::TestCase &test_case, bool actual, sourcemeta::blaze::TestTimestamp, @@ -90,12 +115,11 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options) std::cout << "\n"; } } else { - // Re-run with exhaustive mode to get detailed error output const std::string ref{"$ref"}; sourcemeta::blaze::SimpleOutput output{test_case.data, {std::cref(ref)}}; - test_suite->evaluator.validate(test_suite->schema_exhaustive, - test_case.data, std::ref(output)); + test_suite.evaluator.validate(test_suite.schema_exhaustive, + test_case.data, std::ref(output)); if (!verbose) { std::cout << "\n"; @@ -103,7 +127,7 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options) std::cout << " " << index << "/" << total << " FAIL " << description << "\n\n"; - print(output, test_case.tracker, std::cout); + sourcemeta::jsonschema::print(output, test_case.tracker, std::cout); if (index != total && verbose) { std::cout << "\n"; @@ -124,8 +148,174 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options) } if (!result) { - // Report a different exit code for test failures, to - // distinguish them from other errors - throw Fail{2}; + throw sourcemeta::jsonschema::Fail{2}; + } +} + +auto timestamp_to_unix_ms( + const sourcemeta::blaze::TestTimestamp ×tamp, + const std::chrono::system_clock::time_point &system_ref, + const sourcemeta::blaze::TestTimestamp &steady_ref) -> std::int64_t { + const auto offset{timestamp - steady_ref}; + const auto unix_time{system_ref + offset}; + return std::chrono::duration_cast( + unix_time.time_since_epoch()) + .count(); +} + +auto duration_ms(const sourcemeta::blaze::TestTimestamp &start, + const sourcemeta::blaze::TestTimestamp &end) -> std::int64_t { + return std::chrono::duration_cast(end - start) + .count(); +} + +auto report_as_ctrf(const sourcemeta::core::Options &options) -> void { + bool result{true}; + + const auto system_ref{std::chrono::system_clock::now()}; + const auto steady_ref{std::chrono::steady_clock::now()}; + + auto ctrf_tests{sourcemeta::core::JSON::make_array()}; + std::size_t total_passed{0}; + std::size_t total_failed{0}; + sourcemeta::blaze::TestTimestamp global_start{}; + sourcemeta::blaze::TestTimestamp global_end{}; + bool first_suite{true}; + + for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) { + const auto configuration_path{ + sourcemeta::jsonschema::find_configuration(entry.first)}; + const auto &configuration{sourcemeta::jsonschema::read_configuration( + options, configuration_path)}; + const auto dialect{ + sourcemeta::jsonschema::default_dialect(options, configuration)}; + const auto &schema_resolver{sourcemeta::jsonschema::resolver( + options, options.contains("http"), dialect, configuration)}; + + auto test_suite{parse_test_suite(entry, schema_resolver, dialect, true)}; + + const auto file_path{ + sourcemeta::core::weakly_canonical(entry.first).string()}; + + const auto suite_result{test_suite.run( + [&](const sourcemeta::core::JSON::String &target, std::size_t, + std::size_t, const sourcemeta::blaze::TestCase &test_case, + bool actual, sourcemeta::blaze::TestTimestamp start, + sourcemeta::blaze::TestTimestamp end) { + auto test_object{sourcemeta::core::JSON::make_object()}; + + const auto &name{test_case.description.empty() + ? "" + : test_case.description}; + test_object.assign("name", sourcemeta::core::JSON{name}); + + const bool passed{test_case.valid == actual}; + test_object.assign( + "status", sourcemeta::core::JSON{passed ? "passed" : "failed"}); + + test_object.assign("duration", + sourcemeta::core::JSON{duration_ms(start, end)}); + auto suite{sourcemeta::core::JSON::make_array()}; + suite.push_back(sourcemeta::core::JSON{target}); + test_object.assign("suite", std::move(suite)); + test_object.assign("type", sourcemeta::core::JSON{"unit"}); + test_object.assign("filePath", sourcemeta::core::JSON{file_path}); + + test_object.assign("line", + sourcemeta::core::JSON{static_cast( + std::get<0>(test_case.position))}); + test_object.assign( + "retries", sourcemeta::core::JSON{static_cast(0)}); + test_object.assign("flaky", sourcemeta::core::JSON{false}); + std::ostringstream thread_id_stream; + thread_id_stream << std::this_thread::get_id(); + test_object.assign("threadId", + sourcemeta::core::JSON{thread_id_stream.str()}); + + if (!passed) { + if (!test_case.valid && actual) { + test_object.assign("message", + sourcemeta::core::JSON{"Passed but was " + "expected to fail"}); + } else { + std::ostringstream trace_stream; + const std::string ref{"$ref"}; + sourcemeta::blaze::SimpleOutput output{test_case.data, + {std::cref(ref)}}; + test_suite.evaluator.validate(test_suite.schema_exhaustive, + test_case.data, std::ref(output)); + sourcemeta::jsonschema::print(output, test_case.tracker, + trace_stream); + test_object.assign("trace", + sourcemeta::core::JSON{trace_stream.str()}); + } + } + + ctrf_tests.push_back(test_object); + })}; + + if (first_suite) { + global_start = suite_result.start; + first_suite = false; + } + global_end = suite_result.end; + + total_passed += suite_result.passed; + total_failed += suite_result.total - suite_result.passed; + + if (suite_result.passed != suite_result.total) { + result = false; + } + } + + // Build CTRF output + auto summary{sourcemeta::core::JSON::make_object()}; + summary.assign("tests", sourcemeta::core::JSON{static_cast( + total_passed + total_failed)}); + summary.assign("passed", sourcemeta::core::JSON{ + static_cast(total_passed)}); + summary.assign("failed", sourcemeta::core::JSON{ + static_cast(total_failed)}); + summary.assign("pending", + sourcemeta::core::JSON{static_cast(0)}); + summary.assign("skipped", + sourcemeta::core::JSON{static_cast(0)}); + summary.assign("other", sourcemeta::core::JSON{static_cast(0)}); + summary.assign("start", sourcemeta::core::JSON{timestamp_to_unix_ms( + global_start, system_ref, steady_ref)}); + summary.assign("stop", sourcemeta::core::JSON{timestamp_to_unix_ms( + global_end, system_ref, steady_ref)}); + + auto tool{sourcemeta::core::JSON::make_object()}; + tool.assign("name", sourcemeta::core::JSON{"jsonschema"}); + tool.assign("version", sourcemeta::core::JSON{std::string{ + sourcemeta::jsonschema::PROJECT_VERSION}}); + + auto results{sourcemeta::core::JSON::make_object()}; + results.assign("tool", std::move(tool)); + results.assign("summary", std::move(summary)); + results.assign("tests", std::move(ctrf_tests)); + + auto ctrf{sourcemeta::core::JSON::make_object()}; + ctrf.assign("reportFormat", sourcemeta::core::JSON{"CTRF"}); + ctrf.assign("specVersion", sourcemeta::core::JSON{"0.0.0"}); + ctrf.assign("results", std::move(results)); + + sourcemeta::core::prettify(ctrf, std::cout); + std::cout << "\n"; + + if (!result) { + throw sourcemeta::jsonschema::Fail{2}; + } +} + +} // namespace + +auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options) + -> void { + if (options.contains("json")) { + report_as_ctrf(options); + } else { + report_as_text(options); } } diff --git a/src/main.cc b/src/main.cc index b4b6197f..e4319648 100644 --- a/src/main.cc +++ b/src/main.cc @@ -66,6 +66,7 @@ Global Options: [--ignore/-i ] Run a set of unit tests against a schema. + Pass --json/-j to output results in CTRF format (https://ctrf.io). fmt [schemas-or-directories...] [--check/-c] [--extension/-e ] [--ignore/-i ] [--keep-ordering/-k] diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4cab5c63..2a124db3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -271,6 +271,7 @@ add_jsonschema_test_unix(test/pass_file_target_with_resolve) add_jsonschema_test_unix(test/pass_file_target_with_resolve_verbose) add_jsonschema_test_unix(test/pass_file_target_without_resolve) add_jsonschema_test_unix(test/pass_file_target_without_resolve_verbose) +add_jsonschema_test_unix(test/pass_file_target_without_resolve_json) add_jsonschema_test_unix(test/pass_file_target_fragment) add_jsonschema_test_unix(test/pass_single_resolve_remap) add_jsonschema_test_unix(test/pass_single_resolve_remap_relative) @@ -283,6 +284,11 @@ add_jsonschema_test_unix(test/pass_target_no_extension_json) add_jsonschema_test_unix(test/pass_target_no_extension_yaml) add_jsonschema_test_unix(test/pass_custom_extension_json) add_jsonschema_test_unix(test/pass_custom_extension_yaml) +add_jsonschema_test_unix(test/pass_single_resolve_json) +add_jsonschema_test_unix(test/pass_single_no_description_json) +add_jsonschema_test_unix(test/fail_true_single_resolve_json) +add_jsonschema_test_unix(test/fail_false_single_resolve_json) +add_jsonschema_test_unix(test/fail_unresolvable_json) # Bundle add_jsonschema_test_unix(bundle/pass_into_resolve_directory) diff --git a/test/test/fail_false_single_resolve_json.sh b/test/test/fail_false_single_resolve_json.sh new file mode 100755 index 00000000..16c70be9 --- /dev/null +++ b/test/test/fail_false_single_resolve_json.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "Should fail but passes", + "valid": false, + "data": "valid-string" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --json > "$TMP/output.json" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "2" || exit 1 + +# Validate against CTRF schema +CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json" +"$1" validate "$CTRF_SCHEMA" "$TMP/output.json" + +# Remove dynamic fields for comparison +sed -e '/"duration":/d' \ + -e '/"start":/d' \ + -e '/"stop":/d' \ + -e '/"threadId":/d' \ + "$TMP/output.json" > "$TMP/output_filtered.json" + +VERSION=$("$1" --version) + +cat << EOF > "$TMP/expected.json" +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "results": { + "tool": { + "name": "jsonschema", + "version": "$VERSION" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + }, + "tests": [ + { + "name": "Should fail but passes", + "status": "failed", + "suite": [ "https://example.com" ], + "type": "unit", + "filePath": "$(realpath "$TMP")/test.json", + "line": 4, + "retries": 0, + "flaky": false, + "message": "Passed but was expected to fail" + } + ] + } +} +EOF + +diff "$TMP/output_filtered.json" "$TMP/expected.json" diff --git a/test/test/fail_no_schema.sh b/test/test/fail_no_schema.sh index 78fa3378..dfee845e 100755 --- a/test/test/fail_no_schema.sh +++ b/test/test/fail_no_schema.sh @@ -45,7 +45,6 @@ diff "$TMP/output.txt" "$TMP/expected.txt" test "$CODE" = "1" || exit 1 cat << EOF > "$TMP/expected.txt" -$(realpath "$TMP")/test.json: { "error": "The test document must contain a \`target\` property", "line": 1, diff --git a/test/test/fail_true_single_resolve_json.sh b/test/test/fail_true_single_resolve_json.sh new file mode 100755 index 00000000..4c5768bb --- /dev/null +++ b/test/test/fail_true_single_resolve_json.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "Should pass but fails", + "valid": true, + "data": 123 + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --json > "$TMP/output.json" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "2" || exit 1 + +# Validate against CTRF schema +CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json" +"$1" validate "$CTRF_SCHEMA" "$TMP/output.json" + +# Remove dynamic fields for comparison +sed -e '/"duration":/d' \ + -e '/"start":/d' \ + -e '/"stop":/d' \ + -e '/"threadId":/d' \ + "$TMP/output.json" > "$TMP/output_filtered.json" + +VERSION=$("$1" --version) + +cat << EOF > "$TMP/expected.json" +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "results": { + "tool": { + "name": "jsonschema", + "version": "$VERSION" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + }, + "tests": [ + { + "name": "Should pass but fails", + "status": "failed", + "suite": [ "https://example.com" ], + "type": "unit", + "filePath": "$(realpath "$TMP")/test.json", + "line": 4, + "retries": 0, + "flaky": false, + "trace": "error: Schema validation failure\n The value was expected to be of type string but it was of type integer\n at instance location \"\"\n at evaluate path \"/type\"\n" + } + ] + } +} +EOF + +diff "$TMP/output_filtered.json" "$TMP/expected.json" diff --git a/test/test/fail_unresolvable_json.sh b/test/test/fail_unresolvable_json.sh new file mode 100755 index 00000000..0adbd695 --- /dev/null +++ b/test/test/fail_unresolvable_json.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com/unknown", + "tests": [ + { + "valid": true, + "data": {} + } + ] +} +EOF + +"$1" test "$TMP/test.json" --json > "$TMP/output.json" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.json" +{ + "error": "Could not resolve the reference to an external schema", + "identifier": "https://example.com/unknown", + "filePath": "$(realpath "$TMP")/test.json" +} +EOF + +diff "$TMP/output.json" "$TMP/expected.json" diff --git a/test/test/pass_file_target_without_resolve_json.sh b/test/test/pass_file_target_without_resolve_json.sh new file mode 100755 index 00000000..dce39911 --- /dev/null +++ b/test/test/pass_file_target_without_resolve_json.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +mkdir -p "$TMP/this/is/a/very/very/very/long/path" + +cat << 'EOF' > "$TMP/this/is/a/very/very/very/long/path/schema.json" +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/this/is/a/very/very/very/long/path/test.json" +{ + "target": "./schema.json", + "tests": [ + { + "description": "Valid string", + "valid": true, + "data": "foo" + }, + { + "description": "Invalid type", + "valid": false, + "data": 1 + } + ] +} +EOF + +"$1" test "$TMP/this/is/a/very/very/very/long/path/test.json" --json > "$TMP/output.json" 2>&1 + +# Validate against CTRF schema +CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json" +"$1" validate "$CTRF_SCHEMA" "$TMP/output.json" + +# Remove dynamic fields for comparison +sed -e '/"duration":/d' \ + -e '/"start":/d' \ + -e '/"stop":/d' \ + -e '/"threadId":/d' \ + "$TMP/output.json" > "$TMP/output_filtered.json" + +VERSION=$("$1" --version) + +cat << EOF > "$TMP/expected.json" +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "results": { + "tool": { + "name": "jsonschema", + "version": "$VERSION" + }, + "summary": { + "tests": 2, + "passed": 2, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + }, + "tests": [ + { + "name": "Valid string", + "status": "passed", + "suite": [ + "file://$(realpath "$TMP")/this/is/a/very/very/very/long/path/schema.json" + ], + "type": "unit", + "filePath": "$(realpath "$TMP")/this/is/a/very/very/very/long/path/test.json", + "line": 4, + "retries": 0, + "flaky": false, + }, + { + "name": "Invalid type", + "status": "passed", + "suite": [ + "file://$(realpath "$TMP")/this/is/a/very/very/very/long/path/schema.json" + ], + "type": "unit", + "filePath": "$(realpath "$TMP")/this/is/a/very/very/very/long/path/test.json", + "line": 9, + "retries": 0, + "flaky": false, + } + ] + } +} +EOF + +diff "$TMP/output_filtered.json" "$TMP/expected.json" diff --git a/test/test/pass_single_no_description_json.sh b/test/test/pass_single_no_description_json.sh new file mode 100755 index 00000000..851a3607 --- /dev/null +++ b/test/test/pass_single_no_description_json.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "valid": true, + "data": "foo" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --json > "$TMP/output.json" 2>&1 + +# Validate against CTRF schema +CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json" +"$1" validate "$CTRF_SCHEMA" "$TMP/output.json" + +# Remove dynamic fields for comparison +sed -e '/"duration":/d' \ + -e '/"start":/d' \ + -e '/"stop":/d' \ + -e '/"threadId":/d' \ + "$TMP/output.json" > "$TMP/output_filtered.json" + +VERSION=$("$1" --version) + +cat << EOF > "$TMP/expected.json" +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "results": { + "tool": { + "name": "jsonschema", + "version": "$VERSION" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + }, + "tests": [ + { + "name": "", + "status": "passed", + "suite": [ "https://example.com" ], + "type": "unit", + "filePath": "$(realpath "$TMP")/test.json", + "line": 4, + "retries": 0, + "flaky": false, + } + ] + } +} +EOF + +diff "$TMP/output_filtered.json" "$TMP/expected.json" diff --git a/test/test/pass_single_resolve_json.sh b/test/test/pass_single_resolve_json.sh new file mode 100755 index 00000000..351c0cc6 --- /dev/null +++ b/test/test/pass_single_resolve_json.sh @@ -0,0 +1,94 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "First test", + "valid": true, + "data": "foo" + }, + { + "description": "Invalid type", + "valid": false, + "data": 1 + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --json > "$TMP/output.json" 2>&1 + +# Validate against CTRF schema +CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json" +"$1" validate "$CTRF_SCHEMA" "$TMP/output.json" + +# Remove dynamic fields for comparison +sed -e '/"duration":/d' \ + -e '/"start":/d' \ + -e '/"stop":/d' \ + -e '/"threadId":/d' \ + "$TMP/output.json" > "$TMP/output_filtered.json" + +VERSION=$("$1" --version) + +cat << EOF > "$TMP/expected.json" +{ + "reportFormat": "CTRF", + "specVersion": "0.0.0", + "results": { + "tool": { + "name": "jsonschema", + "version": "$VERSION" + }, + "summary": { + "tests": 2, + "passed": 2, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + }, + "tests": [ + { + "name": "First test", + "status": "passed", + "suite": [ "https://example.com" ], + "type": "unit", + "filePath": "$(realpath "$TMP")/test.json", + "line": 4, + "retries": 0, + "flaky": false, + }, + { + "name": "Invalid type", + "status": "passed", + "suite": [ "https://example.com" ], + "type": "unit", + "filePath": "$(realpath "$TMP")/test.json", + "line": 9, + "retries": 0, + "flaky": false, + } + ] + } +} +EOF + +diff "$TMP/output_filtered.json" "$TMP/expected.json" diff --git a/vendor/ctrf.mask b/vendor/ctrf.mask new file mode 100644 index 00000000..05d5ea94 --- /dev/null +++ b/vendor/ctrf.mask @@ -0,0 +1,2 @@ +website +README.md diff --git a/vendor/ctrf/specification/schema-0.0.0.json b/vendor/ctrf/specification/schema-0.0.0.json new file mode 100644 index 00000000..623223ae --- /dev/null +++ b/vendor/ctrf/specification/schema-0.0.0.json @@ -0,0 +1,251 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "reportFormat": { + "type": "string", + "enum": ["CTRF"] + }, + "specVersion": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "reportId": { "type": "string", "format": "uuid" }, + "timestamp": { "type": "string", "format": "date-time" }, + "generatedBy": { "type": "string" }, + "extra": { "type": "object", "additionalProperties": true }, + "results": { + "type": "object", + "properties": { + "tool": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["name"] + }, + "summary": { + "type": "object", + "properties": { + "tests": { "type": "integer" }, + "passed": { "type": "integer" }, + "failed": { "type": "integer" }, + "skipped": { "type": "integer" }, + "pending": { "type": "integer" }, + "other": { "type": "integer" }, + "flaky": { "type": "integer" }, + "suites": { "type": "integer" }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "duration": { "type": "integer" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["tests", "passed", "failed", "skipped", "pending", "other", "start", "stop"] + }, + "tests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string", "format": "uuid" }, + "name": { "type": "string", "minLength": 1 }, + "status": { "type": "string", "enum": ["passed", "failed", "skipped", "pending", "other"] }, + "duration": { "type": "integer" }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "suite": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "message": { "type": "string" }, + "trace": { "type": "string" }, + "snippet": { "type": "string" }, + "ai": { "type": "string" }, + "line": { "type": "integer" }, + "rawStatus": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "type": { "type": "string" }, + "filePath": { "type": "string" }, + "retries": { "type": "integer" }, + "retryAttempts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "attempt": { "type": "integer", "minimum": 1 }, + "status": { + "type": "string", + "enum": ["passed", "failed", "skipped", "pending", "other"] + }, + "duration": { "type": "integer" }, + "message": { "type": "string" }, + "trace": { "type": "string" }, + "line": { "type": "integer" }, + "snippet": { "type": "string" }, + "stdout": { + "type": "array", + "items": { "type": "string" } + }, + "stderr": { + "type": "array", + "items": { "type": "string" } + }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "attachments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "contentType": { "type": "string" }, + "path": { "type": "string" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["name", "contentType", "path"] + } + }, + "extra": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false, + "required": ["attempt", "status"] + } + }, + "flaky": { "type": "boolean" }, + "stdout": { "type": "array", "items": { "type": "string" } }, + "stderr": { "type": "array", "items": { "type": "string" } }, + "threadId": { "type": "string" }, + "browser": { "type": "string" }, + "device": { "type": "string" }, + "screenshot": { "type": "string" }, + "attachments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "contentType": { "type": "string" }, + "path": { "type": "string" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["name", "contentType", "path"] + } + }, + "parameters": { "type": "object", "additionalProperties": true }, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "status": { "type": "string", "enum": ["passed", "failed", "skipped", "pending", "other"] }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["name", "status"] + } + }, + "insights": { + "type": "object", + "properties": { + "passRate": { "$ref": "#/definitions/metricDelta" }, + "failRate": { "$ref": "#/definitions/metricDelta" }, + "flakyRate": { "$ref": "#/definitions/metricDelta" }, + "averageTestDuration": { "$ref": "#/definitions/metricDelta" }, + "p95TestDuration": { "$ref": "#/definitions/metricDelta" }, + "executedInRuns": { "type": "integer" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false + }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["name", "status", "duration"] + } + }, + "environment": { + "type": "object", + "properties": { + "reportName": { "type": "string" }, + "appName": { "type": "string" }, + "appVersion": { "type": "string" }, + "buildId": { "type": "string" }, + "buildName": { "type": "string" }, + "buildNumber": { "type": "integer" }, + "buildUrl": { "type": "string" }, + "repositoryName": { "type": "string" }, + "repositoryUrl": { "type": "string" }, + "commit": { "type": "string" }, + "branchName": { "type": "string" }, + "osPlatform": { "type": "string" }, + "osRelease": { "type": "string" }, + "osVersion": { "type": "string" }, + "testEnvironment": { "type": "string" }, + "healthy": { "type": "boolean" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false + }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false, + "required": ["tool", "summary", "tests"] + }, + "insights": { + "type": "object", + "properties": { + "passRate": { "$ref": "#/definitions/metricDelta" }, + "failRate": { "$ref": "#/definitions/metricDelta" }, + "flakyRate": { "$ref": "#/definitions/metricDelta" }, + "averageRunDuration": { "$ref": "#/definitions/metricDelta" }, + "p95RunDuration": { "$ref": "#/definitions/metricDelta" }, + "averageTestDuration": { "$ref": "#/definitions/metricDelta" }, + "runsAnalyzed": { "type": "integer" }, + "extra": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": false + }, + "baseline": { + "type": "object", + "properties": { + "reportId": { "type": "string", "format": "uuid" }, + "timestamp": { "type": "string", "format": "date-time" }, + "source": { "type": "string" }, + "buildNumber": { "type": "integer" }, + "buildName": { "type": "string" }, + "buildUrl": { "type": "string", "format": "uri" }, + "commit": { "type": "string" }, + "extra": { + "type": "object", + "additionalProperties": true + } + }, + "required": ["reportId"], + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": ["results", "reportFormat", "specVersion"], + "definitions": { + "metricDelta": { + "type": "object", + "properties": { + "current": { "type": "number" }, + "baseline": { "type": "number" }, + "change": { "type": "number" } + }, + "additionalProperties": false + } + } +} \ No newline at end of file