Skip to content

Commit 06aa419

Browse files
jviottijviotti
committed
[WIP] Emit CTRF output on test when --json is passed
See: #589 Signed-off-by: Juan Cruz Viotti <[email protected]>
1 parent 335bf1c commit 06aa419

14 files changed

+968
-45
lines changed

‎DEPENDENCIES‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ core https://github.com/sourcemeta/core e4d7ae9358710fc138d2afd3179db6d850e4190f
33
jsonbinpack https://github.com/sourcemeta/jsonbinpack 8fae212dc7ec02af4bb0cd4e7fccd42a2471f1c1
44
blaze https://github.com/sourcemeta/blaze 8dba65f8aebfe1ac976168b76e01c20dd406c517
55
hydra https://github.com/sourcemeta/hydra af9f2c54709d620872ead0c3f8f683c15a0fa702
6+
ctrf https://github.com/ctrf-io/ctrf 93ea827d951390190171d37443bff169cf47c808

‎docs/test.markdown‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite).
2121
**If you want to validate that a schema adheres to its metaschema, use the
2222
[`metaschema`](./metaschema.markdown) command instead.**
2323

24+
Pass `--json` to output results in [CTRF (Common Test Report
25+
Format)](https://ctrf.io), a standardized JSON format for test results that
26+
integrates with CI/CD tools and test result dashboards.
27+
2428
Writing tests
2529
-------------
2630

‎src/command_test.cc‎

Lines changed: 234 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,90 @@
11
#include <sourcemeta/blaze/output.h>
22
#include <sourcemeta/blaze/test.h>
33

4+
#include <sourcemeta/core/json.h>
5+
6+
#include <chrono> // std::chrono
47
#include <cstdlib> // EXIT_FAILURE
58
#include <iostream> // std::cerr, std::cout
9+
#include <sstream> // std::ostringstream
610
#include <string> // std::string
11+
#include <thread> // std::this_thread
12+
#include <vector> // std::vector
713

814
#include "command.h"
915
#include "configuration.h"
16+
#include "configure.h"
1017
#include "error.h"
1118
#include "input.h"
1219
#include "logger.h"
1320
#include "resolver.h"
1421
#include "utils.h"
1522

16-
auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
17-
-> void {
18-
bool result{true};
19-
20-
const auto verbose{options.contains("verbose")};
23+
namespace {
2124

22-
for (const auto &entry : for_each_json(options)) {
23-
const auto configuration_path{find_configuration(entry.first)};
24-
const auto &configuration{read_configuration(options, configuration_path)};
25-
const auto dialect{default_dialect(options, configuration)};
26-
27-
const auto &schema_resolver{
28-
resolver(options, options.contains("http"), dialect, configuration)};
29-
30-
std::optional<sourcemeta::blaze::TestSuite> test_suite;
31-
try {
32-
test_suite.emplace(sourcemeta::blaze::TestSuite::parse(
33-
entry.second, entry.positions, entry.first.parent_path(),
34-
schema_resolver, sourcemeta::core::schema_walker,
35-
sourcemeta::blaze::default_schema_compiler, dialect));
36-
} catch (const sourcemeta::blaze::TestParseError &error) {
25+
auto parse_test_suite(const sourcemeta::jsonschema::InputJSON &entry,
26+
const sourcemeta::core::SchemaResolver &schema_resolver,
27+
const std::optional<std::string> &dialect,
28+
const bool json_output) -> sourcemeta::blaze::TestSuite {
29+
try {
30+
return sourcemeta::blaze::TestSuite::parse(
31+
entry.second, entry.positions, entry.first.parent_path(),
32+
schema_resolver, sourcemeta::core::schema_walker,
33+
sourcemeta::blaze::default_schema_compiler, dialect);
34+
} catch (const sourcemeta::blaze::TestParseError &error) {
35+
if (!json_output) {
3736
std::cout << entry.first.string() << ":\n";
38-
throw FileError<sourcemeta::blaze::TestParseError>{
39-
entry.first, error.what(), error.location(), error.line(),
40-
error.column()};
41-
} catch (const sourcemeta::core::SchemaRelativeMetaschemaResolutionError
42-
&error) {
37+
}
38+
throw sourcemeta::jsonschema::FileError<sourcemeta::blaze::TestParseError>{
39+
entry.first, error.what(), error.location(), error.line(),
40+
error.column()};
41+
} catch (
42+
const sourcemeta::core::SchemaRelativeMetaschemaResolutionError &error) {
43+
if (!json_output) {
4344
std::cout << entry.first.string() << ":\n";
44-
throw FileError<
45-
sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{
46-
entry.first, error};
47-
} catch (const sourcemeta::core::SchemaResolutionError &error) {
45+
}
46+
throw sourcemeta::jsonschema::FileError<
47+
sourcemeta::core::SchemaRelativeMetaschemaResolutionError>{entry.first,
48+
error};
49+
} catch (const sourcemeta::core::SchemaResolutionError &error) {
50+
if (!json_output) {
4851
std::cout << entry.first.string() << ":\n";
49-
throw FileError<sourcemeta::core::SchemaResolutionError>{entry.first,
50-
error};
51-
} catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) {
52+
}
53+
throw sourcemeta::jsonschema::FileError<
54+
sourcemeta::core::SchemaResolutionError>{entry.first, error};
55+
} catch (const sourcemeta::core::SchemaUnknownBaseDialectError &) {
56+
if (!json_output) {
5257
std::cout << entry.first.string() << ":\n";
53-
throw FileError<sourcemeta::core::SchemaUnknownBaseDialectError>{
54-
entry.first};
55-
} catch (...) {
58+
}
59+
throw sourcemeta::jsonschema::FileError<
60+
sourcemeta::core::SchemaUnknownBaseDialectError>{entry.first};
61+
} catch (...) {
62+
if (!json_output) {
5663
std::cout << entry.first.string() << ":\n";
57-
throw;
5864
}
65+
throw;
66+
}
67+
}
68+
69+
auto report_as_text(const sourcemeta::core::Options &options) -> void {
70+
bool result{true};
71+
const auto verbose{options.contains("verbose")};
72+
73+
for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) {
74+
const auto configuration_path{
75+
sourcemeta::jsonschema::find_configuration(entry.first)};
76+
const auto &configuration{sourcemeta::jsonschema::read_configuration(
77+
options, configuration_path)};
78+
const auto dialect{
79+
sourcemeta::jsonschema::default_dialect(options, configuration)};
80+
const auto &schema_resolver{sourcemeta::jsonschema::resolver(
81+
options, options.contains("http"), dialect, configuration)};
82+
83+
auto test_suite{parse_test_suite(entry, schema_resolver, dialect, false)};
5984

6085
std::cout << entry.first.string() << ":";
6186

62-
const auto suite_result{test_suite->run(
87+
const auto suite_result{test_suite.run(
6388
[&](const sourcemeta::core::JSON::String &, std::size_t index,
6489
std::size_t total, const sourcemeta::blaze::TestCase &test_case,
6590
bool actual, sourcemeta::blaze::TestTimestamp,
@@ -90,20 +115,19 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
90115
std::cout << "\n";
91116
}
92117
} else {
93-
// Re-run with exhaustive mode to get detailed error output
94118
const std::string ref{"$ref"};
95119
sourcemeta::blaze::SimpleOutput output{test_case.data,
96120
{std::cref(ref)}};
97-
test_suite->evaluator.validate(test_suite->schema_exhaustive,
98-
test_case.data, std::ref(output));
121+
test_suite.evaluator.validate(test_suite.schema_exhaustive,
122+
test_case.data, std::ref(output));
99123

100124
if (!verbose) {
101125
std::cout << "\n";
102126
}
103127

104128
std::cout << " " << index << "/" << total << " FAIL "
105129
<< description << "\n\n";
106-
print(output, test_case.tracker, std::cout);
130+
sourcemeta::jsonschema::print(output, test_case.tracker, std::cout);
107131

108132
if (index != total && verbose) {
109133
std::cout << "\n";
@@ -124,8 +148,174 @@ auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
124148
}
125149

126150
if (!result) {
127-
// Report a different exit code for test failures, to
128-
// distinguish them from other errors
129-
throw Fail{2};
151+
throw sourcemeta::jsonschema::Fail{2};
152+
}
153+
}
154+
155+
auto timestamp_to_unix_ms(
156+
const sourcemeta::blaze::TestTimestamp &timestamp,
157+
const std::chrono::system_clock::time_point &system_ref,
158+
const sourcemeta::blaze::TestTimestamp &steady_ref) -> std::int64_t {
159+
const auto offset{timestamp - steady_ref};
160+
const auto unix_time{system_ref + offset};
161+
return std::chrono::duration_cast<std::chrono::milliseconds>(
162+
unix_time.time_since_epoch())
163+
.count();
164+
}
165+
166+
auto duration_ms(const sourcemeta::blaze::TestTimestamp &start,
167+
const sourcemeta::blaze::TestTimestamp &end) -> std::int64_t {
168+
return std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
169+
.count();
170+
}
171+
172+
auto report_as_ctrf(const sourcemeta::core::Options &options) -> void {
173+
bool result{true};
174+
175+
const auto system_ref{std::chrono::system_clock::now()};
176+
const auto steady_ref{std::chrono::steady_clock::now()};
177+
178+
auto ctrf_tests{sourcemeta::core::JSON::make_array()};
179+
std::size_t total_passed{0};
180+
std::size_t total_failed{0};
181+
sourcemeta::blaze::TestTimestamp global_start{};
182+
sourcemeta::blaze::TestTimestamp global_end{};
183+
bool first_suite{true};
184+
185+
for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) {
186+
const auto configuration_path{
187+
sourcemeta::jsonschema::find_configuration(entry.first)};
188+
const auto &configuration{sourcemeta::jsonschema::read_configuration(
189+
options, configuration_path)};
190+
const auto dialect{
191+
sourcemeta::jsonschema::default_dialect(options, configuration)};
192+
const auto &schema_resolver{sourcemeta::jsonschema::resolver(
193+
options, options.contains("http"), dialect, configuration)};
194+
195+
auto test_suite{parse_test_suite(entry, schema_resolver, dialect, true)};
196+
197+
const auto file_path{
198+
sourcemeta::core::weakly_canonical(entry.first).string()};
199+
200+
const auto suite_result{test_suite.run(
201+
[&](const sourcemeta::core::JSON::String &target, std::size_t,
202+
std::size_t, const sourcemeta::blaze::TestCase &test_case,
203+
bool actual, sourcemeta::blaze::TestTimestamp start,
204+
sourcemeta::blaze::TestTimestamp end) {
205+
auto test_object{sourcemeta::core::JSON::make_object()};
206+
207+
const auto &name{test_case.description.empty()
208+
? "<no description>"
209+
: test_case.description};
210+
test_object.assign("name", sourcemeta::core::JSON{name});
211+
212+
const bool passed{test_case.valid == actual};
213+
test_object.assign(
214+
"status", sourcemeta::core::JSON{passed ? "passed" : "failed"});
215+
216+
test_object.assign("duration",
217+
sourcemeta::core::JSON{duration_ms(start, end)});
218+
auto suite{sourcemeta::core::JSON::make_array()};
219+
suite.push_back(sourcemeta::core::JSON{target});
220+
test_object.assign("suite", std::move(suite));
221+
test_object.assign("type", sourcemeta::core::JSON{"unit"});
222+
test_object.assign("filePath", sourcemeta::core::JSON{file_path});
223+
224+
test_object.assign("line",
225+
sourcemeta::core::JSON{static_cast<std::int64_t>(
226+
std::get<0>(test_case.position))});
227+
test_object.assign(
228+
"retries", sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
229+
test_object.assign("flaky", sourcemeta::core::JSON{false});
230+
std::ostringstream thread_id_stream;
231+
thread_id_stream << std::this_thread::get_id();
232+
test_object.assign("threadId",
233+
sourcemeta::core::JSON{thread_id_stream.str()});
234+
235+
if (!passed) {
236+
if (!test_case.valid && actual) {
237+
test_object.assign("message",
238+
sourcemeta::core::JSON{"Passed but was "
239+
"expected to fail"});
240+
} else {
241+
std::ostringstream trace_stream;
242+
const std::string ref{"$ref"};
243+
sourcemeta::blaze::SimpleOutput output{test_case.data,
244+
{std::cref(ref)}};
245+
test_suite.evaluator.validate(test_suite.schema_exhaustive,
246+
test_case.data, std::ref(output));
247+
sourcemeta::jsonschema::print(output, test_case.tracker,
248+
trace_stream);
249+
test_object.assign("trace",
250+
sourcemeta::core::JSON{trace_stream.str()});
251+
}
252+
}
253+
254+
ctrf_tests.push_back(test_object);
255+
})};
256+
257+
if (first_suite) {
258+
global_start = suite_result.start;
259+
first_suite = false;
260+
}
261+
global_end = suite_result.end;
262+
263+
total_passed += suite_result.passed;
264+
total_failed += suite_result.total - suite_result.passed;
265+
266+
if (suite_result.passed != suite_result.total) {
267+
result = false;
268+
}
269+
}
270+
271+
// Build CTRF output
272+
auto summary{sourcemeta::core::JSON::make_object()};
273+
summary.assign("tests", sourcemeta::core::JSON{static_cast<std::int64_t>(
274+
total_passed + total_failed)});
275+
summary.assign("passed", sourcemeta::core::JSON{
276+
static_cast<std::int64_t>(total_passed)});
277+
summary.assign("failed", sourcemeta::core::JSON{
278+
static_cast<std::int64_t>(total_failed)});
279+
summary.assign("pending",
280+
sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
281+
summary.assign("skipped",
282+
sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
283+
summary.assign("other", sourcemeta::core::JSON{static_cast<std::int64_t>(0)});
284+
summary.assign("start", sourcemeta::core::JSON{timestamp_to_unix_ms(
285+
global_start, system_ref, steady_ref)});
286+
summary.assign("stop", sourcemeta::core::JSON{timestamp_to_unix_ms(
287+
global_end, system_ref, steady_ref)});
288+
289+
auto tool{sourcemeta::core::JSON::make_object()};
290+
tool.assign("name", sourcemeta::core::JSON{"jsonschema"});
291+
tool.assign("version", sourcemeta::core::JSON{std::string{
292+
sourcemeta::jsonschema::PROJECT_VERSION}});
293+
294+
auto results{sourcemeta::core::JSON::make_object()};
295+
results.assign("tool", std::move(tool));
296+
results.assign("summary", std::move(summary));
297+
results.assign("tests", std::move(ctrf_tests));
298+
299+
auto ctrf{sourcemeta::core::JSON::make_object()};
300+
ctrf.assign("reportFormat", sourcemeta::core::JSON{"CTRF"});
301+
ctrf.assign("specVersion", sourcemeta::core::JSON{"0.0.0"});
302+
ctrf.assign("results", std::move(results));
303+
304+
sourcemeta::core::prettify(ctrf, std::cout);
305+
std::cout << "\n";
306+
307+
if (!result) {
308+
throw sourcemeta::jsonschema::Fail{2};
309+
}
310+
}
311+
312+
} // namespace
313+
314+
auto sourcemeta::jsonschema::test(const sourcemeta::core::Options &options)
315+
-> void {
316+
if (options.contains("json")) {
317+
report_as_ctrf(options);
318+
} else {
319+
report_as_text(options);
130320
}
131321
}

‎src/main.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ Global Options:
6666
[--ignore/-i <schemas-or-directories>]
6767
6868
Run a set of unit tests against a schema.
69+
Pass --json/-j to output results in CTRF format (https://ctrf.io).
6970
7071
fmt [schemas-or-directories...] [--check/-c] [--extension/-e <extension>]
7172
[--ignore/-i <schemas-or-directories>] [--keep-ordering/-k]

‎test/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ add_jsonschema_test_unix(test/pass_file_target_with_resolve)
271271
add_jsonschema_test_unix(test/pass_file_target_with_resolve_verbose)
272272
add_jsonschema_test_unix(test/pass_file_target_without_resolve)
273273
add_jsonschema_test_unix(test/pass_file_target_without_resolve_verbose)
274+
add_jsonschema_test_unix(test/pass_file_target_without_resolve_json)
274275
add_jsonschema_test_unix(test/pass_file_target_fragment)
275276
add_jsonschema_test_unix(test/pass_single_resolve_remap)
276277
add_jsonschema_test_unix(test/pass_single_resolve_remap_relative)
@@ -283,6 +284,11 @@ add_jsonschema_test_unix(test/pass_target_no_extension_json)
283284
add_jsonschema_test_unix(test/pass_target_no_extension_yaml)
284285
add_jsonschema_test_unix(test/pass_custom_extension_json)
285286
add_jsonschema_test_unix(test/pass_custom_extension_yaml)
287+
add_jsonschema_test_unix(test/pass_single_resolve_json)
288+
add_jsonschema_test_unix(test/pass_single_no_description_json)
289+
add_jsonschema_test_unix(test/fail_true_single_resolve_json)
290+
add_jsonschema_test_unix(test/fail_false_single_resolve_json)
291+
add_jsonschema_test_unix(test/fail_unresolvable_json)
286292

287293
# Bundle
288294
add_jsonschema_test_unix(bundle/pass_into_resolve_directory)

0 commit comments

Comments
 (0)