Skip to content

Commit 9c9719f

Browse files
RST-JRST-J
authored
Merge pull request #377 from iainbeeston/refactor-common-test-suite
Refactor common test suite
2 parents 20734c1 + 06d1d94 commit 9c9719f

File tree

4 files changed

+133
-51
lines changed

4 files changed

+133
-51
lines changed

‎Rakefile‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,14 @@ task :update_common_tests do
99
unless File.read(".git/config").include?('submodule "test/test-suite"')
1010
sh "git submodule init"
1111
end
12-
sh "git submodule update --remote --quiet"
12+
13+
puts "Updating json-schema common test suite..."
14+
15+
begin
16+
sh "git submodule update --remote --quiet"
17+
rescue StandardError
18+
STDERR.puts "Failed to update common test suite."
19+
end
1320
end
1421

1522
Rake::TestTask.new do |t|
@@ -20,6 +27,4 @@ Rake::TestTask.new do |t|
2027
t.test_files = FileList.new('test/*_test.rb')
2128
end
2229

23-
task :test => :update_common_tests
24-
2530
task :default => :test

‎test/common_test_suite_test.rb‎

Lines changed: 15 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,48 +4,7 @@
44
class CommonTestSuiteTest < Minitest::Test
55
TEST_DIR = File.expand_path('../test-suite/tests', __FILE__)
66

7-
# These are test files which we know fail spectacularly, either because we
8-
# don't support that functionality or because they require external
9-
# dependencies. To allow finer-grained control over which tests to run,
10-
# you can replace `:all` with an array containing the names of individual
11-
# tests to skip.
12-
IGNORED_TESTS = Hash.new { |h,k| h[k] = [] }.merge({
13-
"draft3/optional/jsregex.json" => :all,
14-
"draft3/optional/format.json" => [
15-
"validation of regular expressions",
16-
"validation of e-mail addresses",
17-
"validation of URIs",
18-
"validation of host names",
19-
"validation of CSS colors"
20-
],
21-
"draft3/ref.json" => [
22-
"ref overrides any sibling keywords/remote ref valid, maxItems ignored"
23-
],
24-
"draft4/optional/format.json" => [
25-
"validation of URIs",
26-
"validation of e-mail addresses",
27-
"validation of host names"
28-
],
29-
"draft4/optional/ecmascript-regex.json" => [
30-
"ECMA 262 regex non-compliance/ECMA 262 has no support for \\Z anchor from .NET"
31-
],
32-
"draft4/ref.json" => [
33-
"ref overrides any sibling keywords/remote ref valid, maxItems ignored",
34-
"ref overrides any sibling keywords/ref valid, maxItems ignored"
35-
],
36-
"draft6/optional/format.json" => [
37-
"validation of URIs",
38-
"validation of e-mail addresses",
39-
"validation of host names"
40-
],
41-
"draft6/optional/ecmascript-regex.json" => [
42-
"ECMA 262 regex non-compliance/ECMA 262 has no support for \\Z anchor from .NET"
43-
],
44-
"draft6/ref.json" => [
45-
"ref overrides any sibling keywords/remote ref valid, maxItems ignored",
46-
"ref overrides any sibling keywords/ref valid, maxItems ignored"
47-
]
48-
})
7+
IGNORED_TESTS = YAML.load_file(File.expand_path('../support/test_suite_ignored_tests.yml', __FILE__))
498

509
def setup
5110
Dir["#{TEST_DIR}/../remotes/**/*.json"].each do |path|
@@ -55,6 +14,14 @@ def setup
5514
end
5615
end
5716

17+
def self.skip?(current_test, file_path)
18+
skipped_in_file = file_path.chomp('.json').split('/').inject(IGNORED_TESTS) do |ignored, path_component|
19+
ignored.nil? ? nil : ignored[path_component]
20+
end
21+
22+
!skipped_in_file.nil? && (skipped_in_file == :all || skipped_in_file.include?(current_test))
23+
end
24+
5825
Dir["#{TEST_DIR}/*"].each do |suite|
5926
version = File.basename(suite).to_sym
6027
Dir["#{suite}/**/*.json"].each do |tfile|
@@ -66,13 +33,14 @@ def setup
6633
base_description = test["description"]
6734

6835
test["tests"].each do |t|
69-
next if IGNORED_TESTS[rel_file] == :all
70-
next if IGNORED_TESTS[rel_file].any? { |ignored|
71-
base_description == ignored || "#{base_description}/#{t['description']}" == ignored
72-
}
36+
full_description = "#{base_description}/#{t['description']}"
37+
38+
next if rel_file.include?('/optional/') && skip?(full_description, rel_file)
7339

74-
err_id = "#{rel_file}: #{base_description}/#{t['description']}"
40+
err_id = "#{rel_file}: #{full_description}"
7541
define_method("test_#{err_id}") do
42+
skip if self.class.skip?(full_description, rel_file)
43+
7644
errors = JSON::Validator.fully_validate(schema,
7745
t["data"],
7846
:parse_data => false,

‎test/support/test_suite_ignored_tests.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# These are test files which we know fail spectacularly, either because we
2+
# don't support that functionality or because they require external
3+
# dependencies. To allow finer-grained control over which tests to run,
4+
# you can replace `:all` with an array containing the names of individual
5+
# tests to skip.
6+
draft3:
7+
ref:
8+
- ref overrides any sibling keywords/remote ref valid, maxItems ignored
9+
optional:
10+
jsregex: :all
11+
format:
12+
- validation of regular expressions/a regular expression with unclosed parens is invalid
13+
- validation of e-mail addresses/an invalid e-mail address
14+
- validation of URIs/an invalid URI
15+
- validation of URIs/an invalid protocol-relative URI Reference
16+
- validation of URIs/an invalid URI though valid URI reference
17+
- validation of host names/a host name with a component too long
18+
- validation of host names/a host name containing illegal characters
19+
- validation of host names/a host name starting with an illegal character
20+
- validation of CSS colors/an invalid CSS color code
21+
- validation of CSS colors/an invalid CSS color name
22+
- validation of CSS colors/a CSS color name containing invalid characters
23+
draft4:
24+
ref:
25+
- ref overrides any sibling keywords/remote ref valid, maxItems ignored
26+
- ref overrides any sibling keywords/ref valid, maxItems ignored
27+
optional:
28+
format:
29+
- validation of URIs/an invalid URI
30+
- validation of URIs/an invalid protocol-relative URI Reference
31+
- validation of URIs/an invalid URI though valid URI reference
32+
- validation of e-mail addresses/an invalid e-mail address
33+
- validation of host names/a host name with a component too long
34+
- validation of host names/a host name containing illegal characters
35+
- validation of host names/a host name starting with an illegal character
36+
ecmascript-regex:
37+
- ECMA 262 regex non-compliance/ECMA 262 has no support for \Z anchor from .NET
38+
bignum:
39+
- float comparison with high precision on negative numbers/comparison works for very negative numbers
40+
- float comparison with high precision/comparison works for high numbers
41+
draft6:
42+
allOf:
43+
- allOf with boolean schemas, some false/any value is invalid
44+
- allOf with boolean schemas, all false/any value is invalid
45+
- allOf with boolean schemas, all true/any value is valid
46+
anyOf:
47+
- anyOf with boolean schemas, all false/any value is invalid
48+
- anyOf with boolean schemas, all true/any value is valid
49+
- anyOf with boolean schemas, some true/any value is valid
50+
boolean_schema: :all
51+
const: :all
52+
contains: :all
53+
dependencies:
54+
- dependencies with boolean subschemas/empty object is valid
55+
- dependencies with boolean subschemas/object with both properties is invalid
56+
- dependencies with boolean subschemas/object with property having schema false is invalid
57+
- dependencies with boolean subschemas/object with property having schema true is valid
58+
- dependencies with empty array/empty object
59+
- dependencies with empty array/object with one property
60+
exclusiveMaximum: :all
61+
exclusiveMinimum: :all
62+
items:
63+
- items with boolean schema (false)/any non-empty array is invalid
64+
- items with boolean schema (false)/empty array is valid
65+
- items with boolean schema (true)/any array is valid
66+
- items with boolean schema (true)/empty array is valid
67+
- items with boolean schemas/array with one item is valid
68+
- items with boolean schemas/array with two items is invalid
69+
- items with boolean schemas/empty array is valid
70+
not: :all
71+
oneOf:
72+
- oneOf with boolean schemas, all false/any value is invalid
73+
- oneOf with boolean schemas, all true/any value is invalid
74+
- oneOf with boolean schemas, one true/any value is valid
75+
- oneOf with boolean schemas, more than one true/any value is invalid
76+
patternProperties:
77+
- patternProperties with boolean schemas/object with property matching schema false is invalid
78+
- patternProperties with boolean schemas/object with both properties is invalid
79+
- patternProperties with boolean schemas/object with property matching schema true is valid
80+
- patternProperties with boolean schemas/empty object is valid
81+
properties:
82+
- properties with boolean schema/only 'true' property present is valid
83+
- properties with boolean schema/only 'false' property present is invalid
84+
- properties with boolean schema/no property present is valid
85+
- properties with boolean schema/both properties present is invalid
86+
propertyNames: :all
87+
ref:
88+
- ref overrides any sibling keywords/remote ref valid, maxItems ignored
89+
- ref overrides any sibling keywords/ref valid, maxItems ignored
90+
- $ref to boolean schema true/any value is valid
91+
- $ref to boolean schema false/any value is invalid
92+
required:
93+
- required with empty array/property not required
94+
optional:
95+
bignum:
96+
- float comparison with high precision/comparison works for high numbers
97+
- float comparison with high precision on negative numbers/comparison works for very negative numbers
98+
format:
99+
- validation of URIs/an invalid URI
100+
- validation of URIs/an invalid protocol-relative URI Reference
101+
- validation of URIs/an invalid URI though valid URI reference
102+
- validation of e-mail addresses/an invalid e-mail address
103+
- validation of host names/a host name with a component too long
104+
- validation of host names/a host name containing illegal characters
105+
- validation of host names/a host name starting with an illegal character
106+
ecmascript-regex:
107+
- ECMA 262 regex non-compliance/ECMA 262 has no support for \Z anchor from .NET
108+
zeroTerminatedFloats:
109+
- some languages do not distinguish between different types of numeric value/a float is not an integer even without fractional part

‎test/test-suite‎

Submodule test-suite updated 26 files

0 commit comments

Comments
 (0)