Skip to content

Commit f2b9500

Browse files
wkingwking
committed
schema: Use Validators map and prepare to extend beyond JSON Schema
With image-tools split off into its own repository, the plan seems to be to keep all intra-blob JSON validation in this repository and to move all other validation (e.g. for layers or for walking Merkle trees) in image-tools [1]. All the non-validation logic currently in image/ is moving into image-tools as well [2]. Some requirements (e.g. multi-parameter checks like allowed OS/arch pairs [3]) are difficult to handle in JSON Schema but easy to handle in Go. And callers won't care if we're using JSON Schema or not; they just want to know if their blob is valid. This commit restructures intra-blob validation to ease the path going forward (although it doesn't actually change the current validation significantly). The old method: func (v Validator) Validate(src io.Reader) error is now a new Validator type: type Validator(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) and instead of instantiating an old Validator instance: schema.MediaTypeImageConfig.Validate(reader) there's a Validators registry mapping from the media type strings to the appropriate Validator instance (which may or may not use JSON Schema under the hood). And there's a Validate function (with the same Validator interface) that looks up the appropriate entry in Validators for you so you have: schema.Validate(reader, descriptor, true) By using a Validators map, we make it easy for library consumers to register (or override) intra-blob validators for a particular type. Locations that call Validate(...) will automatically pick up the new validators without needing local changes. All of the old validation was based on JSON Schema, so currently all Validators values are ValidateJSONSchema. As the schema package grows non-JSON-Schema validation, entries will start to look like: var Validators = map[string]Validator{ v1.MediaTypeImageConfig: ValidateConfig, ... } although ValidateConfig will probably use ValidateJSONSchema internally. By passing through a descriptor, we get a chance to validate the digest and size (which we were not doing before). Digest and size validation for a byte array are also exposed directly (as ValidateByteDigest and ValidateByteSize) for use in validators that are not based on ValidateJSONSchema. Access to the digest also gives us a way to print specific error messages on failures. There is also a new 'strict' parameter to distinguish between compliant images (which should always pass when strict is false) and images that only use features which the spec requires implementations to support (which should only pass if strict is true). The current JSON Schemas are not strict, but the config/layer media type checks in ValidateManifest exercise this distinction. Also use go-digest for local hashing now that we're vendoring it. [1]: http://ircbot.wl.linuxfoundation.org/meetings/opencontainers/2016/opencontainers.2016-10-12-21.01.log.html#l-71 [2]: #337 [3]: https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-5.5.5 [4]: #341 Signed-off-by: W. Trevor King <[email protected]>
1 parent d1c7054 commit f2b9500

File tree

7 files changed

+281
-115
lines changed

7 files changed

+281
-115
lines changed

‎schema/config_test.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ import (
1818
"strings"
1919
"testing"
2020

21+
"github.com/opencontainers/go-digest"
2122
"github.com/opencontainers/image-spec/schema"
23+
"github.com/opencontainers/image-spec/specs-go/v1"
2224
)
2325

2426
func TestConfig(t *testing.T) {
@@ -210,9 +212,14 @@ func TestConfig(t *testing.T) {
210212
fail: false,
211213
},
212214
} {
213-
r := strings.NewReader(tt.config)
214-
err := schema.MediaTypeImageConfig.Validate(r)
215-
215+
configBytes := []byte(tt.config)
216+
reader := strings.NewReader(tt.config)
217+
descriptor := v1.Descriptor{
218+
MediaType: v1.MediaTypeImageConfig,
219+
Digest: digest.FromBytes(configBytes).String(),
220+
Size: int64(len(configBytes)),
221+
}
222+
err := schema.Validate(reader, &descriptor, true)
216223
if got := err != nil; tt.fail != got {
217224
t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
218225
}

‎schema/manifest.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright 2016 The Linux Foundation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package schema
16+
17+
import (
18+
"bytes"
19+
"encoding/json"
20+
"fmt"
21+
"io"
22+
"io/ioutil"
23+
24+
"github.com/opencontainers/image-spec/specs-go/v1"
25+
"github.com/pkg/errors"
26+
)
27+
28+
// ValidateManifest validates the given CAS blob as
29+
// application/vnd.oci.image.manifest.v1+json. Calls
30+
// ValidateJSONSchema as well.
31+
func ValidateManifest(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error) {
32+
if descriptor.MediaType != v1.MediaTypeImageManifest {
33+
return fmt.Errorf("unexpected descriptor media type: %q", descriptor.MediaType)
34+
}
35+
36+
buffer, err := ioutil.ReadAll(blob)
37+
if err != nil {
38+
return errors.Wrapf(err, "unable to read %s", descriptor.Digest)
39+
}
40+
41+
err = ValidateJSONSchema(bytes.NewReader(buffer), descriptor, strict)
42+
if err != nil {
43+
return err
44+
}
45+
46+
header := v1.Manifest{}
47+
err = json.Unmarshal(buffer, &header)
48+
if err != nil {
49+
return errors.Wrap(err, "manifest format mismatch")
50+
}
51+
52+
if header.Config.MediaType != v1.MediaTypeImageConfig {
53+
error := fmt.Errorf("warning: config %s has an unknown media type: %s\n", header.Config.Digest, header.Config.MediaType)
54+
if strict {
55+
return error
56+
}
57+
fmt.Println(error)
58+
}
59+
60+
for _, layer := range header.Layers {
61+
if layer.MediaType != v1.MediaTypeImageLayer &&
62+
layer.MediaType != v1.MediaTypeImageLayerNonDistributable {
63+
error := fmt.Errorf("warning: layer %s has an unknown media type: %s\n", layer.Digest, layer.MediaType)
64+
if strict {
65+
return error
66+
}
67+
fmt.Println(error)
68+
}
69+
}
70+
71+
return nil
72+
}

‎schema/manifest_backwards_compatibility_test.go‎

Lines changed: 42 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,11 @@
1515
package schema_test
1616

1717
import (
18-
"crypto/sha256"
19-
"encoding/hex"
20-
"fmt"
18+
"bytes"
2119
"strings"
2220
"testing"
2321

22+
"github.com/opencontainers/go-digest"
2423
"github.com/opencontainers/image-spec/schema"
2524
"github.com/opencontainers/image-spec/specs-go/v1"
2625
)
@@ -45,13 +44,13 @@ func convertFormats(input string) string {
4544

4645
func TestBackwardsCompatibilityManifestList(t *testing.T) {
4746
for i, tt := range []struct {
48-
manifest string
49-
digest string
50-
fail bool
47+
manifestList string
48+
digest string
49+
fail bool
5150
}{
5251
{
5352
digest: "sha256:219f4b61132fe9d09b0ec5c15517be2ca712e4744b0e0cc3be71295b35b2a467",
54-
manifest: `{
53+
manifestList: `{
5554
"schemaVersion": 2,
5655
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
5756
"manifests": [
@@ -110,16 +109,18 @@ func TestBackwardsCompatibilityManifestList(t *testing.T) {
110109
fail: false,
111110
},
112111
} {
113-
sum := sha256.Sum256([]byte(tt.manifest))
114-
got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
115-
if tt.digest != got {
116-
t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
112+
err := schema.ValidateByteDigest([]byte(tt.manifestList), &v1.Descriptor{Digest: tt.digest})
113+
if err != nil {
114+
t.Fatal(err)
117115
}
118-
119-
manifest := convertFormats(tt.manifest)
120-
r := strings.NewReader(manifest)
121-
err := schema.MediaTypeManifestList.Validate(r)
122-
116+
manifestList := []byte(convertFormats(tt.manifestList))
117+
reader := bytes.NewReader(manifestList)
118+
descriptor := v1.Descriptor{
119+
MediaType: v1.MediaTypeImageManifestList,
120+
Digest: digest.FromBytes(manifestList).String(),
121+
Size: int64(len(manifestList)),
122+
}
123+
err = schema.Validate(reader, &descriptor, true)
123124
if got := err != nil; tt.fail != got {
124125
t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
125126
}
@@ -130,6 +131,7 @@ func TestBackwardsCompatibilityManifest(t *testing.T) {
130131
for i, tt := range []struct {
131132
manifest string
132133
digest string
134+
strict bool
133135
fail bool
134136
}{
135137
// manifest pulled from docker hub using hash value
@@ -170,19 +172,22 @@ func TestBackwardsCompatibilityManifest(t *testing.T) {
170172
}
171173
]
172174
}`,
173-
fail: false,
175+
strict: false, // unrecognized config media type application/octet-stream
176+
fail: false,
174177
},
175178
} {
176-
sum := sha256.Sum256([]byte(tt.manifest))
177-
got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
178-
if tt.digest != got {
179-
t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
179+
err := schema.ValidateByteDigest([]byte(tt.manifest), &v1.Descriptor{Digest: tt.digest})
180+
if err != nil {
181+
t.Fatal(err)
180182
}
181-
182-
manifest := convertFormats(tt.manifest)
183-
r := strings.NewReader(manifest)
184-
err := schema.MediaTypeManifest.Validate(r)
185-
183+
manifest := []byte(convertFormats(tt.manifest))
184+
reader := bytes.NewReader(manifest)
185+
descriptor := v1.Descriptor{
186+
MediaType: v1.MediaTypeImageManifest,
187+
Digest: digest.FromBytes(manifest).String(),
188+
Size: int64(len(manifest)),
189+
}
190+
err = schema.Validate(reader, &descriptor, tt.strict)
186191
if got := err != nil; tt.fail != got {
187192
t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
188193
}
@@ -213,16 +218,18 @@ func TestBackwardsCompatibilityConfig(t *testing.T) {
213218
fail: false,
214219
},
215220
} {
216-
sum := sha256.Sum256([]byte(tt.config))
217-
got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
218-
if tt.digest != got {
219-
t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
221+
err := schema.ValidateByteDigest([]byte(tt.config), &v1.Descriptor{Digest: tt.digest})
222+
if err != nil {
223+
t.Fatal(err)
220224
}
221-
222-
config := convertFormats(tt.config)
223-
r := strings.NewReader(config)
224-
err := schema.MediaTypeImageConfig.Validate(r)
225-
225+
config := []byte(convertFormats(tt.config))
226+
reader := bytes.NewReader(config)
227+
descriptor := v1.Descriptor{
228+
MediaType: v1.MediaTypeImageConfig,
229+
Digest: digest.FromBytes(config).String(),
230+
Size: int64(len(config)),
231+
}
232+
err = schema.Validate(reader, &descriptor, true)
226233
if got := err != nil; tt.fail != got {
227234
t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
228235
}

‎schema/manifest_test.go‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@ import (
1818
"strings"
1919
"testing"
2020

21+
"github.com/opencontainers/go-digest"
2122
"github.com/opencontainers/image-spec/schema"
23+
"github.com/opencontainers/image-spec/specs-go/v1"
2224
)
2325

2426
func TestManifest(t *testing.T) {
2527
for i, tt := range []struct {
2628
manifest string
29+
strict bool
2730
fail bool
2831
}{
2932
// expected failure: mediaType does not match pattern
@@ -46,7 +49,8 @@ func TestManifest(t *testing.T) {
4649
]
4750
}
4851
`,
49-
fail: true,
52+
strict: true,
53+
fail: true,
5054
},
5155

5256
// expected failure: config.size is a string, expected integer
@@ -69,7 +73,8 @@ func TestManifest(t *testing.T) {
6973
]
7074
}
7175
`,
72-
fail: true,
76+
strict: true,
77+
fail: true,
7378
},
7479

7580
// expected failure: layers.size is string, expected integer
@@ -92,7 +97,56 @@ func TestManifest(t *testing.T) {
9297
]
9398
}
9499
`,
95-
fail: true,
100+
strict: true,
101+
fail: true,
102+
},
103+
104+
// expected failure: unrecognized layer media type and strict is true
105+
{
106+
manifest: `
107+
{
108+
"schemaVersion": 2,
109+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
110+
"config": {
111+
"mediaType": "application/vnd.oci.image.config.v1+json",
112+
"size": 1470,
113+
"digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b"
114+
},
115+
"layers": [
116+
{
117+
"mediaType": "application/vnd.other.layer",
118+
"size": "675598",
119+
"digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b"
120+
}
121+
]
122+
}
123+
`,
124+
strict: true,
125+
fail: true,
126+
},
127+
128+
// expected success: unrecognized layer media type, but strict is false
129+
{
130+
manifest: `
131+
{
132+
"schemaVersion": 2,
133+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
134+
"config": {
135+
"mediaType": "application/vnd.oci.image.config.v1+json",
136+
"size": 1470,
137+
"digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b"
138+
},
139+
"layers": [
140+
{
141+
"mediaType": "application/vnd.other.layer",
142+
"size": "675598",
143+
"digest": "sha256:c86f7763873b6c0aae22d963bab59b4f5debbed6685761b5951584f6efb0633b"
144+
}
145+
]
146+
}
147+
`,
148+
strict: false,
149+
fail: true,
96150
},
97151

98152
// valid manifest with optional fields
@@ -129,7 +183,8 @@ func TestManifest(t *testing.T) {
129183
}
130184
}
131185
`,
132-
fail: false,
186+
strict: true,
187+
fail: false,
133188
},
134189

135190
// valid manifest with only required fields
@@ -182,9 +237,14 @@ func TestManifest(t *testing.T) {
182237
fail: true,
183238
},
184239
} {
185-
r := strings.NewReader(tt.manifest)
186-
err := schema.MediaTypeManifest.Validate(r)
187-
240+
manifestBytes := []byte(tt.manifest)
241+
reader := strings.NewReader(tt.manifest)
242+
descriptor := v1.Descriptor{
243+
MediaType: v1.MediaTypeImageManifest,
244+
Digest: digest.FromBytes(manifestBytes).String(),
245+
Size: int64(len(manifestBytes)),
246+
}
247+
err := schema.Validate(reader, &descriptor, tt.strict)
188248
if got := err != nil; tt.fail != got {
189249
t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
190250
}

‎schema/schema.go‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,17 @@ import (
2020
"github.com/opencontainers/image-spec/specs-go/v1"
2121
)
2222

23-
// Media types for the OCI image formats
24-
const (
25-
MediaTypeDescriptor Validator = v1.MediaTypeDescriptor
26-
MediaTypeManifest Validator = v1.MediaTypeImageManifest
27-
MediaTypeManifestList Validator = v1.MediaTypeImageManifestList
28-
MediaTypeImageConfig Validator = v1.MediaTypeImageConfig
29-
MediaTypeImageLayer unimplemented = v1.MediaTypeImageLayer
30-
)
31-
3223
var (
3324
// fs stores the embedded http.FileSystem
3425
// having the OCI JSON schema files in root "/".
3526
fs = _escFS(false)
3627

37-
// specs maps OCI schema media types to schema files.
38-
specs = map[Validator]string{
39-
MediaTypeDescriptor: "content-descriptor.json",
40-
MediaTypeManifest: "image-manifest-schema.json",
41-
MediaTypeManifestList: "manifest-list-schema.json",
42-
MediaTypeImageConfig: "config-schema.json",
28+
// Schemas maps OCI media types to JSON Schema files.
29+
Schemas = map[string]string{
30+
v1.MediaTypeDescriptor: "content-descriptor.json",
31+
v1.MediaTypeImageManifest: "image-manifest-schema.json",
32+
v1.MediaTypeImageManifestList: "manifest-list-schema.json",
33+
v1.MediaTypeImageConfig: "config-schema.json",
4334
}
4435
)
4536

0 commit comments

Comments
 (0)