schema: Use Validators map and prepare to extend beyond JSON Schema

With image-tools split off into its own repository, the plan seems to
be to keep all intra-blob JSON validation in this repository and to
move all other validation (e.g. for layers or for walking Merkle
trees) in image-tools [1].  All the non-validation logic currently in
image/ is moving into image-tools as well [2].

Some requirements (e.g. multi-parameter checks like allowed OS/arch
pairs [3]) are difficult to handle in JSON Schema but easy to handle
in Go.  And callers won't care if we're using JSON Schema or not; they
just want to know if their blob is valid.

This commit restructures intra-blob validation to ease the path going
forward (although it doesn't actually change the current validation
significantly).  The old method:

  func (v Validator) Validate(src io.Reader) error

is now a new Validator type:

  type Validator(blob io.Reader, descriptor *v1.Descriptor, strict bool) (err error)

and instead of instantiating an old Validator instance:

  schema.MediaTypeImageConfig.Validate(reader)

there's a Validators registry mapping from the media type strings to
the appropriate Validator instance (which may or may not use JSON
Schema under the hood).  And there's a Validate function (with the
same Validator interface) that looks up the appropriate entry in
Validators for you so you have:

  schema.Validate(reader, descriptor, true)

By using a Validators map, we make it easy for library consumers to
register (or override) intra-blob validators for a particular type.
Locations that call Validate(...) will automatically pick up the new
validators without needing local changes.

All of the old validation was based on JSON Schema, so currently all
Validators values are ValidateJSONSchema.  As the schema package grows
non-JSON-Schema validation, entries will start to look like:

  var Validators = map[string]Validator{
    v1.MediaTypeImageConfig: ValidateConfig,
    ...
  }

although ValidateConfig will probably use ValidateJSONSchema
internally.

By passing through a descriptor, we get a chance to validate the
digest and size (which we were not doing before).  Digest and size
validation for a byte array are also exposed directly (as
ValidateByteDigest and ValidateByteSize) for use in validators that
are not based on ValidateJSONSchema.  Access to the digest also gives
us a way to print specific error messages on failures.  In situations
where you don't know the blob digest, the new DigestByte will help you
calculate it (for a byte array).

There is also a new 'strict' parameter to distinguish between
compliant images (which should only pass when strict is false) and
images that only use features which the spec requires implementations
to support (which should pass regardless of strict).  The current JSON
Schemas are not strict, and I expect we'll soon gain Go code to handle
the distinction (e.g. [4]).  So the presence of 'strict' in the
Validator type is future-proofing our API and not exposing a
currently-implemented feature.

I've made the minimal sane changes to cmd/ and image/, because we're
dropping them from this repository [2] (and continuing them in
runtime-tools).

[1]: http://ircbot.wl.linuxfoundation.org/meetings/opencontainers/2016/opencontainers.2016-10-12-21.01.log.html#l-71
[2]: opencontainers#337
[3]: https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-5.5.5
[4]: opencontainers#341

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

cmd/oci-image-tool/autodetect.go

@@ -22,6 +22,7 @@ import (
 	"os"
 
 	"github.com/opencontainers/image-spec/schema"
+	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -97,10 +98,10 @@ func autodetect(path string) (string, error) {
 	}
 
 	switch {
-	case header.MediaType == string(schema.MediaTypeManifest):
+	case header.MediaType == v1.MediaTypeImageManifest:
 		return typeManifest, nil
 
-	case header.MediaType == string(schema.MediaTypeManifestList):
+	case header.MediaType == v1.MediaTypeImageManifestList:
 		return typeManifestList, nil
 
 	case header.MediaType == "" && header.SchemaVersion == 0 && header.Config != nil:

cmd/oci-image-tool/validate.go

@@ -15,13 +15,16 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
 
 	"github.com/opencontainers/image-spec/image"
 	"github.com/opencontainers/image-spec/schema"
+	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -136,14 +139,35 @@ func (v *validateCmd) validatePath(name string) error {
 	}
 	defer f.Close()
 
+	blob, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	digest, err := schema.DigestByte(blob, "sha256")
+	if err != nil {
+		return err
+	}
+
+	descriptor := v1.Descriptor{
+		Digest: digest,
+		Size: int64(len(blob)),
+	}
+
 	switch typ {
 	case typeManifest:
-		return schema.MediaTypeManifest.Validate(f)
+		descriptor.MediaType = v1.MediaTypeImageManifest
 	case typeManifestList:
-		return schema.MediaTypeManifestList.Validate(f)
+		descriptor.MediaType = v1.MediaTypeImageManifestList
 	case typeConfig:
-		return schema.MediaTypeImageConfig.Validate(f)
+		descriptor.MediaType = v1.MediaTypeImageConfig
 	}
 
-	return fmt.Errorf("type %q unimplemented", typ)
+	reader := bytes.NewReader(blob)
+	return schema.Validate(reader, &descriptor, true)
 }

image/config.go

@@ -15,7 +15,6 @@
 package image
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -25,7 +24,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/opencontainers/image-spec/schema"
 	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -46,10 +44,6 @@ func findConfig(w walker, d *descriptor) (*config, error) {
 			return errors.Wrapf(err, "%s: error reading config", path)
 		}
 
-		if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil {
-			return errors.Wrapf(err, "%s: config validation failed", path)
-		}
-
 		if err := json.Unmarshal(buf, &c); err != nil {
 			return err
 		}

image/manifest.go

@@ -16,7 +16,6 @@ package image
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
@@ -27,7 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/opencontainers/image-spec/schema"
 	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -51,10 +49,6 @@ func findManifest(w walker, d *descriptor) (*manifest, error) {
 			return errors.Wrapf(err, "%s: error reading manifest", path)
 		}
 
-		if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil {
-			return errors.Wrapf(err, "%s: manifest validation failed", path)
-		}
-
 		if err := json.Unmarshal(buf, &m); err != nil {
 			return err
 		}
@@ -90,7 +84,7 @@ func (m *manifest) validate(w walker) error {
 
 func (m *manifest) unpack(w walker, dest string) error {
 	for _, d := range m.Layers {
-		if d.MediaType != string(schema.MediaTypeImageLayer) {
+		if d.MediaType != v1.MediaTypeImageLayer {
 			continue
 		}
 

schema/config_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/opencontainers/image-spec/schema"
+	"github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func TestConfig(t *testing.T) {
@@ -155,9 +156,19 @@ func TestConfig(t *testing.T) {
 			fail: false,
 		},
 	} {
-		r := strings.NewReader(tt.config)
-		err := schema.MediaTypeImageConfig.Validate(r)
+		configBytes := []byte(tt.config)
+		digest, err := schema.DigestByte(configBytes, "sha256")
+		if err != nil {
+			t.Fatal(err)
+		}
 
+		reader := strings.NewReader(tt.config)
+		descriptor := v1.Descriptor{
+			MediaType: v1.MediaTypeImageConfig,
+			Digest: digest,
+			Size: int64(len(configBytes)),
+		}
+		err = schema.Validate(reader, &descriptor, true)
 		if got := err != nil; tt.fail != got {
 			t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
 		}

schema/descriptor.go

@@ -0,0 +1,43 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+)
+
+// DigestByte computes the digest of a blob using the requested
+// algorithm.
+func DigestByte(data []byte, algorithm string) (digest string, err error){
+	var hasher hash.Hash
+	switch algorithm {
+	case "sha256":
+		hasher = sha256.New()
+	default:
+		return "", fmt.Errorf("unrecognized algorithm: %q", algorithm)
+	}
+
+	_, err = hasher.Write(data)
+	if err != nil {
+		return "", err
+	}
+
+	hashBytes := hasher.Sum(nil)
+	hashHex := hex.EncodeToString(hashBytes[:])
+	return fmt.Sprintf("%s:%s", algorithm, hashHex), nil
+}

schema/manifest_backwards_compatibility_test.go

@@ -15,9 +15,6 @@
 package schema_test
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
 	"strings"
 	"testing"
 
@@ -110,16 +107,14 @@ func TestBackwardsCompatibilityManifestList(t *testing.T) {
 			fail: false,
 		},
 	} {
-		sum := sha256.Sum256([]byte(tt.manifest))
-		got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
-		if tt.digest != got {
-			t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
-		}
-
 		manifest := convertFormats(tt.manifest)
-		r := strings.NewReader(manifest)
-		err := schema.MediaTypeManifestList.Validate(r)
-
+		reader := strings.NewReader(manifest)
+		descriptor := v1.Descriptor{
+			MediaType: v1.MediaTypeImageManifestList,
+			Digest: tt.digest,
+			Size: int64(len(manifest)),
+		}
+		err := schema.Validate(reader, &descriptor, true)
 		if got := err != nil; tt.fail != got {
 			t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
 		}
@@ -173,16 +168,14 @@ func TestBackwardsCompatibilityManifest(t *testing.T) {
 			fail: false,
 		},
 	} {
-		sum := sha256.Sum256([]byte(tt.manifest))
-		got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
-		if tt.digest != got {
-			t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
-		}
-
 		manifest := convertFormats(tt.manifest)
-		r := strings.NewReader(manifest)
-		err := schema.MediaTypeManifest.Validate(r)
-
+		reader := strings.NewReader(manifest)
+		descriptor := v1.Descriptor{
+			MediaType: v1.MediaTypeImageManifest,
+			Digest: tt.digest,
+			Size: int64(len(manifest)),
+		}
+		err := schema.Validate(reader, &descriptor, true)
 		if got := err != nil; tt.fail != got {
 			t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
 		}
@@ -213,16 +206,14 @@ func TestBackwardsCompatibilityConfig(t *testing.T) {
 			fail:     false,
 		},
 	} {
-		sum := sha256.Sum256([]byte(tt.manifest))
-		got := fmt.Sprintf("sha256:%s", hex.EncodeToString(sum[:]))
-		if tt.digest != got {
-			t.Errorf("test %d: expected digest %s but got %s", i, tt.digest, got)
-		}
-
 		manifest := convertFormats(tt.manifest)
-		r := strings.NewReader(manifest)
-		err := schema.MediaTypeImageConfig.Validate(r)
-
+		reader := strings.NewReader(manifest)
+		descriptor := v1.Descriptor{
+			MediaType: v1.MediaTypeImageConfig,
+			Digest: tt.digest,
+			Size: int64(len(manifest)),
+		}
+		err := schema.Validate(reader, &descriptor, true)
 		if got := err != nil; tt.fail != got {
 			t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
 		}

schema/manifest_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/opencontainers/image-spec/schema"
+	"github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func TestManifest(t *testing.T) {
@@ -114,9 +115,19 @@ func TestManifest(t *testing.T) {
 			fail: false,
 		},
 	} {
-		r := strings.NewReader(tt.manifest)
-		err := schema.MediaTypeManifest.Validate(r)
+		manifestBytes := []byte(tt.manifest)
+		digest, err := schema.DigestByte(manifestBytes, "sha256")
+		if err != nil {
+			t.Fatal(err)
+		}
 
+		reader := strings.NewReader(tt.manifest)
+		descriptor := v1.Descriptor{
+			MediaType: v1.MediaTypeImageManifest,
+			Digest: digest,
+			Size: int64(len(manifestBytes)),
+		}
+		err = schema.Validate(reader, &descriptor, true)
 		if got := err != nil; tt.fail != got {
 			t.Errorf("test %d: expected validation failure %t but got %t, err %v", i, tt.fail, got, err)
 		}

schema/schema.go

@@ -20,26 +20,17 @@ import (
 	"github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// Media types for the OCI image formats
-const (
-	MediaTypeDescriptor   Validator     = v1.MediaTypeDescriptor
-	MediaTypeManifest     Validator     = v1.MediaTypeImageManifest
-	MediaTypeManifestList Validator     = v1.MediaTypeImageManifestList
-	MediaTypeImageConfig  Validator     = v1.MediaTypeImageConfig
-	MediaTypeImageLayer   unimplemented = v1.MediaTypeImageLayer
-)
-
 var (
 	// fs stores the embedded http.FileSystem
 	// having the OCI JSON schema files in root "/".
 	fs = _escFS(false)
 
-	// specs maps OCI schema media types to schema files.
-	specs = map[Validator]string{
-		MediaTypeDescriptor:   "content-descriptor.json",
-		MediaTypeManifest:     "image-manifest-schema.json",
-		MediaTypeManifestList: "manifest-list-schema.json",
-		MediaTypeImageConfig:  "config-schema.json",
+	// Schemas maps OCI media types to JSON Schema files.
+	Schemas = map[string]string{
+		v1.MediaTypeDescriptor:   "content-descriptor.json",
+		v1.MediaTypeImageManifest:     "image-manifest-schema.json",
+		v1.MediaTypeImageManifestList: "manifest-list-schema.json",
+		v1.MediaTypeImageConfig:  "config-schema.json",
 	}
 )
 

schema/spec_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 
 	"github.com/opencontainers/image-spec/schema"
+	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/russross/blackfriday"
 )
@@ -73,7 +74,19 @@ func validate(t *testing.T, name string) {
 			continue
 		}
 
-		err = schema.Validator(example.Mediatype).Validate(strings.NewReader(example.Body))
+		bodyBytes := []byte(example.Body)
+		digest, err := schema.DigestByte(bodyBytes, "sha256")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		reader := strings.NewReader(example.Body)
+		descriptor := v1.Descriptor{
+			MediaType: example.Mediatype,
+			Digest: digest,
+			Size: int64(len(bodyBytes)),
+		}
+		err = schema.Validate(reader, &descriptor, true)
 		if err == nil {
 			printFields(t, "ok", example.Mediatype, example.Title)
 			t.Log(example.Body, "---")