image: Refactor to use cas/ref engines instead of walkers

The validation/unpacking code doesn't really care what the reference
and CAS implemenations are.  And the new generic interfaces in
image/refs and image/cas will scale better as we add new backends than
the walker interface.  This replaces the simpler interface from
image/reader.go with something more robust.

The old tar/directory distinction between image and imageLayout is
gone.  The new CAS/refs engines don't support directory backends yet
(I plan on adding them once the engine framework lands), but the new
framework will handle tar/directory/... detection inside
layout.NewEngine (and possibly inside a new (cas|refs).NewEngine when
we grow engine types that aren't based on image-layout).

Also replace the old methods like:

  func (d *descriptor) validateContent(r io.Reader) error

with functions like:

  validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error

to avoid local types that duplicate the image-spec types.  This saves
an extra instantiation for folks who want to validate (or whatever) a
specs.Descriptor they have obtained elsewhere.

I'd prefer casLayout and refsLayout for the imported packages, but
Stephen doesn't want camelCase for package names [1].

[1]: opencontainers/image-spec#159 (comment)

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

cmd/oci-create-runtime-bundle/main.go

@@ -23,6 +23,7 @@ import (
 	specs "github.com/opencontainers/image-spec/specs-go"
 	"github.com/opencontainers/image-tools/image"
 	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 )
 
 // gitCommit will be the hash that the binary was built from
@@ -31,7 +32,6 @@ var gitCommit = ""
 
 // supported bundle types
 var bundleTypes = []string{
-	image.TypeImageLayout,
 	image.TypeImage,
 }
 
@@ -109,6 +109,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	ctx := context.Background()
+
 	if _, err := os.Stat(args[1]); os.IsNotExist(err) {
 		v.stderr.Printf("destination path %s does not exist", args[1])
 		os.Exit(1)
@@ -125,11 +127,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) {
 
 	var err error
 	switch v.typ {
-	case image.TypeImageLayout:
-		err = image.CreateRuntimeBundleLayout(args[0], args[1], v.ref, v.root)
-
 	case image.TypeImage:
-		err = image.CreateRuntimeBundle(args[0], args[1], v.ref, v.root)
+		err = image.CreateRuntimeBundle(ctx, args[0], args[1], v.ref, v.root)
 	}
 
 	if err != nil {

cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md

@@ -24,7 +24,7 @@ runtime-spec-compatible `dest/config.json`.
   A directory representing the root filesystem of the container in the OCI runtime bundle. It is strongly recommended to keep the default value. (default "rootfs")
 
 **--type**
-  Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "imageLayout,image"
+  Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "image"
 
 # EXAMPLES
 ```

cmd/oci-image-validate/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/opencontainers/image-tools/image"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 )
 
 // gitCommit will be the hash that the binary was built from
@@ -33,7 +34,6 @@ var gitCommit = ""
 
 // supported validation types
 var validateTypes = []string{
-	image.TypeImageLayout,
 	image.TypeImage,
 	image.TypeManifest,
 	image.TypeManifestList,
@@ -81,7 +81,7 @@ func newValidateCmd(stdout, stderr *log.Logger) *cobra.Command {
 
 	cmd.Flags().StringSliceVar(
 		&v.refs, "ref", nil,
-		`A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout.`,
+		`A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image.`,
 	)
 
 	cmd.Flags().BoolVar(
@@ -107,9 +107,11 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	ctx := context.Background()
+
 	var exitcode int
 	for _, arg := range args {
-		err := v.validatePath(arg)
+		err := v.validatePath(ctx, arg)
 
 		if err == nil {
 			v.stdout.Printf("%s: OK", arg)
@@ -139,7 +141,7 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) {
 	os.Exit(exitcode)
 }
 
-func (v *validateCmd) validatePath(name string) error {
+func (v *validateCmd) validatePath(ctx context.Context, name string) error {
 	var (
 		err error
 		typ = v.typ
@@ -152,10 +154,8 @@ func (v *validateCmd) validatePath(name string) error {
 	}
 
 	switch typ {
-	case image.TypeImageLayout:
-		return image.ValidateLayout(name, v.refs, v.stdout)
 	case image.TypeImage:
-		return image.Validate(name, v.refs, v.stdout)
+		return image.Validate(ctx, name, v.refs, v.stdout)
 	}
 
 	f, err := os.Open(name)

cmd/oci-image-validate/oci-image-validate.1.md

@@ -20,15 +20,15 @@ oci-image-validate \- Validate one or more image files
   Can be specified multiple times to validate multiple references.
   `NAME` must be present in the `refs` subdirectory of the image.
   Defaults to `v1.0`.
-  Only applicable if type is image or imageLayout.
+  Only applicable if type is image.
 
 **--type**
-  Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "imageLayout,image,manifest,manifestList,config"
+  Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "image,manifest,manifestList,config"
 
 # EXAMPLES
 ```
 $ skopeo copy docker://busybox oci:busybox-oci
-$ oci-image-validate --type imageLayout --ref latest busybox-oci
+$ oci-image-validate --type image --ref latest busybox-oci
 busybox-oci: OK
 ```
 

cmd/oci-unpack/main.go

@@ -23,6 +23,7 @@ import (
 	specs "github.com/opencontainers/image-spec/specs-go"
 	"github.com/opencontainers/image-tools/image"
 	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 )
 
 // gitCommit will be the hash that the binary was built from
@@ -31,7 +32,6 @@ var gitCommit = ""
 
 // supported unpack types
 var unpackTypes = []string{
-	image.TypeImageLayout,
 	image.TypeImage,
 }
 
@@ -62,8 +62,8 @@ func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "unpack [src] [dest]",
-		Short: "Unpack an image or image source layout",
-		Long:  `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`,
+		Short: "Unpack an image",
+		Long:  `Unpack the OCI image present at [src] to the destination directory [dest].`,
 		Run:   v.Run,
 	}
 
@@ -101,6 +101,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	ctx := context.Background()
+
 	if v.typ == "" {
 		typ, err := image.Autodetect(args[0])
 		if err != nil {
@@ -112,11 +114,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) {
 
 	var err error
 	switch v.typ {
-	case image.TypeImageLayout:
-		err = image.UnpackLayout(args[0], args[1], v.ref)
-
 	case image.TypeImage:
-		err = image.Unpack(args[0], args[1], v.ref)
+		err = image.Unpack(ctx, args[0], args[1], v.ref)
 	}
 
 	if err != nil {

cmd/oci-unpack/oci-unpack.1.md

@@ -18,7 +18,7 @@ oci-unpack \- Unpack an image or image source layout
   The ref pointing to the manifest to be unpacked. This must be present in the "refs" subdirectory of the image. (default "v1.0")
 
 **--type**
-  Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "imageLayout,image"
+  Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "image"
 
 # EXAMPLES
 ```

image/autodetect.go

@@ -27,7 +27,6 @@ import (
 
 // supported autodetection types
 const (
-	TypeImageLayout  = "imageLayout"
 	TypeImage        = "image"
 	TypeManifest     = "manifest"
 	TypeManifestList = "manifestList"
@@ -43,7 +42,7 @@ func Autodetect(path string) (string, error) {
 	}
 
 	if fi.IsDir() {
-		return TypeImageLayout, nil
+		return TypeImage, nil
 	}
 
 	f, err := os.Open(path)

image/config.go

@@ -18,63 +18,64 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/opencontainers/image-spec/schema"
+	imagespecs "github.com/opencontainers/image-spec/specs-go"
 	"github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/image-tools/image/cas"
+	runtimespecs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
+	"golang.org/x/net/context"
 )
 
-type config v1.Image
+func findConfig(ctx context.Context, engine cas.Engine, descriptor *imagespecs.Descriptor) (config *v1.Image, err error) {
+	err = validateMediaType(descriptor.MediaType, []string{v1.MediaTypeImageConfig})
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid config media type")
+	}
 
-func findConfig(w walker, d *descriptor) (*config, error) {
-	var c config
-	cpath := filepath.Join("blobs", d.algo(), d.hash())
+	err = validateDescriptor(ctx, engine, descriptor)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid config descriptor")
+	}
 
-	switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error {
-		if info.IsDir() || filepath.Clean(path) != cpath {
-			return nil
-		}
-		buf, err := ioutil.ReadAll(r)
-		if err != nil {
-			return errors.Wrapf(err, "%s: error reading config", path)
-		}
+	reader, err := engine.Get(ctx, descriptor.Digest)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest)
+	}
 
-		if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil {
-			return errors.Wrapf(err, "%s: config validation failed", path)
-		}
+	buf, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest)
+	}
 
-		if err := json.Unmarshal(buf, &c); err != nil {
-			return err
-		}
-		// check if the rootfs type is 'layers'
-		if c.RootFS.Type != "layers" {
-			return fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type)
-		}
-		return errEOW
-	}); err {
-	case nil:
-		return nil, fmt.Errorf("%s: config not found", cpath)
-	case errEOW:
-		return &c, nil
-	default:
+	if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil {
+		return nil, errors.Wrapf(err, "%s: config validation failed", descriptor.Digest)
+	}
+
+	var c v1.Image
+	if err := json.Unmarshal(buf, &c); err != nil {
 		return nil, err
 	}
+
+	// check if the rootfs type is 'layers'
+	if c.RootFS.Type != "layers" {
+		return nil, fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type)
+	}
+
+	return &c, nil
 }
 
-func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) {
+func runtimeSpec(c *v1.Image, rootfs string) (*runtimespecs.Spec, error) {
 	if c.OS != "linux" {
 		return nil, fmt.Errorf("%s: unsupported OS", c.OS)
 	}
 
-	var s specs.Spec
-	s.Version = specs.Version
+	var s runtimespecs.Spec
+	s.Version = runtimespecs.Version
 	// we should at least apply the default spec, otherwise this is totally useless
 	s.Process.Terminal = true
 	s.Root.Path = rootfs
@@ -116,12 +117,12 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) {
 	swap := uint64(c.Config.MemorySwap)
 	shares := uint64(c.Config.CPUShares)
 
-	s.Linux.Resources = &specs.Resources{
-		CPU: &specs.CPU{
+	s.Linux.Resources = &runtimespecs.Resources{
+		CPU: &runtimespecs.CPU{
 			Shares: &shares,
 		},
 
-		Memory: &specs.Memory{
+		Memory: &runtimespecs.Memory{
 			Limit:       &mem,
 			Reservation: &mem,
 			Swap:        &swap,
@@ -131,7 +132,7 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) {
 	for vol := range c.Config.Volumes {
 		s.Mounts = append(
 			s.Mounts,
-			specs.Mount{
+			runtimespecs.Mount{
 				Destination: vol,
 				Type:        "bind",
 				Options:     []string{"rbind"},