image: accept arbitrary file streams

This allows the image library to handle any ReadSeeker, enabling
on-the-fly uncompression.

Also, refactor the validate methods to avoid unnecessary copies.

Signed-off-by: Casey Callendrello <[email protected]>

Author: Casey Callendrello

cmd/oci-image-tool/create.go

@@ -47,7 +47,9 @@ func createHandle(context *cli.Context) error {
 		v.ref = context.String("ref")
 	}
 	if context.IsSet("rootfs") {
-		v.root = context.String("roofs")
+		v.root = context.String("rootfs")
+	} else {
+		v.root = "rootfs"
 	}
 
 	if v.typ == "" {
@@ -64,7 +66,7 @@ func createHandle(context *cli.Context) error {
 		err = image.CreateRuntimeBundleLayout(context.Args()[0], context.Args()[1], v.ref, v.root)
 
 	case image.TypeImage:
-		err = image.CreateRuntimeBundle(context.Args()[0], context.Args()[1], v.ref, v.root)
+		err = image.CreateRuntimeBundleFile(context.Args()[0], context.Args()[1], v.ref, v.root)
 
 	default:
 		err = fmt.Errorf("cannot create %q", v.typ)

cmd/oci-image-tool/unpack.go

@@ -60,7 +60,7 @@ func unpackHandle(context *cli.Context) error {
 		err = image.UnpackLayout(context.Args()[0], context.Args()[1], v.ref)
 
 	case image.TypeImage:
-		err = image.Unpack(context.Args()[0], context.Args()[1], v.ref)
+		err = image.UnpackFile(context.Args()[0], context.Args()[1], v.ref)
 
 	default:
 		err = fmt.Errorf("cannot unpack %q", v.typ)

cmd/oci-image-tool/validate.go

@@ -100,7 +100,7 @@ func validatePath(name string) error {
 	case image.TypeImageLayout:
 		return image.ValidateLayout(name, v.refs, v.stdout)
 	case image.TypeImage:
-		return image.Validate(name, v.refs, v.stdout)
+		return image.ValidateFile(name, v.refs, v.stdout)
 	}
 
 	if len(v.refs) != 0 {

image/descriptor.go

@@ -105,28 +105,23 @@ func (d *descriptor) validate(w walker, mts []string) error {
 		return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType)
 	}
 
-	rc, err := w.Get(*d)
+	parsed, err := digest.Parse(d.Digest)
 	if err != nil {
 		return err
 	}
-	defer rc.Close()
-
-	return d.validateContent(rc)
-}
 
-func (d *descriptor) validateContent(r io.Reader) error {
-	parsed, err := digest.Parse(d.Digest)
+	// Copy the contents of the layer in to the verifier
+	verifier := parsed.Verifier()
+	numBytes, err := w.get(*d, verifier)
 	if err != nil {
 		return err
 	}
 
-	verifier := parsed.Verifier()
-	n, err := io.Copy(verifier, r)
 	if err != nil {
 		return errors.Wrap(err, "error generating hash")
 	}
 
-	if n != d.Size {
+	if numBytes != d.Size {
 		return errors.New("size mismatch")
 	}
 

image/image.go

@@ -17,6 +17,7 @@ package image
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -31,16 +32,24 @@ func ValidateLayout(src string, refs []string, out *log.Logger) error {
 	return validate(newPathWalker(src), refs, out)
 }
 
-// Validate walks through the given .tar file and validates the manifest
-// pointed to by the given refs or returns an error if the validation failed.
-func Validate(tarFile string, refs []string, out *log.Logger) error {
+// ValidateFile opens the tar file given by the filename, then calls ValidateReader
+func ValidateFile(tarFile string, refs []string, out *log.Logger) error {
 	f, err := os.Open(tarFile)
 	if err != nil {
 		return errors.Wrap(err, "unable to open file")
 	}
 	defer f.Close()
 
-	return validate(newTarWalker(tarFile, f), refs, out)
+	return Validate(f, refs, out)
+}
+
+// Validate walks through a tar stream and validates the manifest.
+// * Check that all refs point to extant blobs
+// * Checks that all referred blobs are valid
+// * Checks that mime-types are correct
+// returns error on validation failure
+func Validate(r io.ReadSeeker, refs []string, out *log.Logger) error {
+	return validate(newTarWalker(r), refs, out)
 }
 
 var validRefMediaTypes = []string{
@@ -101,17 +110,23 @@ func UnpackLayout(src, dest, ref string) error {
 	return unpack(newPathWalker(src), dest, ref)
 }
 
-// Unpack walks through the given .tar file and, using the layers specified in
-// the manifest pointed to by the given ref, unpacks all layers in the given
-// destination directory or returns an error if the unpacking failed.
-func Unpack(tarFile, dest, ref string) error {
-	f, err := os.Open(tarFile)
+// UnpackFile opens the file pointed by tarFileName and calls Unpack on it.
+func UnpackFile(tarFileName, dest, ref string) error {
+	f, err := os.Open(tarFileName)
 	if err != nil {
 		return errors.Wrap(err, "unable to open file")
 	}
 	defer f.Close()
 
-	return unpack(newTarWalker(tarFile, f), dest, ref)
+	return Unpack(f, dest, ref)
+}
+
+// Unpack walks through the tar stream and, using the layers specified in
+// the manifest pointed to by the given ref, unpacks all layers in the given
+// destination directory or returns an error if the unpacking failed.
+// The destination will be created if it does not exist.
+func Unpack(r io.ReadSeeker, dest, refName string) error {
+	return unpack(newTarWalker(r), dest, refName)
 }
 
 func unpack(w walker, dest, refName string) error {
@@ -143,17 +158,23 @@ func CreateRuntimeBundleLayout(src, dest, ref, root string) error {
 	return createRuntimeBundle(newPathWalker(src), dest, ref, root)
 }
 
-// CreateRuntimeBundle walks through the given .tar file and
-// creates an OCI runtime bundle in the given destination dest
-// or returns an error if the unpacking failed.
-func CreateRuntimeBundle(tarFile, dest, ref, root string) error {
+// CreateRuntimeBundleFile opens the file pointed by tarFile and calls
+// CreateRuntimeBundle.
+func CreateRuntimeBundleFile(tarFile, dest, ref, root string) error {
 	f, err := os.Open(tarFile)
 	if err != nil {
 		return errors.Wrap(err, "unable to open file")
 	}
 	defer f.Close()
 
-	return createRuntimeBundle(newTarWalker(tarFile, f), dest, ref, root)
+	return createRuntimeBundle(newTarWalker(f), dest, ref, root)
+}
+
+// CreateRuntimeBundle walks through the given tar stream and
+// creates an OCI runtime bundle in the given destination dest
+// or returns an error if the unpacking failed.
+func CreateRuntimeBundle(r io.ReadSeeker, dest, ref, root string) error {
+	return createRuntimeBundle(newTarWalker(r), dest, ref, root)
 }
 
 func createRuntimeBundle(w walker, dest, refName, rootfs string) error {
@@ -190,8 +211,7 @@ func createRuntimeBundle(w walker, dest, refName, rootfs string) error {
 		}
 	}
 
-	err = m.unpack(w, filepath.Join(dest, rootfs))
-	if err != nil {
+	if err = m.unpack(w, filepath.Join(dest, rootfs)); err != nil {
 		return err
 	}
 

image/manifest.go

@@ -94,8 +94,8 @@ func (m *manifest) validate(w walker) error {
 func (m *manifest) unpack(w walker, dest string) (retErr error) {
 	// error out if the dest directory is not empty
 	s, err := ioutil.ReadDir(dest)
-	if err != nil && !os.IsNotExist(err) {
-		return errors.Wrap(err, "unable to open file") // err contains dest
+	if err != nil && !os.IsNotExist(err) { // We'll create the dir later
+		return errors.Wrap(err, "unpack: unable to open dest") // err contains dest
 	}
 	if len(s) > 0 {
 		return fmt.Errorf("%s is not empty", dest)
@@ -121,7 +121,7 @@ func (m *manifest) unpack(w walker, dest string) (retErr error) {
 			}
 
 			if err := unpackLayer(dest, r); err != nil {
-				return errors.Wrap(err, "error extracting layer")
+				return errors.Wrap(err, "unpack: error extracting layer")
 			}
 
 			return errEOW