merge branch 'store-unpack-arguments'

Implements #14
Signed-off-by: Aleksa Sarai <[email protected]>

Author: cyphar

cmd/umoci/repack.go

@@ -30,7 +30,6 @@ import (
 	"github.com/cyphar/umoci/image/cas"
 	"github.com/cyphar/umoci/image/generator"
 	"github.com/cyphar/umoci/image/layer"
-	"github.com/cyphar/umoci/pkg/idtools"
 	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/urfave/cli"
 	"github.com/vbatts/go-mtree"
@@ -40,11 +39,10 @@ import (
 var repackCommand = cli.Command{
 	Name:  "repack",
 	Usage: "repacks an OCI runtime bundle into a reference",
-	ArgsUsage: `--image <image-path> --from <reference> --bundle <bundle-path>
+	ArgsUsage: `--image <image-path> --bundle <bundle-path>
 
-Where "<image-path>" is the path to the OCI image, "<reference>" is the name of
-the reference descriptor which was used to generate the original runtime bundle
-and "<bundle-path>" is the destination to repack the image to.
+Where "<image-path>" is the path to the OCI image, and "<bundle-path>" is the
+bundle from which to generate the required layers.
 
 It should be noted that this is not the same as oci-create-layer because it
 uses go-mtree to create diff layers from runtime bundles unpacked with
@@ -57,10 +55,6 @@ manifest and configuration information uses the new diff atop the old manifest.`
 			Name:  "image",
 			Usage: "path to OCI image bundle",
 		},
-		cli.StringFlag{
-			Name:  "from",
-			Usage: "reference descriptor name to repack",
-		},
 		cli.StringFlag{
 			Name:  "bundle",
 			Usage: "destination bundle path",
@@ -69,18 +63,6 @@ manifest and configuration information uses the new diff atop the old manifest.`
 			Name:  "tag",
 			Usage: "tag name for repacked image",
 		},
-		cli.StringSliceFlag{
-			Name:  "uid-map",
-			Usage: "specifies a uid mapping to use when repacking",
-		},
-		cli.StringSliceFlag{
-			Name:  "gid-map",
-			Usage: "specifies a gid mapping to use when repacking",
-		},
-		cli.BoolFlag{
-			Name:  "rootless",
-			Usage: "enable rootless unpacking support",
-		},
 	},
 
 	Action: repack,
@@ -96,48 +78,23 @@ func repack(ctx *cli.Context) error {
 	if bundlePath == "" {
 		return fmt.Errorf("bundle path cannot be empty")
 	}
-	fromName := ctx.String("from")
-	if fromName == "" {
-		return fmt.Errorf("reference name cannot be empty")
-	}
 
-	// Parse map options.
-	mapOptions := layer.MapOptions{
-		Rootless: ctx.Bool("rootless"),
-	}
-	// We need to set mappings if we're in rootless mode.
-	if mapOptions.Rootless {
-		if !ctx.IsSet("uid-map") {
-			ctx.Set("uid-map", fmt.Sprintf("%d:0:1", os.Geteuid()))
-			logrus.WithFields(logrus.Fields{
-				"map.uid": ctx.StringSlice("uid-map"),
-			}).Info("setting default rootless --uid-map option")
-		}
-		if !ctx.IsSet("gid-map") {
-			ctx.Set("gid-map", fmt.Sprintf("%d:0:1", os.Getegid()))
-			logrus.WithFields(logrus.Fields{
-				"map.gid": ctx.StringSlice("gid-map"),
-			}).Info("setting default rootless --gid-map option")
-		}
-	}
-	for _, uidmap := range ctx.StringSlice("uid-map") {
-		idMap, err := idtools.ParseMapping(uidmap)
-		if err != nil {
-			return fmt.Errorf("failure parsing --uid-map %s: %s", uidmap, err)
-		}
-		mapOptions.UIDMappings = append(mapOptions.UIDMappings, idMap)
-	}
-	for _, gidmap := range ctx.StringSlice("gid-map") {
-		idMap, err := idtools.ParseMapping(gidmap)
-		if err != nil {
-			return fmt.Errorf("failure parsing --gid-map %s: %s", gidmap, err)
-		}
-		mapOptions.GIDMappings = append(mapOptions.GIDMappings, idMap)
+	// Read the metadata first.
+	meta, err := ReadBundleMeta(bundlePath)
+	if err != nil {
+		return fmt.Errorf("error reading umoci.json metadata: %s", err)
 	}
+
 	logrus.WithFields(logrus.Fields{
-		"map.uid": mapOptions.UIDMappings,
-		"map.gid": mapOptions.GIDMappings,
-	}).Infof("parsed mappings")
+		"version":     meta.Version,
+		"from":        meta.From,
+		"map_options": meta.MapOptions,
+	}).Debugf("umoci: loaded UmociMeta metadata")
+
+	// FIXME: Implement support for manifest lists.
+	if meta.From.MediaType != v1.MediaTypeImageManifest {
+		return fmt.Errorf("--from descriptor does not point to v1.MediaTypeImageManifest: not implemented: %s", meta.From.MediaType)
+	}
 
 	// Get a reference to the CAS.
 	engine, err := cas.Open(imagePath)
@@ -146,24 +103,13 @@ func repack(ctx *cli.Context) error {
 	}
 	defer engine.Close()
 
-	fromDescriptor, err := engine.GetReference(context.TODO(), fromName)
-	if err != nil {
-		return err
-	}
-
-	// FIXME: Implement support for manifest lists.
-	if fromDescriptor.MediaType != v1.MediaTypeImageManifest {
-		return fmt.Errorf("--from descriptor does not point to v1.MediaTypeImageManifest: not implemented: %s", fromDescriptor.MediaType)
-	}
-
-	mtreeName := strings.Replace(fromDescriptor.Digest, "sha256:", "sha256_", 1)
+	mtreeName := strings.Replace(meta.From.Digest, "sha256:", "sha256_", 1)
 	mtreePath := filepath.Join(bundlePath, mtreeName+".mtree")
 	fullRootfsPath := filepath.Join(bundlePath, layer.RootfsName)
 
 	logrus.WithFields(logrus.Fields{
 		"image":  imagePath,
 		"bundle": bundlePath,
-		"ref":    fromName,
 		"rootfs": layer.RootfsName,
 		"mtree":  mtreePath,
 	}).Debugf("umoci: repacking OCI image")
@@ -185,7 +131,7 @@ func repack(ctx *cli.Context) error {
 		"keywords": keywords,
 	}).Debugf("umoci: parsed mtree spec")
 
-	diffs, err := mtree.Check(fullRootfsPath, spec, keywords, mapOptions.Rootless)
+	diffs, err := mtree.Check(fullRootfsPath, spec, keywords, meta.MapOptions.Rootless)
 	if err != nil {
 		return err
 	}
@@ -194,7 +140,7 @@ func repack(ctx *cli.Context) error {
 		"ndiff": len(diffs),
 	}).Debugf("umoci: checked mtree spec")
 
-	reader, err := layer.GenerateLayer(fullRootfsPath, diffs, &mapOptions)
+	reader, err := layer.GenerateLayer(fullRootfsPath, diffs, &meta.MapOptions)
 	if err != nil {
 		return err
 	}
@@ -250,7 +196,7 @@ func repack(ctx *cli.Context) error {
 		"size":   layerSize,
 	}).Debugf("umoci: generated new diff layer")
 
-	manifestBlob, err := cas.FromDescriptor(context.TODO(), engine, fromDescriptor)
+	manifestBlob, err := cas.FromDescriptor(context.TODO(), engine, &meta.From)
 	if err != nil {
 		return err
 	}
@@ -331,6 +277,7 @@ func repack(ctx *cli.Context) error {
 		"size":      newDescriptor.Size,
 	}).Infof("created new image")
 
+	// FIXME: This should be mandatory.
 	tagName := ctx.String("tag")
 	if tagName == "" {
 		return nil

cmd/umoci/unpack.go

@@ -111,12 +111,13 @@ func unpack(ctx *cli.Context) error {
 		return fmt.Errorf("reference name cannot be empty")
 	}
 
+	var meta UmociMeta
+	meta.Version = ctx.App.Version
+
 	// Parse map options.
-	mapOptions := layer.MapOptions{
-		Rootless: ctx.Bool("rootless"),
-	}
 	// We need to set mappings if we're in rootless mode.
-	if mapOptions.Rootless {
+	meta.MapOptions.Rootless = ctx.Bool("rootless")
+	if meta.MapOptions.Rootless {
 		if !ctx.IsSet("uid-map") {
 			ctx.Set("uid-map", fmt.Sprintf("%d:0:1", os.Geteuid()))
 			logrus.WithFields(logrus.Fields{
@@ -136,18 +137,18 @@ func unpack(ctx *cli.Context) error {
 		if err != nil {
 			return fmt.Errorf("failure parsing --uid-map %s: %s", uidmap, err)
 		}
-		mapOptions.UIDMappings = append(mapOptions.UIDMappings, idMap)
+		meta.MapOptions.UIDMappings = append(meta.MapOptions.UIDMappings, idMap)
 	}
 	for _, gidmap := range ctx.StringSlice("gid-map") {
 		idMap, err := idtools.ParseMapping(gidmap)
 		if err != nil {
 			return fmt.Errorf("failure parsing --gid-map %s: %s", gidmap, err)
 		}
-		mapOptions.GIDMappings = append(mapOptions.GIDMappings, idMap)
+		meta.MapOptions.GIDMappings = append(meta.MapOptions.GIDMappings, idMap)
 	}
 	logrus.WithFields(logrus.Fields{
-		"map.uid": mapOptions.UIDMappings,
-		"map.gid": mapOptions.GIDMappings,
+		"map.uid": meta.MapOptions.UIDMappings,
+		"map.gid": meta.MapOptions.GIDMappings,
 	}).Infof("parsed mappings")
 
 	// Get a reference to the CAS.
@@ -161,18 +162,20 @@ func unpack(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	manifestBlob, err := cas.FromDescriptor(context.TODO(), engine, fromDescriptor)
+	meta.From = *fromDescriptor
+
+	manifestBlob, err := cas.FromDescriptor(context.TODO(), engine, &meta.From)
 	if err != nil {
 		return err
 	}
 	defer manifestBlob.Close()
 
 	// FIXME: Implement support for manifest lists.
 	if manifestBlob.MediaType != v1.MediaTypeImageManifest {
-		return fmt.Errorf("--from descriptor does not point to v1.MediaTypeImageManifest: not implemented: %s", fromDescriptor.MediaType)
+		return fmt.Errorf("--from descriptor does not point to v1.MediaTypeImageManifest: not implemented: %s", meta.From.MediaType)
 	}
 
-	mtreeName := strings.Replace(fromDescriptor.Digest, "sha256:", "sha256_", 1)
+	mtreeName := strings.Replace(meta.From.Digest, "sha256:", "sha256_", 1)
 	mtreePath := filepath.Join(bundlePath, mtreeName+".mtree")
 	fullRootfsPath := filepath.Join(bundlePath, layer.RootfsName)
 
@@ -188,7 +191,7 @@ func unpack(ctx *cli.Context) error {
 
 	// Unpack the runtime bundle.
 	if err := os.MkdirAll(bundlePath, 0755); err != nil {
-		return fmt.Errorf("failed to create bundle path: %q", err)
+		return fmt.Errorf("failed to create bundle path: %s", err)
 	}
 	// XXX: We should probably defer os.RemoveAll(bundlePath).
 
@@ -205,8 +208,8 @@ func unpack(ctx *cli.Context) error {
 	//        extraction). I'm considering reimplementing it just so there are
 	//        competing implementations of this extraction functionality.
 	//           https://github.com/opencontainers/image-tools/issues/74
-	if err := layer.UnpackManifest(context.TODO(), engine, bundlePath, *manifest, &mapOptions); err != nil {
-		return fmt.Errorf("failed to create runtime bundle: %q", err)
+	if err := layer.UnpackManifest(context.TODO(), engine, bundlePath, *manifest, &meta.MapOptions); err != nil {
+		return fmt.Errorf("failed to create runtime bundle: %s", err)
 	}
 
 	// Create the mtree manifest.
@@ -223,21 +226,31 @@ func unpack(ctx *cli.Context) error {
 		"mtree":    mtreePath,
 	}).Debugf("umoci: generating mtree manifest")
 
-	dh, err := mtree.Walk(fullRootfsPath, nil, keywords, mapOptions.Rootless)
+	dh, err := mtree.Walk(fullRootfsPath, nil, keywords, meta.MapOptions.Rootless)
 	if err != nil {
-		return fmt.Errorf("failed to generate mtree spec: %q", err)
+		return fmt.Errorf("failed to generate mtree spec: %s", err)
 	}
 
 	fh, err := os.OpenFile(mtreePath, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
-		return fmt.Errorf("failed to open mtree spec for writing: %q", err)
+		return fmt.Errorf("failed to open mtree spec for writing: %s", err)
 	}
 	defer fh.Close()
 
 	logrus.Debugf("umoci: saving mtree manifest")
 
 	if _, err := dh.WriteTo(fh); err != nil {
-		return fmt.Errorf("failed to write mtree spec: %q", err)
+		return fmt.Errorf("failed to write mtree spec: %s", err)
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"version":     meta.Version,
+		"from":        meta.From,
+		"map_options": meta.MapOptions,
+	}).Debugf("umoci: saving UmociMeta metadata")
+
+	if err := WriteBundleMeta(bundlePath, meta); err != nil {
+		return fmt.Errorf("failed to write umoci.json metadata: %s", err)
 	}
 
 	logrus.Debugf("umoci: unpacking complete")

cmd/umoci/utils.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/cyphar/umoci/image/layer"
+	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// FIXME: This should be moved to a library. Too much of this code is in the
+//        cmd/... code, but should really be refactored to the point where it
+//        can be useful to other people. This is _particularly_ true for the
+//        code which repacks images (the changes to the config, manifest and
+//        CAS should be made into a library).
+
+// UmociMetaName is the name of umoci's metadata file that is stored in all
+// bundles extracted by umoci.
+const UmociMetaName = "umoci.json"
+
+// UmociMeta represents metadata about how umoci unpacked an image to a bundle
+// and other similar information. It is used to keep track of information that
+// is required when repacking an image and other similar bundle information.
+type UmociMeta struct {
+	// Version is the version of umoci used to unpack the bundle. This is used
+	// to future-proof the umoci.json information.
+	Version string `json:"umoci_version"`
+
+	// From is a copy of the descriptor pointing to the image manifest that was
+	// used to unpack the bundle. Essentially it's a resolved form of the
+	// --from argument to umoci-unpack(1).
+	From ispec.Descriptor `json:"from_descriptor"`
+
+	// MapOptions is the parsed version of --uid-map, --gid-map and --rootless
+	// arguments to umoci-unpack(1). While all of these options technically do
+	// not need to be the same for corresponding umoci-unpack(1) and
+	// umoci-repack(1) calls, changing them is not recommended and so the
+	// default should be that they are the same.
+	MapOptions layer.MapOptions `json:"map_options"`
+}
+
+// WriteTo writes a JSON-serialised version of UmociMeta to the given io.Writer.
+func (m UmociMeta) WriteTo(w io.Writer) (int64, error) {
+	buf := new(bytes.Buffer)
+	err := json.NewEncoder(io.MultiWriter(buf, w)).Encode(m)
+	return int64(buf.Len()), err
+}
+
+// WriteBundleMeta writes an umoci.json file to the given bundle path.
+func WriteBundleMeta(bundle string, meta UmociMeta) error {
+	fh, err := os.Create(filepath.Join(bundle, UmociMetaName))
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	_, err = meta.WriteTo(fh)
+	return err
+}
+
+// ReadBundleMeta reads and parses the umoci.json file from a given bundle path.
+func ReadBundleMeta(bundle string) (UmociMeta, error) {
+	var meta UmociMeta
+
+	fh, err := os.Open(filepath.Join(bundle, UmociMetaName))
+	if err != nil {
+		return meta, err
+	}
+	defer fh.Close()
+
+	err = json.NewDecoder(fh).Decode(&meta)
+	return meta, err
+}

image/layer/utils.go

@@ -31,11 +31,11 @@ import (
 type MapOptions struct {
 	// UIDMappings and GIDMappings are the UID and GID mappings to apply when
 	// packing and unpacking image rootfs layers.
-	UIDMappings []rspec.IDMapping
-	GIDMappings []rspec.IDMapping
+	UIDMappings []rspec.IDMapping `json:"uid_mappings"`
+	GIDMappings []rspec.IDMapping `json:"gid_mappings"`
 
 	// Rootless specifies whether any to error out if chown fails.
-	Rootless bool
+	Rootless bool `json:"rootless"`
 }
 
 // mapHeader maps a tar.Header generated from the filesystem so that it