Merge pull request #130 from concourse/load-oci-images

load OCI images with IMAGE_ARG_*

Author: taylorsilva

README.md

@@ -62,12 +62,12 @@ _(As a convention in the list below, all task parameters are specified with a
   to build.
 
 * `$BUILDKIT_SSH` your ssh key location that is mounted in your `Dockerfile`. This is
-  generally used for pulling dependencies from private repositories. 
+  generally used for pulling dependencies from private repositories.
 
   For Example. In your `Dockerfile`, you can mount a key as
   ```
   RUN --mount=type=ssh,id=github_ssh_key pip install -U -r ./hats/requirements-test.txt
-  ``` 
+  ```
 
   Then in your Concourse YAML configuration:
   ```
@@ -122,10 +122,11 @@ _(As a convention in the list below, all task parameters are specified with a
   `(( mysecret ))` expands to in `/run/secrets/mysecret`.
 
 * `$IMAGE_ARG_*`: params prefixed with `IMAGE_ARG_*` point to image tarballs
-  (i.e. `docker save` format) to preload so that they do not have to be fetched
-  during the build. An image reference will be provided as the given build arg
-  name. For example, `IMAGE_ARG_base_image=ubuntu/image.tar` will set
-  `base_image` to a local image reference for using `ubuntu/image.tar`.
+  (i.e. `docker save` format) or path to images in OCI layout format, to preload
+  so that they do not have to be fetched during the build. An image reference
+  will be provided as the given build arg name. For example,
+  `IMAGE_ARG_base_image=ubuntu/image.tar` will set `base_image` to a local image
+  reference for using `ubuntu/image.tar`.
 
   This must be accepted as an argument for use; for example:
 
@@ -134,10 +135,10 @@ _(As a convention in the list below, all task parameters are specified with a
   FROM ${base_image}
   ```
 
-* `$IMAGE_PLATFORM`: Specify the target platform to build the image for. For
-  example `IMAGE_PLATFORM=linux/arm64` will build the image for the Linux OS
-  and `arm64` architecture. By default, images will be built for the current
-  worker's platform that the task is running on.
+* `$IMAGE_PLATFORM`: Specify the target platform(s) to build the image for. For
+  example `IMAGE_PLATFORM=linux/arm64,linux/amd64` will build the image for the
+  Linux OS and architectures `arm64` and `amd64`. By default, images will be
+  built for the current worker's platform that the task is running on.
 
 * `$LABEL_*`: params prefixed with `LABEL_` will be set as image labels.
   For example `LABEL_foo=bar`, will set the `foo` label to `bar`.

registry.go

@@ -5,29 +5,47 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/julienschmidt/httprouter"
 	"github.com/sirupsen/logrus"
 )
 
 type ImageArg struct {
-	Image   v1.Image
-	ArgName string
+	Index        v1.ImageIndex
+	Image        v1.Image
+	BuildArgName string
 }
 type LocalRegistry map[string]ImageArg
 
 func LoadRegistry(imagePaths map[string]string) (LocalRegistry, error) {
 	images := LocalRegistry{}
 	for name, path := range imagePaths {
-		image, err := tarball.ImageFromPath(path, nil)
+		stat, err := os.Stat(path)
 		if err != nil {
-			return nil, fmt.Errorf("image from path: %w", err)
+			return nil, fmt.Errorf("error inspecting path: %w", err)
 		}
 
-		images[strings.ToLower(name)] = ImageArg{Image: image, ArgName: name}
+		var index v1.ImageIndex
+		var image v1.Image
+		if stat.IsDir() {
+			index, err = layout.ImageIndexFromPath(path)
+			if err != nil {
+				return nil, fmt.Errorf("image from path: %w", err)
+			}
+		} else {
+			image, err = tarball.ImageFromPath(path, nil)
+			if err != nil {
+				return nil, fmt.Errorf("image from tarball: %w", err)
+			}
+		}
+
+		images[strings.ToLower(name)] = ImageArg{Index: index, Image: image, BuildArgName: name}
 	}
 
 	return images, nil
@@ -63,7 +81,7 @@ func ServeRegistry(reg LocalRegistry) (string, error) {
 func (registry LocalRegistry) BuildArgs(port string) []string {
 	var buildArgs []string
 	for name, image := range registry {
-		buildArgs = append(buildArgs, fmt.Sprintf("%s=localhost:%s/%s", image.ArgName, port, name))
+		buildArgs = append(buildArgs, fmt.Sprintf("%s=localhost:%s/%s", image.BuildArgName, port, name))
 	}
 
 	return buildArgs
@@ -82,30 +100,94 @@ func (registry LocalRegistry) GetManifest(w http.ResponseWriter, r *http.Request
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
-	image := img.Image
 
-	mt, err := image.MediaType()
-	if err != nil {
-		logrus.Errorf("failed to get media type: %s", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
+	var mediaType types.MediaType
+	var blob []byte
+	var digest v1.Hash
+	var err error
+
+	if img.Image != nil {
+		mediaType, err = img.Image.MediaType()
+		if err != nil {
+			logrus.Errorf("failed to get media type: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		blob, err = img.Image.RawManifest()
+		if err != nil {
+			logrus.Errorf("failed to get manifest: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		digest, err = img.Image.Digest()
+		if err != nil {
+			logrus.Errorf("failed to get digest: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
 
-	blob, err := image.RawManifest()
-	if err != nil {
-		logrus.Errorf("failed to get manifest: %s", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
 	}
 
-	digest, err := image.Digest()
-	if err != nil {
-		logrus.Errorf("failed to get digest: %s", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
+	if img.Index != nil {
+		digest, err = img.Index.Digest()
+		if err != nil {
+			logrus.Errorf("error getting ImageIndex's digest: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		// Check if we were given a Hash. An err means we were NOT given a Hash
+		// and got a string like "latest" or a semver. In that case we return
+		// the ImageIndex itself
+		refHash, err := v1.NewHash(ref)
+		if digest.String() == ref || err != nil {
+			mediaType, err = img.Index.MediaType()
+			if err != nil {
+				logrus.Errorf("error getting MediaType: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			blob, err = img.Index.RawManifest()
+			if err != nil {
+				logrus.Errorf("error getting RawManifest: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+		} else {
+			// TODO: technically there could be nested ImageIndex's, but they're
+			// not common so not bothering to handle those right now
+
+			//try and find ref inside ImageIndex
+			digest = refHash
+
+			image, err := img.Index.Image(digest)
+			if err != nil {
+				logrus.Errorf("error getting Image from ImageIndex: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			mediaType, err = image.MediaType()
+			if err != nil {
+				logrus.Errorf("error getting MediaType from Image: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			blob, err = image.RawManifest()
+			if err != nil {
+				logrus.Errorf("error getting RawManifest from Image: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+		}
+
 	}
 
-	w.Header().Set("Content-Type", string(mt))
+	w.Header().Set("Content-Type", string(mediaType))
 	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(blob)))
 	w.Header().Set("Docker-Content-Digest", digest.String())
 
@@ -133,7 +215,6 @@ func (registry LocalRegistry) GetBlob(w http.ResponseWriter, r *http.Request, p
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
-	image := img.Image
 
 	hash, err := v1.NewHash(dig)
 	if err != nil {
@@ -142,48 +223,89 @@ func (registry LocalRegistry) GetBlob(w http.ResponseWriter, r *http.Request, p
 		return
 	}
 
-	cfgHash, err := image.ConfigName()
-	if err != nil {
-		logrus.Errorf("failed to get config hash: %s", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
+	var layer v1.Layer
 
-	if hash == cfgHash {
-		manifest, err := image.Manifest()
-		if err != nil {
-			logrus.Errorf("get image manifest: %s", err)
-			return
-		}
+	if img.Image != nil {
+		image := img.Image
 
-		cfgBlob, err := image.RawConfigFile()
+		cfgHash, err := image.ConfigName()
 		if err != nil {
-			logrus.Errorf("failed to get config file: %s", err)
+			logrus.Errorf("failed to get config hash: %s", err)
 			w.WriteHeader(http.StatusInternalServerError)
 			return
 		}
 
-		w.Header().Set("Content-Type", string(manifest.Config.MediaType))
-		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(cfgBlob)))
+		if hash == cfgHash {
+			manifest, err := image.Manifest()
+			if err != nil {
+				logrus.Errorf("get image manifest: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			cfgBlob, err := image.RawConfigFile()
+			if err != nil {
+				logrus.Errorf("failed to get config file: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			w.Header().Set("Content-Type", string(manifest.Config.MediaType))
+			w.Header().Set("Content-Length", fmt.Sprintf("%d", len(cfgBlob)))
+
+			if r.Method == "HEAD" {
+				return
+			}
+
+			_, err = w.Write(cfgBlob)
+			if err != nil {
+				logrus.Errorf("write config blob: %s", err)
+				return
+			}
+
+			return
+		}
 
-		if r.Method == "HEAD" {
+		layer, err = image.LayerByDigest(hash)
+		if err != nil {
+			logrus.Errorf("failed to get layer: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
 			return
 		}
+	}
 
-		_, err = w.Write(cfgBlob)
+	if img.Index != nil {
+		index, err := img.Index.IndexManifest()
 		if err != nil {
-			logrus.Errorf("write config blob: %s", err)
+			logrus.Errorf("error getting Manifest from ImageIndex: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
 			return
 		}
 
-		return
-	}
+		// Search all images in the ImageIndex for the requested layer
+		for _, desc := range index.Manifests {
+			if desc.MediaType.IsImage() {
+				img, err := img.Index.Image(desc.Digest)
+				if err != nil {
+					logrus.Errorf("error getting image from ImageIndex: %s", err)
+					w.WriteHeader(http.StatusInternalServerError)
+					return
+				}
+
+				// ignore errors related to not finding the layer and just keep searching
+				l, err := img.LayerByDigest(hash)
+				if err == nil {
+					layer = l
+					break
+				}
+			}
+		}
 
-	layer, err := image.LayerByDigest(hash)
-	if err != nil {
-		logrus.Errorf("failed to get layer: %s", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
+		if layer == nil {
+			logrus.Errorf("layer not found in ImageIndex: %s", err)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
 	}
 
 	size, err := layer.Size()