Add "Builder: oci-import" support

In the case of base images (`debian`, `alpine`, `ubuntu`, etc), using a `Dockerfile` as our method of ingestion doesn't really buy us very much.  It made sense at the time it was implemented ("all `Dockerfile`, all the time"), but at this point they're all some variation on `FROM scratch \n ADD foo.tar.xz / \n CMD ["/bin/some-shell"]`, and cannot reasonably be "rebuilt" when their base image changes (which is one of the key functions of the official images) since they _are_ the base images in question.

Functionally, consuming a tarball in this way isn't _that_ much different from consuming a raw tarball that's part of, say, an OCI image layout (https://github.com/opencontainers/image-spec/blob/v1.0.2/image-layout.md) -- it's some tarball plus some metadata about what to do with it.

For less trivial images, there's a significant difference (and I'm not proposing to use this for anything beyond simple one-layer base images), but for a single layer this would be basically identical.

As a more specific use case, the Debian `rootfs.tar.xz` files are currently [100% reproducible](https://github.com/debuerreotype/debuerreotype).  Unfortunately, some of that gets lost when it gets imported into Docker, and thus it takes some additional effort to get from the Docker-generated rootfs back to the original debuerreotype-generated file.

This adds the ability to consume an OCI image directly, to go even further and have a 100% fully reproducible image digest as well, which makes it easier to trace a given published image back to the reproducible source generated by the upstream tooling (especially if a given image is also pushed by the maintainer elsewhere).

Here's an example `oci-debian` file I was using for testing this:

    Maintainers: Foo (@bar)
    GitRepo: https://github.com/tianon/docker-debian-artifacts.git
    GitFetch: refs/heads/oci-arm32v5
    Architectures: arm32v5
    GitCommit: d6ac440e7760b6b16e3d3da6f2b56736b9c10065
    Builder: oci-import
    File: index.json

    Tags: bullseye, bullseye-20221114, 11.5, 11, latest
    Directory: bullseye/oci

    Tags: bullseye-slim, bullseye-20221114-slim, 11.5-slim, 11-slim
    Directory: bullseye/slim/oci

Author: tianon

cmd/bashbrew/cmd-build.go

@@ -76,6 +76,8 @@ func cmdBuild(c *cli.Context) error {
 			if err != nil {
 				return cli.NewMultiError(fmt.Errorf(`failed calculating "cache hash" for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 			}
+			imageTags := r.Tags(namespace, uniq, entry)
+			tags := append([]string{cacheTag}, imageTags...)
 
 			// check whether we've already built this artifact
 			_, err = dockerInspect("{{.Id}}", cacheTag)
@@ -87,51 +89,50 @@ func cmdBuild(c *cli.Context) error {
 						return cli.NewMultiError(fmt.Errorf(`failed fetching git repo for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 					}
 
-					archive, err := gitArchive(commit, entry.ArchDirectory(arch))
-					if err != nil {
-						return cli.NewMultiError(fmt.Errorf(`failed generating git archive for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
-					}
-					defer archive.Close()
-
-					// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
-
 					switch builder := entry.ArchBuilder(arch); builder {
-					case "classic", "":
+					case "buildkit", "classic", "":
 						var platform string
 						if fromScratch {
 							platform = ociArch.String()
 						}
-						err = dockerBuild(cacheTag, entry.ArchFile(arch), archive, platform)
+
+						archive, err := gitArchive(commit, entry.ArchDirectory(arch))
 						if err != nil {
-							return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)
+							return cli.NewMultiError(fmt.Errorf(`failed generating git archive for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 						}
-					case "buildkit":
-						var platform string
-						if fromScratch {
-							platform = ociArch.String()
+						defer archive.Close()
+
+						if builder == "buildkit" {
+							err = dockerBuildxBuild(tags, entry.ArchFile(arch), archive, platform)
+						} else {
+							// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
+							err = dockerBuild(tags, entry.ArchFile(arch), archive, platform)
 						}
-						err = dockerBuildxBuild(cacheTag, entry.ArchFile(arch), archive, platform)
 						if err != nil {
 							return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 						}
+
+						archive.Close() // be sure this happens sooner rather than later (defer might take a while, and we want to reap zombies more aggressively)
+
+					case "oci-import":
+						err := ociImportBuild(tags, commit, entry.ArchDirectory(arch), entry.ArchFile(arch))
+						if err != nil {
+							return cli.NewMultiError(fmt.Errorf(`failed oci-import build of %q (tags %q)`, r.RepoName, entry.TagsString()), err)
+						}
+
+						fmt.Printf("Importing %s into Docker\n", r.EntryIdentifier(entry))
+						err = ociImportDockerLoad(imageTags)
+						if err != nil {
+							return cli.NewMultiError(fmt.Errorf(`failed oci-import into Docker of %q (tags %q)`, r.RepoName, entry.TagsString()), err)
+						}
+
 					default:
 						return cli.NewMultiError(fmt.Errorf(`unknown builder %q`, builder))
 					}
-					archive.Close() // be sure this happens sooner rather than later (defer might take a while, and we want to reap zombies more aggressively)
 				}
 			} else {
 				fmt.Printf("Using %s (%s)\n", cacheTag, r.EntryIdentifier(entry))
 			}
-
-			for _, tag := range r.Tags(namespace, uniq, entry) {
-				fmt.Printf("Tagging %s\n", tag)
-				if !dryRun {
-					err := dockerTag(cacheTag, tag)
-					if err != nil {
-						return cli.NewMultiError(fmt.Errorf(`failed tagging %q as %q`, cacheTag, tag), err)
-					}
-				}
-			}
 		}
 	}
 

cmd/bashbrew/cmd-push.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 
 	"github.com/urfave/cli"
 )
@@ -38,29 +39,69 @@ func cmdPush(c *cli.Context) error {
 				continue
 			}
 
+			tags := []string{}
 			// we can't use "r.Tags()" here because it will include SharedTags, which we never want to push directly (see "cmd-put-shared.go")
-		TagsLoop:
 			for i, tag := range entry.Tags {
 				if uniq && i > 0 {
 					break
 				}
 				tag = tagRepo + ":" + tag
+				tags = append(tags, tag)
+			}
 
-				if !force {
-					localImageId, _ := dockerInspect("{{.Id}}", tag)
-					registryImageIds := fetchRegistryImageIds(tag)
-					for _, registryImageId := range registryImageIds {
-						if localImageId == registryImageId {
-							fmt.Fprintf(os.Stderr, "skipping %s (remote image matches local)\n", tag)
-							continue TagsLoop
-						}
-					}
+			switch builder := entry.ArchBuilder(arch); builder {
+			case "oci-import":
+				cacheTag, err := r.DockerCacheName(entry)
+				if err != nil {
+					return cli.NewMultiError(fmt.Errorf(`failed calculating "cache hash" for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
+				}
+				desc, err := ociImportLookup(cacheTag)
+				if err != nil {
+					return cli.NewMultiError(fmt.Errorf(`failed looking up descriptor for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 				}
-				fmt.Printf("Pushing %s\n", tag)
+				skip, update, err := ociImportPushFilter(*desc, tags)
+				if err != nil {
+					return cli.NewMultiError(fmt.Errorf(`failed looking up tags for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
+				}
+				if len(skip) > 0 && len(update) == 0 {
+					fmt.Fprintf(os.Stderr, "skipping %s (remote tags all up-to-date)\n", r.EntryIdentifier(entry))
+					continue
+				} else if len(skip) > 0 {
+					fmt.Fprintf(os.Stderr, "partially skipping %s (remote tags up-to-date: %s)\n", r.EntryIdentifier(entry), strings.Join(skip, ", "))
+				}
+				fmt.Printf("Pushing %s to %s\n", desc.Digest, strings.Join(update, ", "))
 				if !dryRun {
-					err = dockerPush(tag)
+					err := ociImportPush(*desc, update)
 					if err != nil {
-						return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
+						return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, r.EntryIdentifier(entry)), err)
+					}
+				}
+
+			default:
+			TagsLoop:
+				for _, tag := range tags {
+					if !force {
+						localImageId, _ := dockerInspect("{{.Id}}", tag)
+						if debugFlag {
+							fmt.Printf("DEBUG: docker inspect %q -> %q\n", tag, localImageId)
+						}
+						registryImageIds := fetchRegistryImageIds(tag)
+						if debugFlag {
+							fmt.Printf("DEBUG: registry inspect %q -> %+v\n", tag, registryImageIds)
+						}
+						for _, registryImageId := range registryImageIds {
+							if localImageId == registryImageId {
+								fmt.Fprintf(os.Stderr, "skipping %s (remote image matches local)\n", tag)
+								continue TagsLoop
+							}
+						}
+					}
+					fmt.Printf("Pushing %s\n", tag)
+					if !dryRun {
+						err = dockerPush(tag)
+						if err != nil {
+							return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
+						}
 					}
 				}
 			}

cmd/bashbrew/containerd.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/content/local"
+	"github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/namespaces"
+
+	"go.etcd.io/bbolt"
+)
+
+func newBuiltinContainerdServices(ctx context.Context) (containerd.ClientOpt, error) {
+	// thanks to https://github.com/Azure/image-rootfs-scanner/blob/e7041e47d1a13e15d73d9c85644542e6758f9f3a/containerd.go#L42-L87 for inspiring this magic
+
+	root := filepath.Join(defaultCache, "containerd")
+	dbPath := filepath.Join(root, "metadata.db")
+	contentRoot := filepath.Join(root, "content")
+
+	cs, err := local.NewStore(contentRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := bbolt.Open(dbPath, 0600, &bbolt.Options{
+		Timeout: 1 * time.Minute,
+	})
+
+	mdb := metadata.NewDB(db, cs, nil)
+	return containerd.WithServices(
+		containerd.WithContentStore(mdb.ContentStore()),
+		containerd.WithImageStore(metadata.NewImageStore(mdb)),
+		containerd.WithLeasesService(metadata.NewLeaseManager(mdb)),
+	), nil
+}
+
+var containerdClientCache *containerd.Client = nil
+
+// the returned client is cached, don't Close() it!
+func newContainerdClient(ctx context.Context) (context.Context, *containerd.Client, error) {
+	ns := "bashbrew"
+	for _, envKey := range []string{
+		`BASHBREW_CONTAINERD_NAMESPACE`,
+		`CONTAINERD_NAMESPACE`,
+	} {
+		if env, ok := os.LookupEnv(envKey); ok {
+			if env != "" {
+				// set-but-empty environment variable means use default explicitly
+				ns = env
+			}
+			break
+		}
+	}
+	ctx = namespaces.WithNamespace(ctx, ns)
+
+	if containerdClientCache != nil {
+		return ctx, containerdClientCache, nil
+	}
+
+	for _, envKey := range []string{
+		`BASHBREW_CONTAINERD_CONTENT_ADDRESS`, // TODO if we ever need to connnect to a containerd instance for something more interesting like running containers, we need to have *that* codepath not use _CONTENT_ variants
+		`BASHBREW_CONTAINERD_ADDRESS`,
+		`CONTAINERD_CONTENT_ADDRESS`,
+		`CONTAINERD_ADDRESS`,
+	} {
+		if socket, ok := os.LookupEnv(envKey); ok {
+			if socket == "" {
+				// we'll use a set-but-empty variable as an explicit request to use our built-in implementation
+				break
+			}
+			client, err := containerd.New(socket)
+			containerdClientCache = client
+			return ctx, client, err
+		}
+	}
+
+	// if we don't have an explicit variable asking us to connect to an existing containerd instance, we set up and use our own in-process content/image store
+	services, err := newBuiltinContainerdServices(ctx)
+	if err != nil {
+		return ctx, nil, err
+	}
+	client, err := containerd.New("", services)
+	containerdClientCache = client
+	return ctx, client, err
+}

cmd/bashbrew/docker.go

@@ -53,6 +53,14 @@ func (r Repo) dockerfileMetadata(entry *manifest.Manifest2822Entry) (*dockerfile
 var dockerfileMetadataCache = map[string]*dockerfileMetadata{}
 
 func (r Repo) archDockerfileMetadata(arch string, entry *manifest.Manifest2822Entry) (*dockerfileMetadata, error) {
+	if builder := entry.ArchBuilder(arch); builder == "oci-import" {
+		return &dockerfileMetadata{
+			Froms: []string{
+				"scratch",
+			},
+		}, nil
+	}
+
 	commit, err := r.fetchGitRepo(arch, entry)
 	if err != nil {
 		return nil, cli.NewMultiError(fmt.Errorf("failed fetching Git repo for arch %q from entry %q", arch, entry.String()), err)
@@ -242,9 +250,13 @@ func (r Repo) dockerBuildUniqueBits(entry *manifest.Manifest2822Entry) ([]string
 	return uniqueBits, nil
 }
 
-func dockerBuild(tag string, file string, context io.Reader, platform string) error {
-	args := []string{"build", "--tag", tag, "--file", file, "--rm", "--force-rm"}
-	args = append(args, "-")
+func dockerBuild(tags []string, file string, context io.Reader, platform string) error {
+	args := []string{"build"}
+	for _, tag := range tags {
+		args = append(args, "--tag", tag)
+	}
+	args = append(args, "--file", file, "--rm", "--force-rm", "-")
+
 	cmd := exec.Command("docker", args...)
 	cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=0")
 	if debugFlag {
@@ -278,7 +290,7 @@ func dockerBuild(tag string, file string, context io.Reader, platform string) er
 
 const dockerfileSyntaxEnv = "BASHBREW_BUILDKIT_SYNTAX"
 
-func dockerBuildxBuild(tag string, file string, context io.Reader, platform string) error {
+func dockerBuildxBuild(tags []string, file string, context io.Reader, platform string) error {
 	dockerfileSyntax, ok := os.LookupEnv(dockerfileSyntaxEnv)
 	if !ok {
 		return fmt.Errorf("missing %q", dockerfileSyntaxEnv)
@@ -289,13 +301,14 @@ func dockerBuildxBuild(tag string, file string, context io.Reader, platform stri
 		"build",
 		"--progress", "plain",
 		"--build-arg", "BUILDKIT_SYNTAX=" + dockerfileSyntax,
-		"--tag", tag,
-		"--file", file,
 	}
 	if platform != "" {
 		args = append(args, "--platform", platform)
 	}
-	args = append(args, "-")
+	for _, tag := range tags {
+		args = append(args, "--tag", tag)
+	}
+	args = append(args, "--file", file, "-")
 
 	cmd := exec.Command("docker", args...)
 	cmd.Stdin = context

cmd/bashbrew/git.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -15,6 +16,7 @@ import (
 
 	"github.com/docker-library/bashbrew/manifest"
 	"github.com/docker-library/bashbrew/pkg/execpipe"
+	"github.com/docker-library/bashbrew/pkg/gitfs"
 
 	goGit "github.com/go-git/go-git/v5"
 	goGitConfig "github.com/go-git/go-git/v5/config"
@@ -94,6 +96,13 @@ func getGitCommit(commit string) (string, error) {
 	return h.String(), nil
 }
 
+func gitCommitFS(commit string) (fs.FS, error) {
+	if err := ensureGitInit(); err != nil {
+		return nil, err
+	}
+	return gitfs.CommitHash(gitRepo, commit)
+}
+
 func gitStream(args ...string) (io.ReadCloser, error) {
 	return execpipe.Run(gitCommand(args...))
 }

cmd/bashbrew/main.go

@@ -167,6 +167,8 @@ func main() {
 			if !debugFlag {
 				// containerd uses logrus, but it defaults to "info" (which is a bit leaky where we use containerd)
 				logrus.SetLevel(logrus.WarnLevel)
+			} else {
+				logrus.SetLevel(logrus.DebugLevel)
 			}
 
 			arch = c.GlobalString("arch")
@@ -401,9 +403,9 @@ func main() {
 			Category: "plumbing",
 		},
 		{
-			Name:  "remote",
-			Usage: "query registries for bashbrew-related data",
-			Before: subcommandBeforeFactory("remote"),
+			Name:     "remote",
+			Usage:    "query registries for bashbrew-related data",
+			Before:   subcommandBeforeFactory("remote"),
 			Category: "plumbing",
 			Subcommands: []cli.Command{
 				{