feat(deamonless): add buildah-based deamonless image puller

this still requires some refactoring and does not implement Unpack

Author: ecordell

cmd/opm/registry/add.go

@@ -38,28 +38,28 @@ func newRegistryAddCmd() *cobra.Command {
 }
 
 func addFunc(cmd *cobra.Command, args []string) error {
-	bundleImages, err := cmd.Flags().GetStringSlice("bundle-images")
+	permissive, err := cmd.Flags().GetBool("permissive")
 	if err != nil {
 		return err
 	}
-	fromFilename, err := cmd.Flags().GetString("database")
+	skipTLS, err := cmd.Flags().GetBool("skip-tls")
 	if err != nil {
 		return err
 	}
-	permissive, err := cmd.Flags().GetBool("permissive")
+	fromFilename, err := cmd.Flags().GetString("database")
 	if err != nil {
 		return err
 	}
-	skipTLS, err := cmd.Flags().GetBool("skip-tls")
+	bundleImages, err := cmd.Flags().GetStringSlice("bundle-images")
 	if err != nil {
 		return err
 	}
 
 	request := registry.AddToRegistryRequest{
-		Bundles:       bundleImages,
-		InputDatabase: fromFilename,
 		Permissive:    permissive,
 		SkipTLS:       skipTLS,
+		InputDatabase: fromFilename,
+		Bundles:       bundleImages,
 	}
 
 	logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})

go.mod

@@ -3,44 +3,40 @@ module github.com/operator-framework/operator-registry
 go 1.13
 
 require (
-	github.com/Microsoft/hcsshim v0.8.7 // indirect
 	github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/containerd/containerd v1.3.2
 	github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c // indirect
+	github.com/containers/buildah v1.14.3
 	github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce
-	github.com/docker/docker-credential-helpers v0.6.3 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang-migrate/migrate/v4 v4.6.2
 	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
-	github.com/google/go-cmp v0.4.0 // indirect
 	github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/opencontainers/image-spec v1.0.1
-	github.com/opencontainers/runc v0.1.1 // indirect
+	github.com/onsi/ginkgo v1.12.0
+	github.com/onsi/gomega v1.9.0
+	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
 	github.com/operator-framework/api v0.1.1
 	github.com/otiai10/copy v1.0.2
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
-	github.com/stretchr/testify v1.4.0
+	github.com/spf13/cobra v0.0.6
+	github.com/stretchr/testify v1.5.1
 	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d // indirect
 	golang.org/x/mod v0.2.0
 	golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	google.golang.org/grpc v1.23.1
+	google.golang.org/grpc v1.24.0
 	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/api v0.17.3
 	k8s.io/apiextensions-apiserver v0.17.3

go.sum

@@ -0,0 +1,164 @@
+package buildahregistry
+
+import (
+	"github.com/containers/storage"
+	"github.com/containers/storage/pkg/idtools"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+
+	//"sync"
+	//
+	//contentlocal "github.com/containerd/containerd/content/local"
+	//"github.com/containerd/containerd/metadata"
+	//"github.com/containerd/containerd/platforms"
+	"github.com/sirupsen/logrus"
+	//bolt "go.etcd.io/bbolt"
+)
+
+type RegistryConfig struct {
+	Log               *logrus.Entry
+	ResolverConfigDir string
+	DBPath            string
+	CacheDir          string
+	PreserveCache     bool
+	SkipTLS           bool
+}
+
+func (r *RegistryConfig) apply(options []RegistryOption) {
+	for _, option := range options {
+		option(r)
+	}
+}
+
+func (r *RegistryConfig) complete() error {
+	if err := os.Mkdir(r.CacheDir, os.ModePerm); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	if r.DBPath == "" {
+		r.DBPath = filepath.Join(r.CacheDir, "metadata.db")
+	}
+
+	return nil
+}
+
+func defaultConfig() *RegistryConfig {
+	config := &RegistryConfig{
+		Log:               logrus.NewEntry(logrus.New()),
+		ResolverConfigDir: "",
+		CacheDir:          "cache",
+	}
+
+	return config
+}
+
+func NewRegistry(options ...RegistryOption) (*Registry, error) {
+	config := defaultConfig()
+	config.apply(options)
+	if err := config.complete(); err != nil {
+		return nil, err
+	}
+
+	var (
+		once   sync.Once
+		closed bool
+	)
+	closeFunc := func() error {
+		defer func() {
+			once.Do(func() {
+				closed = true
+			})
+		}()
+		if closed {
+			// Already closed, no-op
+			return nil
+		}
+
+		if config.PreserveCache {
+			return nil
+		}
+		return os.RemoveAll(config.CacheDir)
+	}
+
+	// TODO: at this point we've overwritten all the defaults, may as well not use this
+	storeOpts, err := storage.DefaultStoreOptionsAutoDetectUID()
+	if err != nil {
+		return nil, err
+	}
+	storeOpts.RootlessStoragePath = config.CacheDir
+	storeOpts.RunRoot = config.CacheDir
+	storeOpts.GraphRoot = config.CacheDir
+	storeOpts.GraphDriverName = "vfs"
+	storeOpts.UIDMap = []idtools.IDMap{
+		{ContainerID: 0, HostID: os.Getuid()},
+	}
+	storeOpts.GIDMap = []idtools.IDMap{
+		{ContainerID: 0, HostID: os.Getgid()},
+	}
+
+	store, err := storage.GetStore(storeOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: probably don't want the signature policy to be here
+	ioutil.WriteFile(path.Join(config.CacheDir, "policy.json"), []byte(`
+{
+    "default": [
+        {
+            "type": "insecureAcceptAnything"
+        }
+    ],
+    "transports":
+        {
+            "docker-daemon":
+                {
+                    "": [{"type":"insecureAcceptAnything"}]
+                }
+        }
+}
+`), os.ModePerm)
+
+	r := &Registry{
+		Store:    store,
+		CacheDir: config.CacheDir,
+		log:      config.Log,
+		close:    closeFunc,
+	}
+	return r, nil
+}
+
+type RegistryOption func(config *RegistryConfig)
+
+func WithLog(log *logrus.Entry) RegistryOption {
+	return func(config *RegistryConfig) {
+		config.Log = log
+	}
+}
+
+func WithResolverConfigDir(path string) RegistryOption {
+	return func(config *RegistryConfig) {
+		config.ResolverConfigDir = path
+	}
+}
+
+func WithCacheDir(dir string) RegistryOption {
+	return func(config *RegistryConfig) {
+		config.CacheDir = dir
+	}
+}
+
+func PreserveCache() RegistryOption {
+	return func(config *RegistryConfig) {
+		config.PreserveCache = true
+	}
+}
+
+func SkipTLS() RegistryOption {
+	return func(config *RegistryConfig) {
+		config.SkipTLS = true
+	}
+}

pkg/image/buildahregistry/options.go

@@ -0,0 +1,57 @@
+package buildahregistry
+
+import (
+	"context"
+	"github.com/containers/storage"
+	"path"
+
+	"github.com/containers/buildah"
+	"github.com/containers/image/v5/types"
+	"github.com/sirupsen/logrus"
+)
+
+type Registry struct {
+	storage.Store
+
+	CacheDir string
+
+	log      *logrus.Entry
+
+	close func() error
+}
+
+// Pull fetches and stores an image by reference.
+func (r *Registry) Pull(ctx context.Context, ref string) error {
+	img, err := buildah.Pull(ctx, ref, buildah.PullOptions{
+		SignaturePolicyPath: path.Join(r.CacheDir, "policy.json"),
+		ReportWriter:        r.log.Writer(),
+		Store:               r.Store,
+		SystemContext:       &types.SystemContext{
+			// TODO: auth stuff goes here too
+			// TODO: if we're okay with buildah's cobra args, there's a function to build this from the standard args
+			SignaturePolicyPath:     path.Join(r.CacheDir, "policy.json"),
+			OCIInsecureSkipTLSVerify:          true,
+			DockerInsecureSkipTLSVerify:       types.OptionalBoolTrue,
+
+		},
+		BlobDirectory:       r.CacheDir,
+		AllTags:             false,
+		RemoveSignatures:    false,
+		MaxRetries:          0,
+		RetryDelay:          0,
+	})
+
+	r.log.Info(img)
+	return err
+}
+
+// Unpack writes the unpackaged content of an image to a directory.
+// If the referenced image does not exist in the registry, an error is returned.
+func (r *Registry) Unpack(ctx context.Context, ref, dir string) error {
+	return nil
+}
+
+func (r *Registry) Close() error {
+	return r.close()
+}
+

pkg/image/buildahregistry/registry.go

@@ -1,4 +1,4 @@
-package unprivileged
+package buildahregistry
 
 import (
 	"context"
@@ -33,15 +33,15 @@ func setupRegistry(t *testing.T, ctx context.Context, rootDir string) string {
 	}
 	config.HTTP.DrainTimeout = time.Duration(2) * time.Second
 
-	dockerRegistry, err := registry.NewRegistry(context.Background(), config)
+	dockerRegistry, err := registry.NewRegistry(ctx, config)
 	require.NoError(t, err)
 
 	go func() {
 		require.NoError(t, dockerRegistry.ListenAndServe())
 	}()
 
 	// Return the registry host string
-	return fmt.Sprintf("127.0.0.1:%d", dockerPort)
+	return fmt.Sprintf("localhost:%d", dockerPort)
 }
 
 func dirChecksum(t *testing.T, dir string) string {
@@ -86,6 +86,7 @@ func TestPullAndUnpack(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {
+			logrus.SetLevel(logrus.DebugLevel)
 			ctx, close := context.WithCancel(context.Background())
 			defer close()
 

pkg/image/unprivileged/registry_test.go renamed to pkg/image/buildahregistry/registry_test.go

@@ -1,4 +1,4 @@
-package unprivileged
+package buildahregistry
 
 import (
 	"crypto/tls"

pkg/image/unprivileged/resolver.go renamed to pkg/image/buildahregistry/resolver.go

@@ -1,4 +1,4 @@
-package unprivileged
+package buildahregistry
 
 import (
 	"github.com/containerd/containerd/content"