feat(cli): Add ability to pull images from private registry (crossplane#6227)

Add method to get docker configfile, and use it to retrieve credentials to pull images from private registries

Signed-off-by: Chuan-Yen Chiang <[email protected]>

Author: cychiang

apis/apiextensions/fn/proto/v1/run_function.pb.go

@@ -21,12 +21,18 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 
+	"github.com/distribution/reference"
+	"github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	typesimage "github.com/docker/docker/api/types/image"
+	registrytypes "github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/registry"
 	"github.com/docker/go-connections/nat"
 
 	"github.com/crossplane/crossplane-runtime/pkg/errors"
@@ -106,6 +112,9 @@ type RuntimeDocker struct {
 	// Cleanup controls how the containers are handled after rendering.
 	Cleanup DockerCleanup
 
+	// ConfigFile contains information like credentials for each registry, default to ~/.docker/config.json
+	ConfigFile *configfile.ConfigFile
+
 	// PullPolicy controls how the runtime image is pulled.
 	PullPolicy DockerPullPolicy
 
@@ -152,13 +161,19 @@ func GetRuntimeDocker(fn pkgv1.Function, log logging.Logger) (*RuntimeDocker, er
 	if err != nil {
 		return nil, errors.Wrapf(err, "cannot get pull policy for Function %q", fn.GetName())
 	}
+
+	// Initial ConfigFile
+	configFile := config.LoadDefaultConfigFile(os.Stderr)
+
 	r := &RuntimeDocker{
 		Image:      fn.Spec.Package,
 		Name:       "",
 		Cleanup:    cleanup,
+		ConfigFile: configFile,
 		PullPolicy: pullPolicy,
 		log:        log,
 	}
+
 	if i := fn.GetAnnotations()[AnnotationKeyRuntimeDockerImage]; i != "" {
 		r.Image = i
 	}
@@ -244,9 +259,16 @@ func (r *RuntimeDocker) createContainer(ctx context.Context, cli *client.Client)
 		PortBindings: bind,
 	}
 
+	options, err := r.getPullOptions()
+	if err != nil {
+		// We can continue to pull an image if we don't have the PullOptions with RegistryAuth
+		// as long as the image is from a public registry. Therefore, we log the error message and continue.
+		r.log.Info("Cannot get pull options", "image", r.Image, "err", err)
+	}
+
 	if r.PullPolicy == AnnotationValueRuntimeDockerPullPolicyAlways {
 		r.log.Debug("Pulling image with pullPolicy: Always", "image", r.Image)
-		err = PullImage(ctx, cli, r.Image)
+		err = PullImage(ctx, cli, r.Image, options)
 		if err != nil {
 			return "", "", errors.Wrapf(err, "cannot pull Docker image %q", r.Image)
 		}
@@ -262,7 +284,7 @@ func (r *RuntimeDocker) createContainer(ctx context.Context, cli *client.Client)
 
 		// The image was not found, but we're allowed to pull it.
 		r.log.Debug("Image not found, pulling", "image", r.Image)
-		err = PullImage(ctx, cli, r.Image)
+		err = PullImage(ctx, cli, r.Image, options)
 		if err != nil {
 			return "", "", errors.Wrapf(err, "cannot pull Docker image %q", r.Image)
 		}
@@ -278,6 +300,37 @@ func (r *RuntimeDocker) createContainer(ctx context.Context, cli *client.Client)
 	return rsp.ID, containerAddr, errors.Wrap(err, "cannot start Docker container")
 }
 
+func (r *RuntimeDocker) getPullOptions() (typesimage.PullOptions, error) {
+	// Resolve auth token by looking into config file
+	named, err := reference.ParseNormalizedNamed(r.Image)
+	if err != nil {
+		return typesimage.PullOptions{}, errors.Wrapf(err, "Image is not a valid reference %s", r.Image)
+	}
+
+	repoInfo, err := registry.ParseRepositoryInfo(named)
+	if err != nil {
+		return typesimage.PullOptions{}, errors.Wrapf(err, "Cannot parse repository info: %s", named.String())
+	}
+
+	configKey := repoInfo.Index.Name
+	if repoInfo.Index.Official {
+		configKey = registry.IndexServer
+	}
+	authConfig, err := r.ConfigFile.GetAuthConfig(configKey)
+	if err != nil {
+		return typesimage.PullOptions{}, errors.Wrapf(err, "Cannot get auth config info with configKey: %s", configKey)
+	}
+
+	encodedAuth, err := registrytypes.EncodeAuthConfig(registrytypes.AuthConfig(authConfig))
+	if err != nil {
+		return typesimage.PullOptions{}, errors.Wrapf(err, "Cannot encode auth config with configKey: %s", configKey)
+	}
+
+	return typesimage.PullOptions{
+		RegistryAuth: encodedAuth,
+	}, nil
+}
+
 // Start a Function as a Docker container.
 func (r *RuntimeDocker) Start(ctx context.Context) (RuntimeContext, error) {
 	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
@@ -327,8 +380,8 @@ type pullClient interface {
 
 // PullImage pulls the supplied image using the supplied client. It blocks until
 // the image has either finished pulling or hit an error.
-func PullImage(ctx context.Context, p pullClient, image string) error {
-	out, err := p.ImagePull(ctx, image, typesimage.PullOptions{})
+func PullImage(ctx context.Context, p pullClient, image string, options typesimage.PullOptions) error {
+	out, err := p.ImagePull(ctx, image, options)
 	if err != nil {
 		return err
 	}

apis/apiextensions/fn/proto/v1beta1/zz_generated_run_function.pb.go

@@ -19,8 +19,10 @@ package render
 import (
 	"context"
 	"io"
+	"os"
 	"testing"
 
+	"github.com/docker/cli/cli/config"
 	"github.com/docker/docker/api/types/image"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -76,6 +78,7 @@ func TestGetRuntimeDocker(t *testing.T) {
 			want: want{
 				rd: &RuntimeDocker{
 					Image:      "test-image-from-annotation",
+					ConfigFile: config.LoadDefaultConfigFile(os.Stderr),
 					Cleanup:    AnnotationValueRuntimeDockerCleanupOrphan,
 					PullPolicy: AnnotationValueRuntimeDockerPullPolicyAlways,
 				},
@@ -102,6 +105,7 @@ func TestGetRuntimeDocker(t *testing.T) {
 			want: want{
 				rd: &RuntimeDocker{
 					Image:      "test-image-from-annotation",
+					ConfigFile: config.LoadDefaultConfigFile(os.Stderr),
 					Cleanup:    AnnotationValueRuntimeDockerCleanupOrphan,
 					Name:       "test-container-name-function",
 					PullPolicy: AnnotationValueRuntimeDockerPullPolicyIfNotPresent,
@@ -125,6 +129,7 @@ func TestGetRuntimeDocker(t *testing.T) {
 			want: want{
 				rd: &RuntimeDocker{
 					Image:      "test-package",
+					ConfigFile: config.LoadDefaultConfigFile(os.Stderr),
 					Cleanup:    AnnotationValueRuntimeDockerCleanupRemove,
 					PullPolicy: AnnotationValueRuntimeDockerPullPolicyIfNotPresent,
 				},
@@ -189,6 +194,7 @@ func TestGetRuntimeDocker(t *testing.T) {
 			want: want{
 				rd: &RuntimeDocker{
 					Image:      "test-package",
+					ConfigFile: config.LoadDefaultConfigFile(os.Stderr),
 					Cleanup:    AnnotationValueRuntimeDockerCleanupStop,
 					PullPolicy: AnnotationValueRuntimeDockerPullPolicyIfNotPresent,
 				},

cmd/crank/render/runtime_docker.go

@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/alecthomas/kong v0.9.0
 	github.com/crossplane/crossplane-runtime v1.19.0-rc.0.0.20241105071456-19d95a69cc03
+	github.com/distribution/reference v0.5.0
 	github.com/docker/docker v27.1.1+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/emicklei/dot v1.6.2
@@ -26,9 +27,9 @@ require (
 	github.com/spf13/afero v1.11.0
 	github.com/upbound/up-sdk-go v0.1.1-0.20240122203953-2d00664aab8e
 	golang.org/x/sync v0.10.0
-	google.golang.org/grpc v1.67.1
+	google.golang.org/grpc v1.68.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
-	google.golang.org/protobuf v1.35.1
+	google.golang.org/protobuf v1.35.2
 	k8s.io/api v0.31.2
 	k8s.io/apiextensions-apiserver v0.31.2
 	k8s.io/apimachinery v0.31.2
@@ -63,7 +64,8 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect
 	github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
@@ -87,9 +89,9 @@ require (
 	github.com/google/certificate-transparency-go v1.2.1 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.6 // indirect
@@ -139,11 +141,14 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.33.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
@@ -187,7 +192,7 @@ require (
 	github.com/dave/jennifer v1.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/docker/cli v24.0.7+incompatible // indirect
+	github.com/docker/cli v27.4.1+incompatible
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.2
 	github.com/docker/go-units v0.5.0 // indirect
@@ -235,22 +240,22 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/vladimirvivien/gexe v0.3.0 // indirect
-	go.opentelemetry.io/otel v1.28.0 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
-	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/otel v1.33.0 // indirect
+	go.opentelemetry.io/otel/metric v1.33.0 // indirect
+	go.opentelemetry.io/otel/trace v1.33.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.33.0
-	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect