From d585ff9bafac0d14129c160a7ebde0304b28b947 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 25 Sep 2025 14:25:39 +0200 Subject: [PATCH] use containerd registry client Signed-off-by: Nicolas De Loof --- internal/ocipush/push.go | 72 +++++++++++++++++++++-------------- internal/registry/registry.go | 18 ++++++--- pkg/compose/publish.go | 55 ++++++++++++++------------ pkg/compose/publish_test.go | 19 ++++----- pkg/compose/pull.go | 2 +- pkg/compose/push.go | 2 +- 6 files changed, 95 insertions(+), 73 deletions(-) diff --git a/internal/ocipush/push.go b/internal/ocipush/push.go index 7cc78997d77..194a3c1edd7 100644 --- a/internal/ocipush/push.go +++ b/internal/ocipush/push.go @@ -26,9 +26,10 @@ import ( "slices" "time" + "github.com/containerd/containerd/v2/core/remotes" pusherrors "github.com/containerd/containerd/v2/core/remotes/errors" + "github.com/containerd/errdefs" "github.com/distribution/reference" - "github.com/docker/buildx/util/imagetools" "github.com/docker/compose/v2/pkg/api" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" @@ -67,11 +68,6 @@ var clientAuthStatusCodes = []int{ http.StatusProxyAuthRequired, } -type Pushable struct { - Descriptor v1.Descriptor - Data []byte -} - func DescriptorForComposeFile(path string, content []byte) v1.Descriptor { return v1.Descriptor{ MediaType: ComposeYAMLMediaType, @@ -81,6 +77,7 @@ func DescriptorForComposeFile(path string, content []byte) v1.Descriptor { "com.docker.compose.version": api.ComposeVersion, "com.docker.compose.file": filepath.Base(path), }, + Data: content, } } @@ -93,27 +90,23 @@ func DescriptorForEnvFile(path string, content []byte) v1.Descriptor { "com.docker.compose.version": api.ComposeVersion, "com.docker.compose.envfile": filepath.Base(path), }, + Data: content, } } -func PushManifest( - ctx context.Context, - resolver *imagetools.Resolver, - named reference.Named, - layers []Pushable, - ociVersion api.OCIVersion, -) error { +func PushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error { // Check if we need an extra empty layer for the manifest config if ociVersion == api.OCIVersion1_1 || ociVersion == "" { - if err := resolver.Push(ctx, named, v1.DescriptorEmptyJSON, v1.DescriptorEmptyJSON.Data); err != nil { + err := push(ctx, resolver, named, v1.DescriptorEmptyJSON) + if err != nil { return err } } // prepare to push the manifest by pushing the layers layerDescriptors := make([]v1.Descriptor, len(layers)) for i := range layers { - layerDescriptors[i] = layers[i].Descriptor - if err := resolver.Push(ctx, named, layers[i].Descriptor, layers[i].Data); err != nil { + layerDescriptors[i] = layers[i] + if err := push(ctx, resolver, named, layers[i]); err != nil { return err } } @@ -135,19 +128,38 @@ func PushManifest( return err } -func createAndPushManifest( - ctx context.Context, - resolver *imagetools.Resolver, - named reference.Named, - layers []v1.Descriptor, - ociVersion api.OCIVersion, -) error { +func push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, descriptor v1.Descriptor) error { + fullRef, err := reference.WithDigest(reference.TagNameOnly(ref), descriptor.Digest) + if err != nil { + return err + } + + pusher, err := resolver.Pusher(ctx, fullRef.String()) + if err != nil { + return err + } + push, err := pusher.Push(ctx, descriptor) + if errdefs.IsAlreadyExists(err) { + return nil + } + if err != nil { + return err + } + defer func() { + _ = push.Close() + }() + + _, err = push.Write(descriptor.Data) + return err +} + +func createAndPushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error { toPush, err := generateManifest(layers, ociVersion) if err != nil { return err } for _, p := range toPush { - err = resolver.Push(ctx, named, p.Descriptor, p.Data) + err = push(ctx, resolver, named, p) if err != nil { return err } @@ -163,8 +175,8 @@ func isNonAuthClientError(statusCode int) bool { return !slices.Contains(clientAuthStatusCodes, statusCode) } -func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]Pushable, error) { - var toPush []Pushable +func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.Descriptor, error) { + var toPush []v1.Descriptor var config v1.Descriptor var artifactType string switch ociCompat { @@ -184,16 +196,17 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]Pusha MediaType: ComposeEmptyConfigMediaType, Digest: digest.FromBytes(configData), Size: int64(len(configData)), + Data: configData, } // N.B. OCI 1.0 does NOT support specifying the artifact type, so it's // left as an empty string to omit it from the marshaled JSON artifactType = "" - toPush = append(toPush, Pushable{Descriptor: config, Data: configData}) + toPush = append(toPush, config) case api.OCIVersion1_1: config = v1.DescriptorEmptyJSON artifactType = ComposeProjectArtifactType // N.B. the descriptor has the data embedded in it - toPush = append(toPush, Pushable{Descriptor: config, Data: make([]byte, len(config.Data))}) + toPush = append(toPush, config) default: return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat) } @@ -220,7 +233,8 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]Pusha "com.docker.compose.version": api.ComposeVersion, }, ArtifactType: artifactType, + Data: manifest, } - toPush = append(toPush, Pushable{Descriptor: manifestDescriptor, Data: manifest}) + toPush = append(toPush, manifestDescriptor) return toPush, nil } diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 76433ca4b04..0ee73883070 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -16,22 +16,28 @@ package registry -import "github.com/distribution/reference" - const ( + // DefaultNamespace is the default namespace + DefaultNamespace = "docker.io" + // DefaultRegistryHost is the hostname for the default (Docker Hub) registry + // used for pushing and pulling images. This hostname is hard-coded to handle + // the conversion from image references without registry name (e.g. "ubuntu", + // or "ubuntu:latest"), as well as references using the "docker.io" domain + // name, which is used as canonical reference for images on Docker Hub, but + // does not match the domain-name of Docker Hub's registry. + DefaultRegistryHost = "registry-1.docker.io" // IndexHostname is the index hostname, used for authentication and image search. IndexHostname = "index.docker.io" // IndexServer is used for user auth and image search - IndexServer = "https://index.docker.io/v1/" + IndexServer = "https://" + IndexHostname + "/v1/" // IndexName is the name of the index IndexName = "docker.io" ) // GetAuthConfigKey special-cases using the full index address of the official // index as the AuthConfig key, and uses the (host)name[:port] for private indexes. -func GetAuthConfigKey(reposName reference.Named) string { - indexName := reference.Domain(reposName) - if indexName == IndexName || indexName == IndexHostname { +func GetAuthConfigKey(indexName string) string { + if indexName == IndexName || indexName == IndexHostname || indexName == DefaultRegistryHost { return IndexServer } return indexName diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 3408b6f083d..623559a92dc 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -27,17 +27,18 @@ import ( "github.com/DefangLabs/secret-detector/pkg/scanner" "github.com/DefangLabs/secret-detector/pkg/secrets" - "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" + "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/distribution/reference" - "github.com/docker/buildx/util/imagetools" "github.com/docker/cli/cli/command" "github.com/docker/compose/v2/internal/ocipush" + "github.com/docker/compose/v2/internal/registry" "github.com/docker/compose/v2/pkg/api" "github.com/docker/compose/v2/pkg/compose/transform" "github.com/docker/compose/v2/pkg/progress" "github.com/docker/compose/v2/pkg/prompt" + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { @@ -64,11 +65,27 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return err } - resolver := imagetools.New(imagetools.Opt{ - Auth: s.configFile(), + config := s.dockerCli.ConfigFile() + + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: docker.ConfigureDefaultRegistries( + docker.WithAuthorizer(docker.NewDockerAuthorizer( + docker.WithAuthCreds(func(host string) (string, string, error) { + host = registry.GetAuthConfigKey(host) + auth, err := config.GetAuthConfig(host) + if err != nil { + return "", "", err + } + if auth.IdentityToken != "" { + return "", auth.IdentityToken, nil + } + return auth.Username, auth.Password, nil + }), + )), + ), }) - var layers []ocipush.Pushable + var layers []v1.Descriptor extFiles := map[string]string{} for _, file := range project.ComposeFiles { data, err := processFile(ctx, file, project, extFiles) @@ -77,10 +94,7 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re } layerDescriptor := ocipush.DescriptorForComposeFile(file, data) - layers = append(layers, ocipush.Pushable{ - Descriptor: layerDescriptor, - Data: data, - }) + layers = append(layers, layerDescriptor) } extLayers, err := processExtends(ctx, project, extFiles) @@ -100,10 +114,7 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re } layerDescriptor := ocipush.DescriptorForComposeFile("image-digests.yaml", yaml) - layers = append(layers, ocipush.Pushable{ - Descriptor: layerDescriptor, - Data: yaml, - }) + layers = append(layers, layerDescriptor) } w := progress.ContextWriter(ctx) @@ -131,8 +142,8 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return nil } -func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]ocipush.Pushable, error) { - var layers []ocipush.Pushable +func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) { + var layers []v1.Descriptor moreExtFiles := map[string]string{} for xf, hash := range extFiles { data, err := processFile(ctx, xf, project, moreExtFiles) @@ -142,10 +153,7 @@ func processExtends(ctx context.Context, project *types.Project, extFiles map[st layerDescriptor := ocipush.DescriptorForComposeFile(hash, data) layerDescriptor.Annotations["com.docker.compose.extends"] = "true" - layers = append(layers, ocipush.Pushable{ - Descriptor: layerDescriptor, - Data: data, - }) + layers = append(layers, layerDescriptor) } for f, hash := range moreExtFiles { if _, ok := extFiles[f]; ok { @@ -343,8 +351,8 @@ func acceptPublishBindMountDeclarations(cli command.Cli) (bool, error) { return confirm, err } -func envFileLayers(project *types.Project) []ocipush.Pushable { - var layers []ocipush.Pushable +func envFileLayers(project *types.Project) []v1.Descriptor { + var layers []v1.Descriptor for _, service := range project.Services { for _, envFile := range service.EnvFiles { f, err := os.ReadFile(envFile.Path) @@ -353,10 +361,7 @@ func envFileLayers(project *types.Project) []ocipush.Pushable { continue } layerDescriptor := ocipush.DescriptorForEnvFile(envFile.Path, f) - layers = append(layers, ocipush.Pushable{ - Descriptor: layerDescriptor, - Data: f, - }) + layers = append(layers, layerDescriptor) } } return layers diff --git a/pkg/compose/publish_test.go b/pkg/compose/publish_test.go index 3cecf595281..8176844337e 100644 --- a/pkg/compose/publish_test.go +++ b/pkg/compose/publish_test.go @@ -23,7 +23,6 @@ import ( "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" - "github.com/docker/compose/v2/internal/ocipush" "github.com/docker/compose/v2/pkg/api" v1 "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" @@ -58,17 +57,15 @@ services: b, err := os.ReadFile("testdata/publish/common.yaml") assert.NilError(t, err) - assert.DeepEqual(t, []ocipush.Pushable{ + assert.DeepEqual(t, []v1.Descriptor{ { - Descriptor: v1.Descriptor{ - MediaType: "application/vnd.docker.compose.file+yaml", - Digest: "sha256:d3ba84507b56ec783f4b6d24306b99a15285f0a23a835f0b668c2dbf9c59c241", - Size: 32, - Annotations: map[string]string{ - "com.docker.compose.extends": "true", - "com.docker.compose.file": "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml", - "com.docker.compose.version": api.ComposeVersion, - }, + MediaType: "application/vnd.docker.compose.file+yaml", + Digest: "sha256:d3ba84507b56ec783f4b6d24306b99a15285f0a23a835f0b668c2dbf9c59c241", + Size: 32, + Annotations: map[string]string{ + "com.docker.compose.extends": "true", + "com.docker.compose.file": "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml", + "com.docker.compose.version": api.ComposeVersion, }, Data: b, }, diff --git a/pkg/compose/pull.go b/pkg/compose/pull.go index aea3403c3b8..684e6635f2e 100644 --- a/pkg/compose/pull.go +++ b/pkg/compose/pull.go @@ -280,7 +280,7 @@ func ImageDigestResolver(ctx context.Context, file *configfile.ConfigFile, apiCl } func encodedAuth(ref reference.Named, configFile driver.Auth) (string, error) { - authConfig, err := configFile.GetAuthConfig(registry.GetAuthConfigKey(ref)) + authConfig, err := configFile.GetAuthConfig(registry.GetAuthConfigKey(reference.Domain(ref))) if err != nil { return "", err } diff --git a/pkg/compose/push.go b/pkg/compose/push.go index 97477eed866..8d8b3691b0e 100644 --- a/pkg/compose/push.go +++ b/pkg/compose/push.go @@ -90,7 +90,7 @@ func (s *composeService) pushServiceImage(ctx context.Context, tag string, confi return err } - authConfig, err := configFile.GetAuthConfig(registry.GetAuthConfigKey(ref)) + authConfig, err := configFile.GetAuthConfig(registry.GetAuthConfigKey(reference.Domain(ref))) if err != nil { return err }