Merge pull request #20 from infosiftr/registry

Use containerd resolver/fetcher to query registry for "skip checking"

Author: yosifkit

Dockerfile

@@ -1,25 +1,29 @@
+FROM golang:1.13-buster AS build
+
+SHELL ["bash", "-Eeuo", "pipefail", "-xc"]
+
+WORKDIR /usr/src/bashbrew
+
+COPY go.mod go.sum ./
+RUN go mod download; go mod verify
+
+COPY . .
+
+RUN CGO_ENABLED=0 ./bashbrew.sh --version; \
+	cp -al bin/bashbrew /
+
 FROM tianon/docker-tianon
 
 SHELL ["bash", "-Eeuo", "pipefail", "-xc"]
 
 RUN apt-get update; \
 	apt-get install -y --no-install-recommends \
 		git \
-		golang-go \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
-WORKDIR /usr/src/bashbrew
-
-COPY go.mod go.sum ./
-RUN go mod download; go mod verify
-
-COPY . .
-RUN export CGO_ENABLED=0; \
-	bash -x ./bashbrew.sh --version; \
-	rm -r ~/.cache/go-build; \
-	mv bin/bashbrew /usr/local/bin/; \
-	bashbrew --version
+COPY --from=build /bashbrew /usr/local/bin/
+RUN bashbrew --version
 
 ENV BASHBREW_CACHE /bashbrew-cache
 # make sure our default cache dir exists and is writable by anyone (similar to /tmp)

cmd/bashbrew/cmd-push.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"time"
 
 	"github.com/urfave/cli"
 )
@@ -47,10 +46,10 @@ func cmdPush(c *cli.Context) error {
 				tag = tagRepo + ":" + tag
 
 				if !force {
-					created := dockerCreated(tag)
-					lastUpdated := fetchDockerHubTagMeta(tag).lastUpdatedTime()
-					if !created.After(lastUpdated) {
-						fmt.Fprintf(os.Stderr, "skipping %s (created %s, last updated %s)\n", tag, created.Local().Format(time.RFC3339), lastUpdated.Local().Format(time.RFC3339))
+					localImageId, _ := dockerInspect("{{.Id}}", tag)
+					registryImageId := fetchRegistryImageId(tag)
+					if registryImageId != "" && localImageId == registryImageId {
+						fmt.Fprintf(os.Stderr, "skipping %s (remote image matches local)\n", tag)
 						continue
 					}
 				}

cmd/bashbrew/cmd-put-shared.go

@@ -4,19 +4,19 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"reflect"
+	"sort"
 	"strings"
-	"time"
 
 	"github.com/urfave/cli"
 
 	"github.com/docker-library/bashbrew/architecture"
 	"github.com/docker-library/bashbrew/manifest"
 )
 
-func entriesToManifestToolYaml(singleArch bool, r Repo, entries ...*manifest.Manifest2822Entry) (string, time.Time, int, error) {
+func entriesToManifestToolYaml(singleArch bool, r Repo, entries ...*manifest.Manifest2822Entry) (string, []string, error) {
 	yaml := ""
-	mru := time.Time{}
-	expectedNumber := 0
+	remoteDigests := []string{}
 	entryIdentifiers := []string{}
 	for _, entry := range entries {
 		entryIdentifiers = append(entryIdentifiers, r.EntryIdentifier(entry))
@@ -41,29 +41,26 @@ func entriesToManifestToolYaml(singleArch bool, r Repo, entries ...*manifest.Man
 			}
 
 			archImage := fmt.Sprintf("%s/%s:%s", archNamespace, r.RepoName, entry.Tags[0])
-			archImageMeta := fetchDockerHubTagMeta(archImage)
-			if archU := archImageMeta.lastUpdatedTime(); archU.After(mru) {
-				mru = archU
-			}
 
-			// count up how many images we expect to push successfully in this manifest list
-			expectedNumber += len(archImageMeta.Images)
-			// for non-manifest-list tags, this will be 1 and for failed lookups it'll be 0
+			// keep track of how many images we expect to push successfully in this manifest list (and what their manifest digests are)
+			// for non-manifest-list tags, this will be exactly 1 and for failed lookups it'll be 0
 			// (and if one of _these_ tags is a manifest list, we've goofed somewhere)
-			if len(archImageMeta.Images) != 1 {
-				fmt.Fprintf(os.Stderr, "warning: expected 1 image for %q; got %d\n", archImage, len(archImageMeta.Images))
+			archImageDigests := fetchRegistryManiestListDigests(archImage)
+			if len(archImageDigests) != 1 {
+				fmt.Fprintf(os.Stderr, "warning: expected 1 image for %q; got %d\n", archImage, len(archImageDigests))
 			}
+			remoteDigests = append(remoteDigests, archImageDigests...)
 
-			yaml += fmt.Sprintf("  - image: %s\n    platform:\n", archImage)
+			yaml += fmt.Sprintf("  - image: %s\n", archImage)
+			yaml += fmt.Sprintf("    platform:\n")
 			yaml += fmt.Sprintf("      os: %s\n", ociArch.OS)
 			yaml += fmt.Sprintf("      architecture: %s\n", ociArch.Architecture)
 			if ociArch.Variant != "" {
 				yaml += fmt.Sprintf("      variant: %s\n", ociArch.Variant)
 			}
 		}
 	}
-
-	return "manifests:\n" + yaml, mru, expectedNumber, nil
+	return "manifests:\n" + yaml, remoteDigests, nil
 }
 
 func tagsToManifestToolYaml(repo string, tags ...string) string {
@@ -130,37 +127,25 @@ func cmdPutShared(c *cli.Context) error {
 
 		failed := []string{}
 		for _, group := range sharedTagGroups {
-			yaml, mostRecentPush, expectedNumber, err := entriesToManifestToolYaml(singleArch, *r, group.Entries...)
+			yaml, expectedRemoteDigests, err := entriesToManifestToolYaml(singleArch, *r, group.Entries...)
 			if err != nil {
 				return err
 			}
 
-			if expectedNumber < 1 {
-				// if "expectedNumber" comes back as 0, we've probably got an API issue, so let's count up what we probably _should_ push
+			sort.Strings(expectedRemoteDigests)
+			if len(expectedRemoteDigests) < 1 {
+				// if "expectedRemoteDigests" comes back empty, we've probably got an API issue (or a build error/push timing problem)
 				fmt.Fprintf(os.Stderr, "warning: no images expected to push for %q\n", fmt.Sprintf("%s:%s", targetRepo, group.SharedTags[0]))
-				for _, entry := range group.Entries {
-					expectedNumber += len(entry.Architectures)
-				}
 			}
 
 			tagsToPush := []string{}
 			for _, tag := range group.SharedTags {
 				image := fmt.Sprintf("%s:%s", targetRepo, tag)
 				if !force {
-					hubMeta := fetchDockerHubTagMeta(image)
-					tagUpdated := hubMeta.lastUpdatedTime()
-					doPush := false
-					if mostRecentPush.After(tagUpdated) {
-						// if one of the images that make up the manifest list has been updated since the manifest list was last pushed, we probably need to push
-						doPush = true
-					}
-					if !singleArch && len(hubMeta.Images) != expectedNumber {
-						// if we're supposed to push more (or less) images than the current manifest list contains, we probably need to push
-						// this _should_ already be accounting for tags that haven't been pushed yet (see notes above in "entriesToManifestToolYaml" where this is calculated)
-						doPush = true
-					}
-					if !doPush {
-						fmt.Fprintf(os.Stderr, "skipping %s (created %s, last updated %s)\n", image, mostRecentPush.Local().Format(time.RFC3339), tagUpdated.Local().Format(time.RFC3339))
+					remoteDigests := fetchRegistryManiestListDigests(image)
+					sort.Strings(remoteDigests)
+					if reflect.DeepEqual(remoteDigests, expectedRemoteDigests) {
+						fmt.Fprintf(os.Stderr, "skipping %s (%d remote digests up-to-date)\n", image, len(remoteDigests))
 						continue
 					}
 				}

cmd/bashbrew/hub.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/remotes"
+	dockerremote "github.com/containerd/containerd/remotes/docker"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var registryImageIdCache = map[string]string{}
+
+// assumes the provided image name is NOT a manifest list (used for testing whether we need to "bashbrew push" or whether the remote image is already up-to-date)
+// this does NOT handle authentication, and will return the empty string for repositories which require it (causing "bashbrew push" to simply shell out to "docker push" which will handle authentication appropriately)
+func fetchRegistryImageId(image string) string {
+	ctx := context.Background()
+
+	ref, resolver, err := fetchRegistryResolveHelper(image)
+	if err != nil {
+		return ""
+	}
+
+	name, desc, err := resolver.Resolve(ctx, ref)
+	if err != nil {
+		return ""
+	}
+
+	if desc.MediaType != images.MediaTypeDockerSchema2Manifest && desc.MediaType != ocispec.MediaTypeImageManifest {
+		return ""
+	}
+
+	digest := desc.Digest.String()
+	if id, ok := registryImageIdCache[digest]; ok {
+		return id
+	}
+
+	fetcher, err := resolver.Fetcher(ctx, name)
+	if err != nil {
+		return ""
+	}
+
+	r, err := fetcher.Fetch(ctx, desc)
+	if err != nil {
+		return ""
+	}
+	defer r.Close()
+
+	var manifest ocispec.Manifest
+	if err := json.NewDecoder(r).Decode(&manifest); err != nil {
+		return ""
+	}
+	id := manifest.Config.Digest.String()
+	if id != "" {
+		registryImageIdCache[digest] = id
+	}
+	return id
+}
+
+var registryManifestListCache = map[string][]string{}
+
+// returns a list of manifest list element digests for the given image name (which might be just one entry, if it's not a manifest list)
+func fetchRegistryManiestListDigests(image string) []string {
+	ctx := context.Background()
+
+	ref, resolver, err := fetchRegistryResolveHelper(image)
+	if err != nil {
+		return nil
+	}
+
+	name, desc, err := resolver.Resolve(ctx, ref)
+	if err != nil {
+		return nil
+	}
+
+	digest := desc.Digest.String()
+	if desc.MediaType == images.MediaTypeDockerSchema2Manifest || desc.MediaType == ocispec.MediaTypeImageManifest {
+		return []string{digest}
+	}
+
+	if desc.MediaType != images.MediaTypeDockerSchema2ManifestList && desc.MediaType != ocispec.MediaTypeImageIndex {
+		return nil
+	}
+
+	if digests, ok := registryManifestListCache[digest]; ok {
+		return digests
+	}
+
+	fetcher, err := resolver.Fetcher(ctx, name)
+	if err != nil {
+		return nil
+	}
+
+	r, err := fetcher.Fetch(ctx, desc)
+	if err != nil {
+		return nil
+	}
+	defer r.Close()
+
+	var manifestList ocispec.Index
+	if err := json.NewDecoder(r).Decode(&manifestList); err != nil {
+		return nil
+	}
+	digests := []string{}
+	for _, manifest := range manifestList.Manifests {
+		if manifest.Digest != "" {
+			digests = append(digests, manifest.Digest.String())
+		}
+	}
+	if len(digests) > 0 {
+		registryManifestListCache[digest] = digests
+	}
+	return digests
+}
+
+func fetchRegistryResolveHelper(image string) (string, remotes.Resolver, error) {
+	ref, err := docker.ParseAnyReference(image)
+	if err != nil {
+		return "", nil, err
+	}
+	if namedRef, ok := ref.(docker.Named); ok {
+		// add ":latest" if necessary
+		namedRef = docker.TagNameOnly(namedRef)
+		ref = namedRef
+	}
+	return ref.String(), dockerremote.NewResolver(dockerremote.ResolverOptions{}), nil
+}

cmd/bashbrew/registry.go

@@ -1,15 +1,24 @@
 module github.com/docker-library/bashbrew
 
-go 1.11
+go 1.13
 
 require (
+	github.com/containerd/containerd v1.4.0
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/urfave/cli v1.22.4
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
 	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
 	golang.org/x/sys v0.0.0-20200821140526-fda516888d29 // indirect
+	google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect
+	google.golang.org/grpc v1.32.0 // indirect
+	gotest.tools/v3 v3.0.2 // indirect
 	pault.ag/go/debian v0.0.0-20190530135403-b831f604d664
 	pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a
 )