Add a Reference wrapper type to ociref.Reference to handle our "Docker references" logic more cleanly

This also allows us to add explicit JSON round-tripping which can handle normalizing/denormalizing for us, so our output strings no longer contain `docker.io/[library/]`. 🎉

Author: tianon

.test/builds.json

@@ -9,7 +9,7 @@
         "size": 946,
         "annotations": {
           "com.docker.official-images.bashbrew.arch": "windows-amd64",
-          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+          "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
         },
         "platform": {
           "architecture": "amd64",
@@ -19,7 +19,7 @@
       }
     ],
     "annotations": {
-      "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+      "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
     }
   },
   {
@@ -32,7 +32,7 @@
         "size": 861,
         "annotations": {
           "com.docker.official-images.bashbrew.arch": "amd64",
-          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
+          "org.opencontainers.image.ref.name": "tianon/test@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
           "org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee",
           "org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world",
           "org.opencontainers.image.url": "https://hub.docker.com/_/hello-world",
@@ -49,7 +49,7 @@
         "size": 946,
         "annotations": {
           "com.docker.official-images.bashbrew.arch": "windows-amd64",
-          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+          "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
         },
         "platform": {
           "architecture": "amd64",
@@ -63,7 +63,7 @@
         "size": 946,
         "annotations": {
           "com.docker.official-images.bashbrew.arch": "windows-amd64",
-          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5"
+          "org.opencontainers.image.ref.name": "tianon/test@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5"
         },
         "platform": {
           "architecture": "amd64",
@@ -73,7 +73,7 @@
       }
     ],
     "annotations": {
-      "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac"
+      "org.opencontainers.image.ref.name": "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac"
     }
   }
 ]

.test/cache-builds.json

@@ -55,7 +55,7 @@ var (
 )
 
 func resolveIndex(ctx context.Context, img string, diskCacheForSure bool) (*ocispec.Index, error) {
-	ref, err := registry.ParseRefNormalized(img)
+	ref, err := registry.ParseRef(img)
 	if err != nil {
 		return nil, err
 	}
@@ -90,7 +90,7 @@ func resolveIndex(ctx context.Context, img string, diskCacheForSure bool) (*ocis
 	if diskCacheForSure {
 		saveCacheMutex.Lock()
 		if saveCache != nil {
-			saveCache.Indexes[refString] = index
+			saveCache.Indexes[ref] = index
 		}
 		saveCacheMutex.Unlock()
 	}
@@ -137,7 +137,7 @@ func resolveArchIndex(ctx context.Context, img string, arch string, diskCacheFor
 }
 
 type cacheFileContents struct {
-	Indexes map[string]*ocispec.Index `json:"indexes"`
+	Indexes map[registry.Reference]*ocispec.Index `json:"indexes"`
 }
 
 var (
@@ -152,7 +152,7 @@ func loadCacheFromFile() error {
 
 	// now that we know we have a file we want cache to go into (and come from), let's initialize the "saveCache" (which will be written when the whole process is done / we're successful, and *only* caches staging images)
 	saveCacheMutex.Lock()
-	saveCache = &cacheFileContents{Indexes: map[string]*ocispec.Index{}}
+	saveCache = &cacheFileContents{Indexes: map[registry.Reference]*ocispec.Index{}}
 	saveCacheMutex.Unlock()
 
 	f, err := os.Open(cacheFile)
@@ -180,7 +180,7 @@ func loadCacheFromFile() error {
 			panic(err)
 		}
 		if index2 != index {
-			panic("index2 != index??? " + img)
+			panic("index2 != index??? " + img.String())
 		}
 	}
 

.test/lookup-test.json

@@ -16,7 +16,7 @@ func main() {
 	defer stop()
 
 	for _, img := range os.Args[1:] {
-		ref, err := registry.ParseRefNormalized(img)
+		ref, err := registry.ParseRef(img)
 		if err != nil {
 			panic(err)
 		}

cmd/builds/main.go

@@ -10,23 +10,56 @@ import (
 	"cuelabs.dev/go/oci/ociregistry/ociref"
 )
 
-// parse a ref like `hello-world:latest` into an [ociref.Reference] object, with Docker Hub canonicalization applied: `docker.io/library/hello-world:latest`
+// parse a string ref like `hello-world:latest` directly into a [Reference] object, with Docker Hub canonicalization applied: `docker.io/library/hello-world:latest`
 //
-// See also [ociref.ParseRelative]
-//
-// NOTE: this explicitly does *not* normalize Tag to `:latest` because it's useful to be able to parse a reference and know it did not specify either tag or digest (and `if ref.Tag == "" { ref.Tag = "latest" }` is really trivial code outside this for that case)
-func ParseRefNormalized(img string) (ociref.Reference, error) {
-	ref, err := ociref.ParseRelative(img)
+// See also [Reference.Normalize] and [ociref.ParseRelative] (which are the underlying implementation details of this method).
+func ParseRef(img string) (Reference, error) {
+	r, err := ociref.ParseRelative(img)
 	if err != nil {
-		return ociref.Reference{}, err
+		return Reference{}, err
 	}
+	ref := Reference(r)
+	ref.Normalize()
+	return ref, nil
+}
+
+// copy ociref.Reference so we can add methods (especially for JSON round-trip, but also Docker-isms like the implied default [Reference.Host] and `library/` prefix for DOI)
+type Reference ociref.Reference
+
+// normalize Docker Hub refs like `hello-world:latest`: `docker.io/library/hello-world:latest`
+//
+// NOTE: this explicitly does *not* normalize Tag to `:latest` because it's useful to be able to parse a reference and know it did not specify either tag or digest (and `if ref.Tag == "" { ref.Tag = "latest" }` is really trivial code outside this for that case)
+func (ref *Reference) Normalize() {
 	if dockerHubHosts[ref.Host] {
 		// normalize Docker Hub host value
 		ref.Host = dockerHubCanonical
 		// normalize Docker Official Images to library/ prefix
 		if !strings.Contains(ref.Repository, "/") {
 			ref.Repository = "library/" + ref.Repository
 		}
+		// add an error return and return an error if we have more than one "/" in Repository?  probably not worth embedding that many "Hub" implementation details this low (since it'll error appropriately on use of such invalid references anyhow)
 	}
-	return ref, nil
+}
+
+// like [ociref.Reference.String], but with Docker Hub "denormalization" applied (no explicit `docker.io` host, no `library/` prefix for DOI)
+func (ref Reference) String() string {
+	if ref.Host == dockerHubCanonical {
+		ref.Host = ""
+		ref.Repository = strings.TrimPrefix(ref.Repository, "library/")
+	}
+	return ociref.Reference(ref).String()
+}
+
+// implements [encoding.TextMarshaler] (especially for [Reference]-in-JSON)
+func (ref Reference) MarshalText() ([]byte, error) {
+	return []byte(ref.String()), nil
+}
+
+// implements [encoding.TextUnmarshaler] (especially for [Reference]-from-JSON)
+func (ref *Reference) UnmarshalText(text []byte) error {
+	r, err := ParseRef(string(text))
+	if err == nil {
+		*ref = r
+	}
+	return err
 }

cmd/lookup/main.go

@@ -1,12 +1,33 @@
 package registry_test
 
 import (
+	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/docker-library/meta-scripts/registry"
+
+	"cuelabs.dev/go/oci/ociregistry/ociref"
 )
 
-func TestParseRefNormalized(t *testing.T) {
+func toJson(t *testing.T, v any) string {
+	t.Helper()
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatal("unexpected JSON error", err)
+	}
+	return string(b)
+}
+
+func fromJson(t *testing.T, j string, v any) {
+	t.Helper()
+	err := json.Unmarshal([]byte(j), v)
+	if err != nil {
+		t.Fatal("unexpected JSON error", err)
+	}
+}
+
+func TestParseRef(t *testing.T) {
 	t.Parallel()
 
 	for _, o := range []struct {
@@ -28,17 +49,38 @@ func TestParseRefNormalized(t *testing.T) {
 		{"registry.hub.docker.com/library/hello-world", "docker.io/library/hello-world"},
 	} {
 		o := o // https://github.com/golang/go/issues/60078
+		dockerOut := strings.TrimPrefix(strings.TrimPrefix(o.out, "docker.io/library/"), "docker.io/")
+
 		t.Run(o.in, func(t *testing.T) {
-			ref, err := registry.ParseRefNormalized(o.in)
+			ref, err := registry.ParseRef(o.in)
 			if err != nil {
 				t.Fatal("unexpected error", err)
-				return
 			}
 
-			out := ref.String()
+			out := ociref.Reference(ref).String()
 			if out != o.out {
 				t.Fatalf("expected %q, got %q", o.out, out)
-				return
+			}
+
+			out = ref.String()
+			if out != dockerOut {
+				t.Fatalf("expected %q, got %q", dockerOut, out)
+			}
+		})
+
+		t.Run(o.in+" JSON", func(t *testing.T) {
+			json := toJson(t, o.in) // "hello-world:latest" (string straight to JSON so we can unmarshal it as a Reference)
+			var ref registry.Reference
+			fromJson(t, json, &ref)
+			out := ociref.Reference(ref).String()
+			if out != o.out {
+				t.Fatalf("expected %q, got %q", o.out, out)
+			}
+
+			json = toJson(t, ref)   // "hello-world:latest" (take our reference and convert it to JSON so we can verify it goes out correctly)
+			fromJson(t, json, &out) // back to a string
+			if out != dockerOut {
+				t.Fatalf("expected %q, got %q", dockerOut, out)
 			}
 		})
 	}

registry/ref.go

@@ -9,12 +9,11 @@ import (
 	"github.com/docker-library/bashbrew/architecture"
 
 	"cuelabs.dev/go/oci/ociregistry"
-	"cuelabs.dev/go/oci/ociregistry/ociref"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // returns a synthesized [ocispec.Index] object for the given reference that includes automatically pulling up [ocispec.Platform] objects for entries missing them plus annotations for bashbrew architecture ([AnnotationBashbrewArch]) and where to find the "upstream" object if it needs to be copied/pulled ([ocispec.AnnotationRefName])
-func SynthesizeIndex(ctx context.Context, ref ociref.Reference) (*ocispec.Index, error) {
+func SynthesizeIndex(ctx context.Context, ref Reference) (*ocispec.Index, error) {
 	// consider making this a full ociregistry.Interface object? GetManifest(digest) not returning an object with that digest would certainly be Weird though so maybe that's a misguided idea (with very minimal actual benefit, at least right now)
 
 	client, err := Client(ref.Host, nil)
@@ -137,8 +136,8 @@ func SynthesizeIndex(ctx context.Context, ref ociref.Reference) (*ocispec.Index,
 	return &index, nil
 }
 
-// given a (potentially `nil`) map of annotations, add [ocispec.AnnotationRefName] including the supplied [ociref.Reference] (but with [ociref.Reference.Digest] set to a new value)
-func setRefAnnotation(annotations *map[string]string, ref ociref.Reference, digest ociregistry.Digest) {
+// given a (potentially `nil`) map of annotations, add [ocispec.AnnotationRefName] including the supplied [Reference] (but with [Reference.Digest] set to a new value)
+func setRefAnnotation(annotations *map[string]string, ref Reference, digest ociregistry.Digest) {
 	if *annotations == nil {
 		// "assignment to nil map" 🙃
 		*annotations = map[string]string{}
@@ -148,7 +147,7 @@ func setRefAnnotation(annotations *map[string]string, ref ociref.Reference, dige
 }
 
 // given a manifest descriptor (and optionally an existing [ociregistry.BlobReader] on the manifest object itself), make sure it has a valid [ocispec.Platform] object if possible, querying down into the [ocispec.Image] ("config" blob) if necessary
-func normalizeManifestPlatform(ctx context.Context, m *ocispec.Descriptor, r ociregistry.BlobReader, client ociregistry.Interface, ref ociref.Reference) error {
+func normalizeManifestPlatform(ctx context.Context, m *ocispec.Descriptor, r ociregistry.BlobReader, client ociregistry.Interface, ref Reference) error {
 	if m.Platform == nil || m.Platform.OS == "" || m.Platform.Architecture == "" {
 		// if missing (or obviously invalid) "platform", we need to (maybe) reach downwards and synthesize
 		m.Platform = nil