Implement more fine-grained mutexes in registryCache

In short, this keeps the "data mutex" as-is, but only acquires it when we have some data to shove in (and only for the limited instructions necessary to update the data), and adds a new mutex per-tag or per-digest to prevent concurrent requests for the exact same upstream content (which is the main reason for this cache in the first place, so feels appropriate).

Without this, our current mutex means that any other efforts to parallelize will ultimately bottleneck on our registry cache.

In order to test this effectively, I've added a `--parallel` flag to `lookup` which just hyper-aggressively runs every single lookup in parallel inside a goroutine.

My first test of this was doing a lookup of the first tag of every DOI repository (so ~147 concurrent lookups in total), and I found it was consistently ~1m57s both with and without this change.  Our Hub rate limiter is pegged at ~100/min, which seems consistent with that result.  If I increase that limit to ~200/min, I was able to achieve a small speedup with this change (~43s down to ~30s).

In other words, for this to actually be effective as a speedup against Docker Hub if we implement parallel deploy, for example, we'll *also* have to increase our rate limit (which I think is fairly safe now that we handle 429 by explicitly emptying the limiter).

In order to *really* test this effectively, I took a different approach.  I spun up a local registry (`docker run -dit -p 127.0.0.1:5000:5000 -p [::1]:5000:5000 --name registry registry`), and copied `hello-world:linux` into it 10,000 times (something like `crane cp hello-world:linux localhost:5000/hello && echo localhost:5000/hello:{1..10000} | xargs -rtn1 -P$(nproc) crane cp localhost:5000/hello` -- could also be done with `jq` and `deploy` if you are ambitious 👀).

Then I used `time ./bin/lookup --parallel $(crane ls --full-ref localhost:5000/hello) > /dev/null` to establish some benchmarks.

Without this change, it's pretty consistently in the 15-20s range on my local system.  With this change, it drops down an order of magnitude to be in the 3-6s range.

Author: tianon

cmd/lookup/main.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"sync"
 
 	"github.com/docker-library/meta-scripts/registry"
 )
@@ -22,6 +23,16 @@ func main() {
 	)
 
 	args := os.Args[1:]
+
+	var (
+		parallel = false
+		wg       sync.WaitGroup
+	)
+	if len(args) > 0 && args[0] == "--parallel" {
+		args = args[1:]
+		parallel = true
+	}
+
 	for len(args) > 0 {
 		img := args[0]
 		args = args[1:]
@@ -35,53 +46,66 @@ func main() {
 			continue
 		}
 
-		ref, err := registry.ParseRef(img)
-		if err != nil {
-			panic(err)
-		}
-
-		var obj any
-		if opts == zeroOpts {
-			// if we have no explicit type and didn't request a HEAD, invoke SynthesizeIndex instead of Lookup
-			obj, err = registry.SynthesizeIndex(ctx, ref)
-			if err != nil {
-				panic(err)
-			}
-		} else {
-			r, err := registry.Lookup(ctx, ref, &opts)
+		do := func(opts registry.LookupOptions) {
+			ref, err := registry.ParseRef(img)
 			if err != nil {
 				panic(err)
 			}
-			if r != nil {
-				desc := r.Descriptor()
-				if opts.Head {
-					obj = desc
-				} else {
-					b, err := io.ReadAll(r)
-					if err != nil {
-						r.Close()
-						panic(err)
-					}
-					if opts.Type == registry.LookupTypeManifest {
-						// if it was a manifest lookup, cast the byte slice to json.RawMessage so we get the actual JSON (not base64)
-						obj = json.RawMessage(b)
-					} else {
-						obj = b
-					}
-				}
-				err = r.Close()
+
+			var obj any
+			if opts == zeroOpts {
+				// if we have no explicit type and didn't request a HEAD, invoke SynthesizeIndex instead of Lookup
+				obj, err = registry.SynthesizeIndex(ctx, ref)
 				if err != nil {
 					panic(err)
 				}
 			} else {
-				obj = nil
+				r, err := registry.Lookup(ctx, ref, &opts)
+				if err != nil {
+					panic(err)
+				}
+				if r != nil {
+					desc := r.Descriptor()
+					if opts.Head {
+						obj = desc
+					} else {
+						b, err := io.ReadAll(r)
+						if err != nil {
+							r.Close()
+							panic(err)
+						}
+						if opts.Type == registry.LookupTypeManifest {
+							// if it was a manifest lookup, cast the byte slice to json.RawMessage so we get the actual JSON (not base64)
+							obj = json.RawMessage(b)
+						} else {
+							obj = b
+						}
+					}
+					err = r.Close()
+					if err != nil {
+						panic(err)
+					}
+				} else {
+					obj = nil
+				}
+			}
+
+			e := json.NewEncoder(os.Stdout)
+			e.SetIndent("", "\t")
+			if err := e.Encode(obj); err != nil {
+				panic(err)
 			}
 		}
 
-		e := json.NewEncoder(os.Stdout)
-		e.SetIndent("", "\t")
-		if err := e.Encode(obj); err != nil {
-			panic(err)
+		if parallel {
+			wg.Add(1)
+			go func(opts registry.LookupOptions) {
+				defer wg.Done()
+				// TODO synchronize output so that it still arrives in-order?  maybe the randomness is part of the charm?
+				do(opts)
+			}(opts)
+		} else {
+			do(opts)
 		}
 
 		// reset state
@@ -91,4 +115,8 @@ func main() {
 	if opts != zeroOpts {
 		panic("dangling --type, --head, etc (without a following reference for it to apply to)")
 	}
+
+	if parallel {
+		wg.Wait()
+	}
 }