pull fallback: allow anonymous auth with docker.io and upload manifest from fallback should work

Author: gabivlj

README.md

@@ -110,6 +110,14 @@ the target registry and setup the credentials.
 
 **Never put a registry password/token inside the wrangler.toml, please always use `wrangler secrets put`**
 
+You can also use docker.io with anonymous authentication:
+```
+REGISTRIES_JSON = "[{ \"registry\": \"https://index.docker.io/\" }]"
+```
+
+You can also set your `docker.io` credentials in the configuration to not have any rate-limiting.
+
+
 ### Known limitations
 
 Right now there is some limitations with this container registry.

src/registry/http.ts

@@ -149,14 +149,35 @@ export class RegistryHTTPClient implements Registry {
   }
 
   authBase64(): string {
+    const configuration = this.configuration;
+    if (configuration.username === undefined) {
+      return "";
+    }
+
     return btoa(this.configuration.username + ":" + this.password());
   }
 
   password(): string {
-    return (this.env as unknown as Record<string, string>)[this.configuration.password_env] ?? "";
+    const configuration = this.configuration;
+    if (configuration.username === undefined) {
+      return "";
+    }
+
+    return (this.env as unknown as Record<string, string>)[configuration.password_env] ?? "";
   }
 
   async authenticate(namespace: string): Promise<HTTPContext> {
+    const emptyAuthentication = {
+      authContext: {
+        authType: "none",
+        realm: "",
+        scope: "",
+        service: this.url.host,
+      },
+      repository: this.url.pathname,
+      accessToken: "",
+    } as const;
+
     const res = await fetch(`${this.url.protocol}//${this.url.host}/v2/`, {
       headers: {
         "User-Agent": "Docker-Client/24.0.5 (linux)",
@@ -165,16 +186,7 @@ export class RegistryHTTPClient implements Registry {
     });
 
     if (res.ok) {
-      return {
-        authContext: {
-          authType: "none",
-          realm: "",
-          scope: "",
-          service: this.url.host,
-        },
-        repository: this.url.pathname,
-        accessToken: "",
-      };
+      return emptyAuthentication;
     }
 
     if (res.status !== 401) {
@@ -212,11 +224,12 @@ export class RegistryHTTPClient implements Registry {
   async authenticateBearerSimple(ctx: AuthContext, params: URLSearchParams): Promise<Response> {
     params.delete("password");
     console.log("sending authentication parameters:", ctx.realm + "?" + params.toString());
+
     return await fetch(ctx.realm + "?" + params.toString(), {
       headers: {
-        "Authorization": "Basic " + this.authBase64(),
         "Accept": "application/json",
         "User-Agent": "Docker-Client/24.0.5 (linux)",
+        ...(this.configuration.username !== undefined ? { Authorization: "Basic " + this.authBase64() } : {}),
       },
     });
   }
@@ -227,8 +240,8 @@ export class RegistryHTTPClient implements Registry {
       // explicitely include that we don't want an offline_token.
       scope: `repository:${ctx.scope}:pull,push`,
       client_id: "r2registry",
-      grant_type: "password",
-      password: this.password(),
+      grant_type: this.configuration.username === undefined ? "none" : "password",
+      password: this.configuration.username === undefined ? "" : this.password(),
     });
     let response = await fetch(ctx.realm, {
       headers: {
@@ -468,7 +481,7 @@ export class RegistryHTTPClient implements Registry {
     _namespace: string,
     _reference: string,
     _readableStream: ReadableStream<any>,
-    _contentType: string,
+    {}: {},
   ): Promise<PutManifestResponse | RegistryError> {
     throw new Error("unimplemented");
   }

src/registry/r2.ts

@@ -292,11 +292,23 @@ export class R2Registry implements Registry {
     namespace: string,
     reference: string,
     readableStream: ReadableStream<any>,
-    contentType: string,
+    {
+      contentType,
+      checkLayers,
+    }: {
+      contentType: string;
+      checkLayers?: boolean;
+    },
   ): Promise<PutManifestResponse | RegistryError> {
     const key = await this.gc.markForInsertion(namespace);
     try {
-      return this.putManifestInner(namespace, reference, readableStream, contentType);
+      return this.putManifestInner(
+        namespace,
+        reference,
+        readableStream,
+        contentType,
+        checkLayers !== undefined ? checkLayers === true : true,
+      );
     } finally {
       // if this fails, at some point it will be expired
       await this.gc.cleanInsertion(namespace, key);
@@ -308,6 +320,7 @@ export class R2Registry implements Registry {
     reference: string,
     readableStream: ReadableStream<any>,
     contentType: string,
+    checkLayers: boolean,
   ): Promise<PutManifestResponse | RegistryError> {
     const gcMarker = await this.gc.getGCMarker(name);
     const env = this.env;
@@ -322,8 +335,10 @@ export class R2Registry implements Registry {
     const text = await blob.text();
     const manifestJSON = JSON.parse(text);
     const manifest = manifestSchema.parse(manifestJSON);
-    const verifyManifestErr = await this.verifyManifest(name, manifest);
-    if (verifyManifestErr !== null) return { response: verifyManifestErr };
+    if (checkLayers) {
+      const verifyManifestErr = await this.verifyManifest(name, manifest);
+      if (verifyManifestErr !== null) return { response: verifyManifestErr };
+    }
 
     if (!(await this.gc.checkCanInsertData(name, gcMarker))) {
       console.error("Manifest can't be uploaded as there is/was a garbage collection going");
@@ -774,7 +789,7 @@ export class R2Registry implements Registry {
     if (hashedState === null || !hashedState.state) {
       return {
         response: new Response(null, { status: 404 }),
-      }
+      };
     }
     const state = hashedState.state;
 

src/registry/registry.ts

@@ -5,12 +5,18 @@ import z from "zod";
 import { GarbageCollectionMode } from "./garbage-collector";
 
 // Defines a registry and how it's configured
-const registryConfiguration = z.object({
-  registry: z.string().url(),
-  password_env: z.string(),
-  username: z.string(),
-});
-
+const registryConfiguration = z
+  .object({
+    registry: z.string().url(),
+  })
+  .and(
+    z
+      .object({
+        password_env: z.string(),
+        username: z.string(),
+      })
+      .or(z.undefined()),
+  );
 export type RegistryConfiguration = z.infer<typeof registryConfiguration>;
 
 export function registries(env: Env): RegistryConfiguration[] {
@@ -132,7 +138,10 @@ export interface Registry {
     namespace: string,
     reference: string,
     readableStream: ReadableStream<any>,
-    contentType: string,
+    options: {
+      contentType: string;
+      checkLayers?: boolean;
+    },
   ): Promise<PutManifestResponse | RegistryError>;
 
   // starts a new upload

src/router.ts

@@ -149,7 +149,10 @@ v2Router.head("/:name+/manifests/:reference", async (req, env: Env) => {
         }
 
         const [putResponse, err] = await wrap(
-          env.REGISTRY_CLIENT.putManifest(name, reference, manifestResponse.stream, manifestResponse.contentType),
+          env.REGISTRY_CLIENT.putManifest(name, reference, manifestResponse.stream, {
+            contentType: manifestResponse.contentType,
+            checkLayers: false,
+          }),
         );
         if (err) {
           console.error("Error sync manifest into client:", errorString(err));
@@ -209,7 +212,10 @@ v2Router.get("/:name+/manifests/:reference", async (req, env: Env, context: Exec
     context.waitUntil(
       (async () => {
         const [response, err] = await wrap(
-          env.REGISTRY_CLIENT.putManifest(name, reference, s2, getManifestResponse.contentType),
+          env.REGISTRY_CLIENT.putManifest(name, reference, s2, {
+            contentType: getManifestResponse.contentType,
+            checkLayers: false,
+          }),
         );
         if (err) {
           console.error("Error uploading asynchronously the manifest ", reference, "into main registry");
@@ -243,7 +249,7 @@ v2Router.put("/:name+/manifests/:reference", async (req, env: Env) => {
 
   const { name, reference } = req.params;
   const [res, err] = await wrap<PutManifestResponse | RegistryError, Error>(
-    env.REGISTRY_CLIENT.putManifest(name, reference, req.body!, req.headers.get("Content-Type")!),
+    env.REGISTRY_CLIENT.putManifest(name, reference, req.body!, { contentType: req.headers.get("Content-Type")! }),
   );
   if (err) {
     console.error("Error putting manifest:", errorString(err));
@@ -563,9 +569,7 @@ v2Router.get("/:name+/tags/list", async (req, env: Env) => {
       cursor: tags.cursor,
     });
     // Filter out sha256 manifest
-    manifestTags = manifestTags.concat(
-      tags.objects.filter((tag) => !tag.key.startsWith(`${name}/manifests/sha256:`)),
-    );
+    manifestTags = manifestTags.concat(tags.objects.filter((tag) => !tag.key.startsWith(`${name}/manifests/sha256:`)));
   }
 
   const keys = manifestTags.map((object) => object.key.split("/").pop()!);