Merge pull request jupyterhub#1586 from manics/docker-registry

Add compatibility for fetching tokens from other Docker registries

Author: consideRatio

binderhub/registry.py

@@ -4,6 +4,7 @@
 import base64
 import json
 import os
+import re
 from urllib.parse import urlparse
 
 from tornado import httpclient
@@ -131,8 +132,7 @@ def _default_token_url(self):
         elif self.url.endswith(".docker.io"):
             return "https://auth.docker.io/token?service=registry.docker.io"
         else:
-            # is gcr.io's token url common? If so, it might be worth defaulting
-            # to https://registry.host/v2/token?service=registry.host
+            # If necessary we'll look for the WWW-Authenticate header
             return ""
 
     username = Unicode(
@@ -185,30 +185,87 @@ def _default_password(self):
             base64.b64decode(b64_auth.encode("utf-8")).decode("utf-8").split(":", 1)[1]
         )
 
+    def _parse_www_authenticate_header(self, header):
+        # Header takes the form
+        # WWW-Authenticate: Bearer realm="https://uk-london-1.ocir.io/12345678/docker/token",service="uk-london-1.ocir.io",scope=""
+        self.log.debug(f"Parsing WWW-Authenticate {header}")
+
+        if not header.lower().startswith("bearer "):
+            raise ValueError(f"Only WWW-Authenticate Bearer type supported: {header}")
+        try:
+            realm = re.search(r'realm="([^"]+)"', header).group(1)
+            # Should service and scope parameters be optional instead of just empty?
+            service = re.search(r'service="([^"]*)"', header).group(1)
+            scope = re.search(r'scope="([^"]*)"', header).group(1)
+            return realm, service, scope
+        except AttributeError:
+            raise ValueError(
+                f"Expected WWW-Authenticate to include realm service scope: {header}"
+            ) from None
+
+    async def _get_token(self, client, token_url, service, scope):
+        auth_req = httpclient.HTTPRequest(
+            url_concat(
+                token_url,
+                {
+                    "scope": scope,
+                    "service": service,
+                },
+            ),
+            auth_username=self.username,
+            auth_password=self.password,
+        )
+        self.log.debug(
+            f"Getting registry token from {token_url} service={service} scope={scope}"
+        )
+        auth_resp = await client.fetch(auth_req)
+        response_body = json.loads(auth_resp.body.decode("utf-8", "replace"))
+
+        if "token" in response_body.keys():
+            token = response_body["token"]
+        elif "access_token" in response_body.keys():
+            token = response_body["access_token"]
+        else:
+            raise ValueError(f"No token in response from registry: {response_body}")
+        return token
+
+    async def _get_image_manifest_from_www_authenticate(
+        self, client, www_auth_header, url
+    ):
+        realm, service, scope = self._parse_www_authenticate_header(www_auth_header)
+        token = await self._get_token(client, realm, service, scope)
+        req = httpclient.HTTPRequest(
+            url,
+            headers={"Authorization": f"Bearer {token}"},
+        )
+        self.log.debug(f"Getting image manifest from {url}")
+        try:
+            resp = await client.fetch(req)
+        except httpclient.HTTPError as e:
+            if e.code == 404:
+                return None
+            else:
+                raise
+        return json.loads(resp.body.decode("utf-8"))
+
     async def get_image_manifest(self, image, tag):
+        """
+        Get the manifest for an image.
+
+        image: The image name without the registry and tag
+        tag: The image tag
+        """
         client = httpclient.AsyncHTTPClient()
         url = f"{self.url}/v2/{image}/manifests/{tag}"
+        token = None
         # first, get a token to perform the manifest request
         if self.token_url:
-            auth_req = httpclient.HTTPRequest(
-                url_concat(
-                    self.token_url,
-                    {
-                        "scope": f"repository:{image}:pull",
-                        "service": "container_registry",
-                    },
-                ),
-                auth_username=self.username,
-                auth_password=self.password,
+            token = await self._get_token(
+                client,
+                self.token_url,
+                scope=f"repository:{image}:pull",
+                service="container_registry",
             )
-            auth_resp = await client.fetch(auth_req)
-            response_body = json.loads(auth_resp.body.decode("utf-8", "replace"))
-
-            if "token" in response_body.keys():
-                token = response_body["token"]
-            elif "access_token" in response_body.keys():
-                token = response_body["access_token"]
-
             req = httpclient.HTTPRequest(
                 url,
                 headers={"Authorization": f"Bearer {token}"},
@@ -221,16 +278,26 @@ async def get_image_manifest(self, image, tag):
                 auth_password=self.password,
             )
 
+        self.log.debug(f"Getting image manifest from {url}")
         try:
             resp = await client.fetch(req)
         except httpclient.HTTPError as e:
             if e.code == 404:
                 # 404 means it doesn't exist
                 return None
+            elif (
+                e.code == 401 and not token and "www-authenticate" in e.response.headers
+            ):
+                # Unauthorised. If we don't have a token, try and get one using
+                # information from the WWW-Authenticate header
+                # https://stackoverflow.com/questions/56193110/how-can-i-use-docker-registry-http-api-v2-to-obtain-a-list-of-all-repositories-i/68654659#68654659
+                www_auth_header = e.response.headers["www-authenticate"]
+                return await self._get_image_manifest_from_www_authenticate(
+                    client, www_auth_header, url
+                )
             else:
                 raise
-        else:
-            return json.loads(resp.body.decode("utf-8"))
+        return json.loads(resp.body.decode("utf-8"))
 
 
 class FakeRegistry(DockerRegistry):

binderhub/tests/test_registry.py

@@ -2,7 +2,10 @@
 import base64
 import json
 import os
+from random import randint
 
+import pytest
+from tornado import httpclient
 from tornado.web import Application, HTTPError, RequestHandler
 
 from binderhub.registry import DockerRegistry
@@ -59,17 +62,62 @@ def test_registry_gcr_defaults(tmpdir):
     assert registry.password == "{...}"
 
 
+@pytest.mark.parametrize(
+    "header,expected",
+    [
+        (
+            'Bearer realm="https://example.org/abc/token",service="example.org",scope=""',
+            ("https://example.org/abc/token", "example.org", ""),
+        ),
+        (
+            'BEARER scope="abc",service="example.org",realm="https://example.org/abc/token"',
+            ("https://example.org/abc/token", "example.org", "abc"),
+        ),
+    ],
+)
+def test_parse_www_authenticate_header(header, expected):
+    registry = DockerRegistry()
+    assert expected == registry._parse_www_authenticate_header(header)
+
+
+@pytest.mark.parametrize(
+    "header,expected",
+    [
+        (
+            'basic realm="https://example.org/abc/token"',
+            "Only WWW-Authenticate Bearer type supported",
+        ),
+        (
+            'bearer realm="https://example.org/abc/token"',
+            "Expected WWW-Authenticate to include realm service scope",
+        ),
+    ],
+)
+def test_parse_www_authenticate_header_invalid(header, expected):
+    registry = DockerRegistry()
+    with pytest.raises(ValueError) as excinfo:
+        registry._parse_www_authenticate_header(header)
+    assert excinfo.value.args[0].startswith(expected)
+
+
 # Mock the registry API calls made by get_image_manifest
 
 
 class MockTokenHandler(RequestHandler):
     """Mock handler for the registry token handler"""
 
-    def initialize(self, test_handle):
+    def initialize(self, test_handle, service=None, scope=None):
         self.test_handle = test_handle
+        self.service = service
+        self.scope = scope
 
     def get(self):
-        self.get_argument("scope")
+        scope = self.get_argument("scope")
+        if self.scope:
+            assert scope == self.scope
+        service = self.get_argument("service")
+        if self.service:
+            assert service == self.service
         auth_header = self.request.headers.get("Authorization", "")
         if not auth_header.startswith("Basic "):
             raise HTTPError(401, "No basic auth")
@@ -104,8 +152,52 @@ def get(self, image, tag):
         # get_image_manifest never looks at the contents here
         self.write(json.dumps({"image": image, "tag": tag}))
 
+    def write_error(self, status_code, **kwargs):
+        err_cls, err, traceback = kwargs["exc_info"]
+        if status_code == 401:
+            r = self.request
+            self.set_header(
+                "WWW-Authenticate",
+                f'Bearer realm="{r.protocol}://{r.host}/token",service="service=1",scope="scope-2"',
+            )
+            super().write_error(status_code, **kwargs)
+
+
+async def test_get_token():
+    username = "user"
+    password = "pass"
+    test_handle = {"username": username, "password": password}
+    app = Application(
+        [
+            (r"/token", MockTokenHandler, {"test_handle": test_handle}),
+        ]
+    )
+    ip = "127.0.0.1"
+    port = randint(10000, 65535)
+    app.listen(port, ip)
+
+    registry = DockerRegistry(
+        url="https://example.org", username=username, password=password
+    )
+
+    assert registry.url == "https://example.org"
+    assert registry.auth_config_url == "https://example.org"
+    # token_url should be unset, since it should be determined by the caller from a
+    # WWW-Authenticate header
+    assert registry.token_url == ""
+    assert registry.username == username
+    assert registry.password == password
+    token = await registry._get_token(
+        httpclient.AsyncHTTPClient(),
+        f"http://{ip}:{port}/token",
+        "service.1",
+        "scope.2",
+    )
+    assert token == test_handle["token"]
+
 
-async def test_get_image_manifest(tmpdir, request):
+@pytest.mark.parametrize("token_url_known", [True, False])
+async def test_get_image_manifest(tmpdir, token_url_known):
     username = "asdf"
     password = "asdf;ouyag"
     test_handle = {"username": username, "password": password}
@@ -120,7 +212,7 @@ async def test_get_image_manifest(tmpdir, request):
         ]
     )
     ip = "127.0.0.1"
-    port = 10504
+    port = randint(10000, 65535)
     url = f"http://{ip}:{port}"
     app.listen(port, ip)
     config_json = tmpdir.join("dockerconfig.json")
@@ -137,12 +229,16 @@ async def test_get_image_manifest(tmpdir, request):
             },
             f,
         )
+    if token_url_known:
+        token_url = url + "/token"
+    else:
+        token_url = ""
     registry = DockerRegistry(
-        docker_config_path=str(config_json), token_url=url + "/token", url=url
+        docker_config_path=str(config_json), token_url=token_url, url=url
     )
     assert registry.url == url
     assert registry.auth_config_url == url
-    assert registry.token_url == url + "/token"
+    assert registry.token_url == token_url
     assert registry.username == username
     assert registry.password == password
     manifest = await registry.get_image_manifest("myimage", "abc123")