ref_url and resolved_spec methods for repo providers (#974)

ref_url and resolved_spec methods for repo providers

Author: betatim

binderhub/builder.py

@@ -313,16 +313,28 @@ async def get(self, provider_prefix, _unescaped_spec):
             push_secret = None
 
         BuildClass = FakeBuild if self.settings.get('fake_build') else Build
+
         binder_url = '{proto}://{host}{base_url}v2/{provider}/{spec}'.format(
             proto=self.request.protocol,
             host=self.request.host,
             base_url=self.settings['base_url'],
             provider=provider_prefix,
             spec=spec,
         )
+        resolved_spec = await provider.get_resolved_spec()
+        persistent_binder_url = '{proto}://{host}{base_url}v2/{provider}/{spec}'.format(
+            proto=self.request.protocol,
+            host=self.request.host,
+            base_url=self.settings['base_url'],
+            provider=provider_prefix,
+            spec=resolved_spec,
+        )
+        ref_url = await provider.get_resolved_ref_url()
         appendix = self.settings['appendix'].format(
             binder_url=binder_url,
             repo_url=repo_url,
+            persistent_binder_url=persistent_binder_url,
+            ref_url=ref_url,
         )
 
         self.build = build = BuildClass(

binderhub/repoproviders.py

@@ -164,10 +164,20 @@ def repo_config(self, settings):
     def get_resolved_ref(self):
         raise NotImplementedError("Must be overridden in child class")
 
+    @gen.coroutine
+    def get_resolved_spec(self):
+        """Return the spec with resolved ref."""
+        raise NotImplementedError("Must be overridden in child class")
+
     def get_repo_url(self):
         """Return the git clone-able repo URL"""
         raise NotImplementedError("Must be overridden in the child class")
 
+    @gen.coroutine
+    def get_resolved_ref_url(self):
+        """Return the URL of repository at this commit in history"""
+        raise NotImplementedError("Must be overridden in child class")
+
     def get_build_slug(self):
         """Return a unique build slug"""
         raise NotImplementedError("Must be overriden in the child class")
@@ -185,9 +195,15 @@ class FakeProvider(RepoProvider):
     async def get_resolved_ref(self):
         return "1a2b3c4d5e6f"
 
+    async def get_resolved_spec(self):
+        return "fake/repo/1a2b3c4d5e6f"
+
     def get_repo_url(self):
         return "https://example.com/fake/repo.git"
 
+    async def get_resolved_ref_url(self):
+        return "https://example.com/fake/repo/tree/1a2b3c4d5e6f"
+
     def get_build_slug(self):
         return '{user}-{repo}'.format(user='Rick', repo='Morty')
 
@@ -208,11 +224,25 @@ def get_resolved_ref(self):
         self.record_id = r.effective_url.rsplit("/", maxsplit=1)[1]
         return self.record_id
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'record_id'):
+            self.record_id = await self.get_resolved_ref()
+        # zenodo registers a DOI which represents all versions of a software package
+        # and it always resolves to latest version
+        # for that case, we have to replace the version number in DOIs with
+        # the specific (resolved) version (record_id)
+        resolved_spec = self.spec.split("zenodo")[0] + "zenodo." + self.record_id
+        return resolved_spec
+
     def get_repo_url(self):
         # While called repo URL, the return value of this function is passed
         # as argument to repo2docker, hence we return the spec as is.
         return self.spec
 
+    async def get_resolved_ref_url(self):
+        resolved_spec = await self.get_resolved_spec()
+        return f"https://doi.org/{resolved_spec}"
+
     def get_build_slug(self):
         return "zenodo-{}".format(self.record_id)
 
@@ -241,11 +271,25 @@ def get_resolved_ref(self):
 
         return self.record_id
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'record_id'):
+            self.record_id = await self.get_resolved_ref()
+
+        # spec without version is accepted as version 1 - check get_resolved_ref method
+        # for that case, we have to replace the version number in DOIs with
+        # the specific (resolved) version (record_id)
+        resolved_spec = self.spec.split("figshare")[0] + "figshare." + self.record_id
+        return resolved_spec
+
     def get_repo_url(self):
         # While called repo URL, the return value of this function is passed
         # as argument to repo2docker, hence we return the spec as is.
         return self.spec
 
+    async def get_resolved_ref_url(self):
+        resolved_spec = await self.get_resolved_spec()
+        return f"https://doi.org/{resolved_spec}"
+
     def get_build_slug(self):
         return "figshare-{}".format(self.record_id)
 
@@ -270,8 +314,8 @@ class GitRepoProvider(RepoProvider):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        url, unresolved_ref = self.spec.split('/', 1)
-        self.repo = urllib.parse.unquote(url)
+        self.url, unresolved_ref = self.spec.split('/', 1)
+        self.repo = urllib.parse.unquote(self.url)
         self.unresolved_ref = urllib.parse.unquote(unresolved_ref)
         if not self.unresolved_ref:
             raise ValueError("`unresolved_ref` must be specified as a query parameter for the basic git provider")
@@ -302,9 +346,18 @@ def get_resolved_ref(self):
 
         return self.resolved_ref
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f"{self.url}/{self.resolved_ref}"
+
     def get_repo_url(self):
         return self.repo
 
+    async def get_resolved_ref_url(self):
+        # not possible to construct ref url of unknown git provider
+        return self.get_repo_url()
+
     def get_build_slug(self):
         return self.repo
 
@@ -374,8 +427,8 @@ def _default_git_credentials(self):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        quoted_namespace, unresolved_ref = self.spec.split('/', 1)
-        self.namespace = urllib.parse.unquote(quoted_namespace)
+        self.quoted_namespace, unresolved_ref = self.spec.split('/', 1)
+        self.namespace = urllib.parse.unquote(self.quoted_namespace)
         self.unresolved_ref = urllib.parse.unquote(unresolved_ref)
         if not self.unresolved_ref:
             raise ValueError("An unresolved ref is required")
@@ -410,13 +463,22 @@ def get_resolved_ref(self):
         self.resolved_ref = ref_info['id']
         return self.resolved_ref
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f"{self.quoted_namespace}/{self.resolved_ref}"
+
     def get_build_slug(self):
         # escape the name and replace dashes with something else.
         return '-'.join(p.replace('-', '_-') for p in self.namespace.split('/'))
 
     def get_repo_url(self):
-        return "https://{hostname}/{namespace}.git".format(
-            hostname=self.hostname, namespace=self.namespace)
+        return f"https://{self.hostname}/{self.namespace}.git"
+
+    async def get_resolved_ref_url(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f"https://{self.hostname}/{self.namespace}/tree/{self.resolved_ref}"
 
 
 class GitHubRepoProvider(RepoProvider):
@@ -500,8 +562,12 @@ def __init__(self, *args, **kwargs):
         self.repo = strip_suffix(self.repo, ".git")
 
     def get_repo_url(self):
-        return "https://{hostname}/{user}/{repo}".format(
-            hostname=self.hostname, user=self.user, repo=self.repo)
+        return f"https://{self.hostname}/{self.user}/{self.repo}"
+
+    async def get_resolved_ref_url(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f"https://{self.hostname}/{self.user}/{self.repo}/tree/{self.resolved_ref}"
 
     @gen.coroutine
     def github_api_request(self, api_url, etag=None):
@@ -621,6 +687,11 @@ def get_resolved_ref(self):
         )
         return self.resolved_ref
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f"{self.user}/{self.repo}/{self.resolved_ref}"
+
     def get_build_slug(self):
         return '{user}-{repo}'.format(user=self.user, repo=self.repo)
 
@@ -639,6 +710,7 @@ class GistRepoProvider(GitHubRepoProvider):
     """
 
     name = Unicode('Gist')
+    hostname = Unicode('gist.github.com')
 
     allow_secret_gist = Bool(
         default_value=False,
@@ -657,7 +729,12 @@ def __init__(self, *args, **kwargs):
             self.unresolved_ref = ''
 
     def get_repo_url(self):
-        return f'https://gist.github.com/{self.user}/{self.gist_id}.git'
+        return f'https://{self.hostname}/{self.user}/{self.gist_id}.git'
+
+    async def get_resolved_ref_url(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f'https://{self.hostname}/{self.user}/{self.gist_id}/{self.resolved_ref}'
 
     @gen.coroutine
     def get_resolved_ref(self):
@@ -689,5 +766,10 @@ def get_resolved_ref(self):
 
         return self.resolved_ref
 
+    async def get_resolved_spec(self):
+        if not hasattr(self, 'resolved_ref'):
+            self.resolved_ref = await self.get_resolved_ref()
+        return f'{self.user}/{self.gist_id}/{self.resolved_ref}'
+
     def get_build_slug(self):
         return self.gist_id

binderhub/tests/test_repoproviders.py

@@ -37,45 +37,82 @@ def test_spec_processing(spec, raw_user, raw_repo, raw_ref):
     assert raw_ref == unresolved_ref
 
 
-async def test_zenodo():
-    spec = '10.5281/zenodo.3242074'
-
+@pytest.mark.parametrize('spec,resolved_spec,resolved_ref,resolved_ref_url,build_slug', [
+    ['10.5281/zenodo.3242074',
+     '10.5281/zenodo.3242074',
+     '3242074',
+     'https://doi.org/10.5281/zenodo.3242074',
+     'zenodo-3242074'],
+    # 10.5281/zenodo.3242073 -> This DOI represents all versions, and will always resolve to the latest one
+    # for now it is 3242074
+    ['10.5281/zenodo.3242073',
+     '10.5281/zenodo.3242074',
+     '3242074',
+     'https://doi.org/10.5281/zenodo.3242074',
+     'zenodo-3242074'],
+])
+async def test_zenodo(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug):
     provider = ZenodoProvider(spec=spec)
 
     # have to resolve the ref first
     ref = await provider.get_resolved_ref()
-    assert ref == '3242074'
+    assert ref == resolved_ref
 
     slug = provider.get_build_slug()
-    assert slug == 'zenodo-3242074'
+    assert slug == build_slug
     repo_url = provider.get_repo_url()
     assert repo_url == spec
-
-
-async def test_figshare():
-    spec = '10.6084/m9.figshare.9782777.v1'
-
+    ref_url = await provider.get_resolved_ref_url()
+    assert ref_url == resolved_ref_url
+    spec = await provider.get_resolved_spec()
+    assert spec == resolved_spec
+
+
+@pytest.mark.parametrize('spec,resolved_spec,resolved_ref,resolved_ref_url,build_slug', [
+    ['10.6084/m9.figshare.9782777.v1',
+     '10.6084/m9.figshare.9782777.v1',
+     '9782777.v1',
+     'https://doi.org/10.6084/m9.figshare.9782777.v1',
+     'figshare-9782777.v1'],
+    # spec without version is accepted as version 1 - check FigshareProvider.get_resolved_ref()
+    ['10.6084/m9.figshare.9782777',
+     '10.6084/m9.figshare.9782777.v1',
+     '9782777.v1',
+     'https://doi.org/10.6084/m9.figshare.9782777.v1',
+     'figshare-9782777.v1'],
+])
+async def test_figshare(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug):
     provider = FigshareProvider(spec=spec)
 
     # have to resolve the ref first
     ref = await provider.get_resolved_ref()
-    assert ref == '9782777.v1'
+    assert ref == resolved_ref
 
     slug = provider.get_build_slug()
-    assert slug == 'figshare-9782777.v1'
+    assert slug == build_slug
     repo_url = provider.get_repo_url()
     assert repo_url == spec
+    ref_url = await provider.get_resolved_ref_url()
+    assert ref_url == resolved_ref_url
+    spec = await provider.get_resolved_spec()
+    assert spec == resolved_spec
 
 
 @pytest.mark.github_api
 def test_github_ref():
-    provider = GitHubRepoProvider(spec='jupyterhub/zero-to-jupyterhub-k8s/v0.4')
+    namespace = 'jupyterhub/zero-to-jupyterhub-k8s'
+    spec = f'{namespace}/v0.4'
+    provider = GitHubRepoProvider(spec=spec)
     slug = provider.get_build_slug()
     assert slug == 'jupyterhub-zero-to-jupyterhub-k8s'
     full_url = provider.get_repo_url()
-    assert full_url == 'https://github.com/jupyterhub/zero-to-jupyterhub-k8s'
+    assert full_url == f'https://github.com/{namespace}'
     ref = IOLoop().run_sync(provider.get_resolved_ref)
     assert ref == 'f7f3ff6d1bf708bdc12e5f10e18b2a90a4795603'
+    ref_url = IOLoop().run_sync(provider.get_resolved_ref_url)
+    assert ref_url == f'https://github.com/{namespace}/tree/{ref}'
+    resolved_spec = IOLoop().run_sync(provider.get_resolved_spec)
+    assert resolved_spec == f'{namespace}/{ref}'
 
 
 def test_not_banned():
@@ -249,20 +286,29 @@ def test_git_ref(url, unresolved_ref, resolved_ref):
     assert full_url == url
     ref = IOLoop().run_sync(provider.get_resolved_ref)
     assert ref == resolved_ref
+    ref_url = IOLoop().run_sync(provider.get_resolved_ref_url)
+    assert ref_url == full_url
+    resolved_spec = IOLoop().run_sync(provider.get_resolved_spec)
+    assert resolved_spec == quote(url, safe='') + f'/{resolved_ref}'
 
 
 def test_gitlab_ref():
+    namespace = 'gitlab-org/gitlab-foss'
     spec = '{}/{}'.format(
-        quote('gitlab-org/gitlab-foss', safe=''),
+        quote(namespace, safe=''),
         quote('v10.0.6')
     )
     provider = GitLabRepoProvider(spec=spec)
     slug = provider.get_build_slug()
     assert slug == 'gitlab_-org-gitlab_-foss'
     full_url = provider.get_repo_url()
-    assert full_url == 'https://gitlab.com/gitlab-org/gitlab-foss.git'
+    assert full_url == f'https://gitlab.com/{namespace}.git'
     ref = IOLoop().run_sync(provider.get_resolved_ref)
     assert ref == 'b3344b7f17c335a817c5d7608c5e47fd7cabc023'
+    ref_url = IOLoop().run_sync(provider.get_resolved_ref_url)
+    assert ref_url == f'https://gitlab.com/{namespace}/tree/{ref}'
+    resolved_spec = IOLoop().run_sync(provider.get_resolved_spec)
+    assert resolved_spec == quote(namespace, safe='') + f'/{ref}'
 
 
 @pytest.mark.github_api
@@ -273,9 +319,13 @@ def test_gist_ref():
     slug = provider.get_build_slug()
     assert slug == '8a658f7f63b13768d1e75fa2464f5092'
     full_url = provider.get_repo_url()
-    assert full_url == 'https://gist.github.com/mariusvniekerk/8a658f7f63b13768d1e75fa2464f5092.git'
+    assert full_url == f'https://gist.github.com/{spec}.git'
     ref = IOLoop().run_sync(provider.get_resolved_ref)
     assert ref == '7daa381aae8409bfe28193e2ed8f767c26371237'
+    ref_url = IOLoop().run_sync(provider.get_resolved_ref_url)
+    assert ref_url == f'https://gist.github.com/{spec}/{ref}'
+    resolved_spec = IOLoop().run_sync(provider.get_resolved_spec)
+    assert resolved_spec == f'{spec}/{ref}'
 
 
 @pytest.mark.github_api