Add a Software Heritage provider

Software Heritage¹ is a non-profit multi-stakeholder initiative which collects,
preserves and shares all software that is publicly available in source code
form.

It provides a public API allowing to retrieve any piece of source code
that has been ingested and is identified via its SWHID², including a
growing number of jupyter notbooks.

Thanks to the recent support for SWHID added in repo2docker, this now
bring support to SWHID to binderhub as a Provider.

¹ https://www.softwareheritage.org
² https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html#persistent-identifiers

Author: douardda

binderhub/app.py

@@ -49,7 +49,7 @@
 from .repoproviders import (GitHubRepoProvider, GitRepoProvider,
                             GitLabRepoProvider, GistRepoProvider,
                             ZenodoProvider, FigshareProvider, HydroshareProvider,
-                            DataverseProvider)
+                            DataverseProvider, SWHIDProvider)
 from .metrics import MetricsHandler
 
 from .utils import ByteSpecification, url_path_join
@@ -465,6 +465,7 @@ def _add_slash(self, proposal):
             'figshare': FigshareProvider,
             'hydroshare': HydroshareProvider,
             'dataverse': DataverseProvider,
+            'swh': SWHIDProvider,
         },
         config=True,
         help="""

binderhub/event-schemas/launch.json

@@ -14,7 +14,8 @@
                 "Zenodo",
                 "Figshare",
                 "Hydroshare",
-                "Dataverse"
+                "Dataverse",
+                "SWHID"
             ],
             "description": "Provider for the repository being launched"
         },

binderhub/main.py

@@ -19,6 +19,7 @@
     "figshare": "Figshare",
     "hydroshare": "Hydroshare",
     "dataverse": "Dataverse",
+    "swh": "Software Heritage",
 }
 
 

binderhub/repoproviders.py

@@ -961,3 +961,41 @@ async def get_resolved_spec(self):
 
     def get_build_slug(self):
         return self.gist_id
+
+
+class SWHIDProvider(RepoProvider):
+    """Provide contents of a SWHID record
+
+    Users must provide a spec consisting of the SWHID of a directory or revision.
+    """
+    name = Unicode("SWHID")
+
+    async def get_resolved_ref(self):
+        client = AsyncHTTPClient()
+        req = HTTPRequest("https://archive.softwareheritage.org/api/1/known/",
+                          method="POST",
+                          headers={'content-type': 'application/json'},
+                          body=json.dumps([self.spec]),
+                          user_agent="BinderHub")
+        r = await client.fetch(req)
+        r.rethrow()
+
+        response = json.loads(r.body)
+        if response[self.spec]["known"]:
+            return self.spec
+        raise RuntimeError(f"Unknown SWHID {self.spec}")
+
+
+    async def get_resolved_spec(self):
+        return self.spec
+
+    def get_repo_url(self):
+        # While called repo URL, the return value of this function is passed
+        # as argument to repo2docker, hence we return the spec as is.
+        return self.spec
+
+    async def get_resolved_ref_url(self):
+        return self.spec
+
+    def get_build_slug(self):
+        return "swh-{}".format(self.spec)

binderhub/static/js/index.js

@@ -113,7 +113,7 @@ function getBuildFormValues() {
 
   var ref = $('#ref').val().trim() || $("#ref").attr("placeholder");
   if (providerPrefix === 'zenodo' || providerPrefix === 'figshare' || providerPrefix === 'dataverse' ||
-      providerPrefix === 'hydroshare') {
+      providerPrefix === 'hydroshare' || providerPrefix === 'swh') {
     ref = "";
   }
   var path = $('#filepath').val().trim();

binderhub/tests/test_repoproviders.py

@@ -12,6 +12,7 @@
     GitLabRepoProvider,
     GitRepoProvider,
     HydroshareProvider,
+    SWHIDProvider,
     ZenodoProvider,
     strip_suffix,
     tokenize_spec,
@@ -490,3 +491,27 @@ def test_gist_secret():
 
     provider = GistRepoProvider(spec=spec, allow_secret_gist=True)
     assert IOLoop().run_sync(provider.get_resolved_ref) is not None
+
+
+@pytest.mark.parametrize('spec,resolved_spec,resolved_ref,resolved_ref_url,build_slug', [
+    ['swh:1:rev:c30614ec4587418fb264efb466cba58991029f16',
+     'swh:1:rev:c30614ec4587418fb264efb466cba58991029f16',
+     'swh:1:rev:c30614ec4587418fb264efb466cba58991029f16',
+     'swh:1:rev:c30614ec4587418fb264efb466cba58991029f16',
+     'swh-swh:1:rev:c30614ec4587418fb264efb466cba58991029f16'],
+])
+async def test_swh(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug):
+    provider = SWHIDProvider(spec=spec)
+
+    # have to resolve the ref first
+    ref = await provider.get_resolved_ref()
+    assert ref == resolved_ref
+
+    slug = provider.get_build_slug()
+    assert slug == build_slug
+    repo_url = provider.get_repo_url()
+    assert repo_url == spec
+    ref_url = await provider.get_resolved_ref_url()
+    assert ref_url == resolved_ref_url
+    spec = await provider.get_resolved_spec()
+    assert spec == resolved_spec

doc/developer/repoproviders.rst

@@ -38,6 +38,12 @@ Currently supported providers, their prefixes and specs are:
     +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
     | Git        | ``git``            | ``<url-escaped-url>/<commit-sha>``                          | A generic repository provider for URLs that point directly to a git repository.                                                           |
     +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
+    | SWHID      | ``swh``            | ``<SWHID>``                                                 | A `Software Heritage persistent identifier`_.                                                                                                |
+    +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
+
+
+.. _`Software Heritage persistent identifier`: https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html#persistent-identifiers
+
 
 Adding a new repository provider
 ================================

doc/reference/repoproviders.rst

@@ -71,3 +71,10 @@ Module: :mod:`binderhub.repoproviders`
 
 .. autoconfigurable:: GitRepoProvider
     :members:
+
+
+:class:`SWHIDProvider`
+---------------------------
+
+.. autoconfigurable:: SWHIDProvider
+    :members: