Merge pull request #1205 from consideRatio/pr/coroutine-to-async

[MRG] Replace gen.coroutine/yeild with modern async/await

Author: betatim

binderhub/registry.py

@@ -6,7 +6,7 @@
 import os
 from urllib.parse import urlparse
 
-from tornado import gen, httpclient
+from tornado import httpclient
 from tornado.httputil import url_concat
 from traitlets.config import LoggingConfigurable
 from traitlets import Dict, Unicode, default
@@ -185,8 +185,7 @@ def _default_password(self):
             base64.b64decode(b64_auth.encode("utf-8")).decode("utf-8").split(":", 1)[1]
         )
 
-    @gen.coroutine
-    def get_image_manifest(self, image, tag):
+    async def get_image_manifest(self, image, tag):
         client = httpclient.AsyncHTTPClient()
         url = "{}/v2/{}/manifests/{}".format(self.url, image, tag)
         # first, get a token to perform the manifest request
@@ -196,7 +195,7 @@ def get_image_manifest(self, image, tag):
                 auth_username=self.username,
                 auth_password=self.password,
             )
-            auth_resp = yield client.fetch(auth_req)
+            auth_resp = await client.fetch(auth_req)
             response_body = json.loads(auth_resp.body.decode("utf-8", "replace"))
 
             if "token" in response_body.keys():
@@ -215,7 +214,7 @@ def get_image_manifest(self, image, tag):
             )
 
         try:
-            resp = yield client.fetch(req)
+            resp = await client.fetch(req)
         except httpclient.HTTPError as e:
             if e.code == 404:
                 # 404 means it doesn't exist

binderhub/repoproviders.py

@@ -18,7 +18,6 @@
 import escapism
 from prometheus_client import Gauge
 
-from tornado import gen
 from tornado.httpclient import AsyncHTTPClient, HTTPError, HTTPRequest
 from tornado.httputil import url_concat
 
@@ -161,21 +160,18 @@ def repo_config(self, settings):
                 repo_config.update(config)
         return repo_config
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         raise NotImplementedError("Must be overridden in child class")
 
-    @gen.coroutine
-    def get_resolved_spec(self):
+    async def get_resolved_spec(self):
         """Return the spec with resolved ref."""
         raise NotImplementedError("Must be overridden in child class")
 
     def get_repo_url(self):
         """Return the git clone-able repo URL"""
         raise NotImplementedError("Must be overridden in the child class")
 
-    @gen.coroutine
-    def get_resolved_ref_url(self):
+    async def get_resolved_ref_url(self):
         """Return the URL of repository at this commit in history"""
         raise NotImplementedError("Must be overridden in child class")
 
@@ -216,12 +212,11 @@ class ZenodoProvider(RepoProvider):
     """
     name = Unicode("Zenodo")
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         client = AsyncHTTPClient()
         req = HTTPRequest("https://doi.org/{}".format(self.spec),
                           user_agent="BinderHub")
-        r = yield client.fetch(req)
+        r = await client.fetch(req)
         self.record_id = r.effective_url.rsplit("/", maxsplit=1)[1]
         return self.record_id
 
@@ -256,12 +251,11 @@ class FigshareProvider(RepoProvider):
     name = Unicode("Figshare")
     url_regex = re.compile(r"(.*)/articles/([^/]+)/(\d+)(/)?(\d+)?")
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         client = AsyncHTTPClient()
         req = HTTPRequest("https://doi.org/{}".format(self.spec),
                           user_agent="BinderHub")
-        r = yield client.fetch(req)
+        r = await client.fetch(req)
 
         match = self.url_regex.match(r.effective_url)
         article_id = match.groups()[2]
@@ -298,20 +292,19 @@ def get_build_slug(self):
 class DataverseProvider(RepoProvider):
     name = Unicode("Dataverse")
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         client = AsyncHTTPClient()
         req = HTTPRequest("https://doi.org/{}".format(self.spec),
                           user_agent="BinderHub")
-        r = yield client.fetch(req)
+        r = await client.fetch(req)
 
         search_url = urllib.parse.urlunparse(
             urllib.parse.urlparse(r.effective_url)._replace(
                 path="/api/datasets/:persistentId"
             )
         )
         req = HTTPRequest(search_url, user_agent="BinderHub")
-        r = yield client.fetch(req)
+        r = await client.fetch(req)
         resp = json.loads(r.body)
 
         assert resp["status"] == "OK"
@@ -365,13 +358,12 @@ def _parse_resource_id(self, spec):
         resource_id = match.groups()[0]
         return resource_id
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         client = AsyncHTTPClient()
         self.resource_id = self._parse_resource_id(self.spec)
         req = HTTPRequest("https://www.hydroshare.org/hsapi/resource/{}/scimeta/elements".format(self.resource_id),
                           user_agent="BinderHub")
-        r = yield client.fetch(req)
+        r = await client.fetch(req)
 
         def parse_date(json_body):
             json_response = json.loads(json_body)
@@ -430,8 +422,7 @@ def __init__(self, *args, **kwargs):
         if not self.unresolved_ref:
             raise ValueError("`unresolved_ref` must be specified as a query parameter for the basic git provider")
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         if hasattr(self, 'resolved_ref'):
             return self.resolved_ref
 
@@ -542,8 +533,7 @@ def __init__(self, *args, **kwargs):
         if not self.unresolved_ref:
             raise ValueError("An unresolved ref is required")
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         if hasattr(self, 'resolved_ref'):
             return self.resolved_ref
 
@@ -561,7 +551,7 @@ def get_resolved_ref(self):
             api_url = url_concat(api_url, self.auth)
 
         try:
-            resp = yield client.fetch(api_url, user_agent="BinderHub")
+            resp = await client.fetch(api_url, user_agent="BinderHub")
         except HTTPError as e:
             if e.code == 404:
                 return None
@@ -680,8 +670,7 @@ async def get_resolved_ref_url(self):
             self.resolved_ref = await self.get_resolved_ref()
         return f"https://{self.hostname}/{self.user}/{self.repo}/tree/{self.resolved_ref}"
 
-    @gen.coroutine
-    def github_api_request(self, api_url, etag=None):
+    async def github_api_request(self, api_url, etag=None):
         client = AsyncHTTPClient()
 
         request_kwargs = {}
@@ -702,7 +691,7 @@ def github_api_request(self, api_url, etag=None):
         )
 
         try:
-            resp = yield client.fetch(req)
+            resp = await client.fetch(req)
         except HTTPError as e:
             if e.code == 304:
                 resp = e.response
@@ -760,8 +749,7 @@ def github_api_request(self, api_url, etag=None):
 
         return resp
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         if hasattr(self, 'resolved_ref'):
             return self.resolved_ref
 
@@ -781,7 +769,7 @@ def get_resolved_ref(self):
                 return None
             etag = None
 
-        resp = yield self.github_api_request(api_url, etag=etag)
+        resp = await self.github_api_request(api_url, etag=etag)
         if resp is None:
             self.log.debug("Caching 404 on %s", api_url)
             self.cache_404.set(api_url, True)
@@ -862,15 +850,14 @@ async def get_resolved_ref_url(self):
             self.resolved_ref = await self.get_resolved_ref()
         return f'https://{self.hostname}/{self.user}/{self.gist_id}/{self.resolved_ref}'
 
-    @gen.coroutine
-    def get_resolved_ref(self):
+    async def get_resolved_ref(self):
         if hasattr(self, 'resolved_ref'):
             return self.resolved_ref
 
         api_url = f"https://api.github.com/gists/{self.gist_id}"
         self.log.debug("Fetching %s", api_url)
 
-        resp = yield self.github_api_request(api_url)
+        resp = await self.github_api_request(api_url)
         if resp is None:
             return None