Merge pull request #183 from radam9/master

fix UnicodeDecodeError when downloading non-json content

Author: jph00

00_core.ipynb

@@ -88,6 +88,16 @@
     "    params += [_mk_param(k, **_mk_sig_detls(v)) for k,v in anno_args.items()]\n",
     "    return Signature(params)\n",
     "\n",
+    "def _decode_response(path: str) -> bool:\n",
+    "    \"checks if a endpoint needs to have it's response from `fastcore.core.urlsend` decoded or just return json\"\n",
+    "    needs_decode = (\n",
+    "        \"/orgs/{org}/migrations/{migration_id}/archive\",\n",
+    "        \"/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}\",\n",
+    "        \"/repos/{owner}/{repo}/tarball/{ref}\",\n",
+    "        \"/repos/{owner}/{repo}/zipball/{ref}\",\n",
+    "    )\n",
+    "    return path not in needs_decode\n",
+    "\n",
     "class _GhObj: pass"
    ]
   },
@@ -99,11 +109,12 @@
    "source": [
     "#|export\n",
     "class _GhVerb(_GhObj):\n",
-    "    __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,__doc__'.split(',')\n",
+    "    __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,decode,__doc__'.split(',')\n",
     "    def __init__(self, path, verb, oper, summary, url, params, data, preview, client, kwargs):\n",
     "        tag,*name = oper.split('/')\n",
     "        name = '__'.join(name)\n",
     "        name = name.replace('-','_')\n",
+    "        decode = _decode_response(path)\n",
     "        path,_,_ = partial_format(path, **kwargs)\n",
     "        route_ps = stringfmt_names(path)\n",
     "        __doc__ = summary\n",
@@ -117,7 +128,7 @@
     "        for a,b in zip(args,flds): kwargs[b]=a\n",
     "        route_p,query_p,data_p = [{p:kwargs[p] for p in o if p in kwargs}\n",
     "                                 for o in (self.route_ps,self.params,d)]\n",
-    "        return self.client(self.path, self.verb, headers=headers, route=route_p, query=query_p, data=data_p)\n",
+    "        return self.client(self.path, self.verb, headers=headers, decode=self.decode, route=route_p, query=query_p, data=data_p)\n",
     "\n",
     "    def __str__(self): return f'{self.tag}.{self.name}{signature(self)}\\n{self.doc_url}'\n",
     "    @property\n",
@@ -201,17 +212,17 @@
     "        self.debug,self.limit_cb,self.limit_rem = debug,limit_cb,5000\n",
     "        self.gh_host = gh_host or GH_HOST\n",
     "\n",
-    "    def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None):\n",
+    "    def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None, decode=True):\n",
     "        \"Call a fully specified `path` using HTTP `verb`, passing arguments to `fastcore.core.urlsend`\"\n",
     "        if verb is None: verb = 'POST' if data else 'GET'\n",
     "        headers = {**self.headers,**(headers or {})}\n",
     "        if not path.startswith(('http://', 'https://')):\n",
     "            path = self.gh_host + path\n",
     "        if route:\n",
     "            for k,v in route.items(): route[k] = quote(str(route[k]))\n",
-    "        return_json = ('json' in headers['Accept'])\n",
+    "        return_json = ('json' in headers['Accept']) and (decode is True)\n",
     "        debug = self.debug if self.debug else print_summary if os.getenv('GHAPI_DEBUG') else None\n",
-    "        res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, debug=debug, return_headers=True,\n",
+    "        res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, decode=decode, debug=debug, return_headers=True,\n",
     "                                     route=route or None, query=query or None, data=data or None, return_json=return_json, timeout=timeout)\n",
     "        if 'X-RateLimit-Remaining' in self.recv_hdrs:\n",
     "            newlim = self.recv_hdrs['X-RateLimit-Remaining']\n",

ghapi/core.py

@@ -36,15 +36,26 @@ def _mk_sig(req_args, opt_args, anno_args):
     params += [_mk_param(k, **_mk_sig_detls(v)) for k,v in anno_args.items()]
     return Signature(params)
 
+def _decode_response(path: str) -> bool:
+    "checks if a endpoint needs to have it's response from `fastcore.core.urlsend` decoded or just return json"
+    needs_decode = (
+        "/orgs/{org}/migrations/{migration_id}/archive",
+        "/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}",
+        "/repos/{owner}/{repo}/tarball/{ref}",
+        "/repos/{owner}/{repo}/zipball/{ref}",
+    )
+    return path not in needs_decode
+
 class _GhObj: pass
 
 # %% ../00_core.ipynb 7
 class _GhVerb(_GhObj):
-    __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,__doc__'.split(',')
+    __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,decode,__doc__'.split(',')
     def __init__(self, path, verb, oper, summary, url, params, data, preview, client, kwargs):
         tag,*name = oper.split('/')
         name = '__'.join(name)
         name = name.replace('-','_')
+        decode = _decode_response(path)
         path,_,_ = partial_format(path, **kwargs)
         route_ps = stringfmt_names(path)
         __doc__ = summary
@@ -58,7 +69,7 @@ def __call__(self, *args, headers=None, **kwargs):
         for a,b in zip(args,flds): kwargs[b]=a
         route_p,query_p,data_p = [{p:kwargs[p] for p in o if p in kwargs}
                                  for o in (self.route_ps,self.params,d)]
-        return self.client(self.path, self.verb, headers=headers, route=route_p, query=query_p, data=data_p)
+        return self.client(self.path, self.verb, headers=headers, decode=self.decode, route=route_p, query=query_p, data=data_p)
 
     def __str__(self): return f'{self.tag}.{self.name}{signature(self)}\n{self.doc_url}'
     @property
@@ -107,17 +118,17 @@ def __init__(self, owner=None, repo=None, token=None, jwt_token=None, debug=None
         self.debug,self.limit_cb,self.limit_rem = debug,limit_cb,5000
         self.gh_host = gh_host or GH_HOST
 
-    def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None):
+    def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None, decode=True):
         "Call a fully specified `path` using HTTP `verb`, passing arguments to `fastcore.core.urlsend`"
         if verb is None: verb = 'POST' if data else 'GET'
         headers = {**self.headers,**(headers or {})}
         if not path.startswith(('http://', 'https://')):
             path = self.gh_host + path
         if route:
             for k,v in route.items(): route[k] = quote(str(route[k]))
-        return_json = ('json' in headers['Accept'])
+        return_json = ('json' in headers['Accept']) and (decode is True)
         debug = self.debug if self.debug else print_summary if os.getenv('GHAPI_DEBUG') else None
-        res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, debug=debug, return_headers=True,
+        res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, decode=decode, debug=debug, return_headers=True,
                                      route=route or None, query=query or None, data=data or None, return_json=return_json, timeout=timeout)
         if 'X-RateLimit-Remaining' in self.recv_hdrs:
             newlim = self.recv_hdrs['X-RateLimit-Remaining']

settings.ini

@@ -16,7 +16,7 @@ language = English
 custom_sidebar = False
 license = apache2
 status = 5
-requirements = fastcore>=1.7.0
+requirements = fastcore>=1.7.2
 dev_requirements = jsonref matplotlib
 console_scripts = ghapi=ghapi.cli:ghapi
 	ghpath=ghapi.cli:ghpath