Merge pull request #1225 from minrk/kube-api-timeout

set _request_timeout on most kubernetes API requests

Author: consideRatio

binderhub/build.py

@@ -12,7 +12,7 @@
 from tornado.ioloop import IOLoop
 from tornado.log import app_log
 
-from .utils import rendezvous_rank
+from .utils import rendezvous_rank, KUBE_REQUEST_TIMEOUT
 
 
 class Build:
@@ -186,6 +186,7 @@ def get_affinity(self):
         dind_pods = self.api.list_namespaced_pod(
             self.namespace,
             label_selector="component=dind,app=binder",
+            _request_timeout=KUBE_REQUEST_TIMEOUT,
         )
 
         if self.sticky_builds and dind_pods:
@@ -304,7 +305,11 @@ def submit(self):
         )
 
         try:
-            ret = self.api.create_namespaced_pod(self.namespace, self.pod)
+            ret = self.api.create_namespaced_pod(
+                self.namespace,
+                self.pod,
+                _request_timeout=KUBE_REQUEST_TIMEOUT,
+            )
         except client.rest.ApiException as e:
             if e.status == 409:
                 # Someone else created it!
@@ -320,10 +325,11 @@ def submit(self):
             w = watch.Watch()
             try:
                 for f in w.stream(
-                        self.api.list_namespaced_pod,
-                        self.namespace,
-                        label_selector="name={}".format(self.name),
-                        timeout_seconds=30,
+                    self.api.list_namespaced_pod,
+                    self.namespace,
+                    label_selector="name={}".format(self.name),
+                    timeout_seconds=30,
+                    _request_timeout=KUBE_REQUEST_TIMEOUT,
                 ):
                     if f['type'] == 'DELETED':
                         self.progress('pod.phasechange', 'Deleted')
@@ -348,11 +354,13 @@ def stream_logs(self):
         """Stream a pod's logs"""
         app_log.info("Watching logs of %s", self.name)
         for line in self.api.read_namespaced_pod_log(
-                self.name,
-                self.namespace,
-                follow=True,
-                tail_lines=self.log_tail_lines,
-                _preload_content=False):
+            self.name,
+            self.namespace,
+            follow=True,
+            tail_lines=self.log_tail_lines,
+            _request_timeout=(3, None),
+            _preload_content=False,
+        ):
             if self.stop_event.is_set():
                 app_log.info("Stopping logs of %s", self.name)
                 return
@@ -382,7 +390,9 @@ def cleanup(self):
             self.api.delete_namespaced_pod(
                 name=self.name,
                 namespace=self.namespace,
-                body=client.V1DeleteOptions(grace_period_seconds=0))
+                body=client.V1DeleteOptions(grace_period_seconds=0),
+                _request_timeout=KUBE_REQUEST_TIMEOUT,
+            )
         except client.rest.ApiException as e:
             if e.status == 404:
                 # Is ok, someone else has already deleted it

binderhub/builder.py

@@ -22,6 +22,7 @@
 
 from .base import BaseHandler
 from .build import Build, FakeBuild
+from .utils import KUBE_REQUEST_TIMEOUT
 
 # Separate buckets for builds and launches.
 # Builds and launches have very different characteristic times,
@@ -475,9 +476,11 @@ async def launch(self, kube, provider):
 
         # TODO: run a watch to keep this up to date in the background
         pool = self.settings['executor']
-        f = pool.submit(kube.list_namespaced_pod,
+        f = pool.submit(
+            kube.list_namespaced_pod,
             self.settings["build_namespace"],
             label_selector='app=jupyterhub,component=singleuser-server',
+            _request_timeout=KUBE_REQUEST_TIMEOUT,
         )
         # concurrent.futures.Future isn't awaitable
         # wrap in tornado Future

binderhub/health.py

@@ -8,6 +8,7 @@
 from tornado.log import app_log
 
 from .base import BaseHandler
+from .utils import KUBE_REQUEST_TIMEOUT
 
 
 def retry(_f=None, *, delay=1, attempts=3):
@@ -120,6 +121,7 @@ async def _get_pods(self):
                     namespace,
                     label_selector=label_selector,
                     _preload_content=False,
+                    _request_timeout=KUBE_REQUEST_TIMEOUT,
                 )
             )
             for label_selector in label_selectors

binderhub/utils.py

@@ -6,6 +6,14 @@
 from traitlets import Integer, TraitError
 
 
+# default _request_timeout for kubernetes api requests
+# tuple of two timeouts: (connect_timeout, read_timeout)
+# the most important of these is the connect_timeout,
+# which can hang for a *very* long time when there are internal
+# kubernetes connection issues
+KUBE_REQUEST_TIMEOUT = (3, 30)
+
+
 def blake2b_hash_as_int(b):
     """Compute digest of the bytes `b` using the Blake2 hash function.