|
| 1 | +"""A MultiTerminalManager for use in the notebook webserver |
| 2 | +- raises HTTPErrors |
| 3 | +- creates REST API models |
| 4 | +""" |
| 5 | + |
| 6 | +# Copyright (c) Jupyter Development Team. |
| 7 | +# Distributed under the terms of the Modified BSD License. |
| 8 | + |
| 9 | +import warnings |
| 10 | + |
| 11 | +from datetime import timedelta |
| 12 | +from notebook._tz import utcnow, isoformat |
| 13 | +from terminado import NamedTermManager |
| 14 | +from tornado import web |
| 15 | +from tornado.ioloop import IOLoop, PeriodicCallback |
| 16 | +from traitlets import Integer, validate |
| 17 | +from traitlets.config import LoggingConfigurable |
| 18 | +from ..prometheus.metrics import TERMINAL_CURRENTLY_RUNNING_TOTAL |
| 19 | + |
| 20 | + |
| 21 | +class TerminalManager(LoggingConfigurable, NamedTermManager): |
| 22 | + """ """ |
| 23 | + |
| 24 | + _culler_callback = None |
| 25 | + |
| 26 | + _initialized_culler = False |
| 27 | + |
| 28 | + cull_inactive_timeout = Integer(0, config=True, |
| 29 | + help="""Timeout (in seconds) in which a terminal has been inactive and ready to be culled. |
| 30 | + Values of 0 or lower disable culling.""" |
| 31 | + ) |
| 32 | + |
| 33 | + cull_interval_default = 300 # 5 minutes |
| 34 | + cull_interval = Integer(cull_interval_default, config=True, |
| 35 | + help="""The interval (in seconds) on which to check for terminals exceeding the inactive timeout value.""" |
| 36 | + ) |
| 37 | + |
| 38 | + # ------------------------------------------------------------------------- |
| 39 | + # Methods for managing terminals |
| 40 | + # ------------------------------------------------------------------------- |
| 41 | + def __init__(self, *args, **kwargs): |
| 42 | + super(TerminalManager, self).__init__(*args, **kwargs) |
| 43 | + |
| 44 | + def create(self): |
| 45 | + """Create a new terminal.""" |
| 46 | + name, term = self.new_named_terminal() |
| 47 | + # Monkey-patch last-activity, similar to kernels. Should we need |
| 48 | + # more functionality per terminal, we can look into possible sub- |
| 49 | + # classing or containment then. |
| 50 | + term.last_activity = utcnow() |
| 51 | + model = self.get_terminal_model(name) |
| 52 | + # Increase the metric by one because a new terminal was created |
| 53 | + TERMINAL_CURRENTLY_RUNNING_TOTAL.inc() |
| 54 | + # Ensure culler is initialized |
| 55 | + self._initialize_culler() |
| 56 | + return model |
| 57 | + |
| 58 | + def get(self, name): |
| 59 | + """Get terminal 'name'.""" |
| 60 | + model = self.get_terminal_model(name) |
| 61 | + return model |
| 62 | + |
| 63 | + def list(self): |
| 64 | + """Get a list of all running terminals.""" |
| 65 | + models = [self.get_terminal_model(name) for name in self.terminals] |
| 66 | + |
| 67 | + # Update the metric below to the length of the list 'terms' |
| 68 | + TERMINAL_CURRENTLY_RUNNING_TOTAL.set( |
| 69 | + len(models) |
| 70 | + ) |
| 71 | + return models |
| 72 | + |
| 73 | + async def terminate(self, name, force=False): |
| 74 | + """Terminate terminal 'name'.""" |
| 75 | + self._check_terminal(name) |
| 76 | + await super(TerminalManager, self).terminate(name, force=force) |
| 77 | + |
| 78 | + # Decrease the metric below by one |
| 79 | + # because a terminal has been shutdown |
| 80 | + TERMINAL_CURRENTLY_RUNNING_TOTAL.dec() |
| 81 | + |
| 82 | + async def terminate_all(self): |
| 83 | + """Terminate all terminals.""" |
| 84 | + terms = [name for name in self.terminals] |
| 85 | + for term in terms: |
| 86 | + await self.terminate(term, force=True) |
| 87 | + |
| 88 | + def get_terminal_model(self, name): |
| 89 | + """Return a JSON-safe dict representing a terminal. |
| 90 | + For use in representing terminals in the JSON APIs. |
| 91 | + """ |
| 92 | + self._check_terminal(name) |
| 93 | + term = self.terminals[name] |
| 94 | + model = { |
| 95 | + "name": name, |
| 96 | + "last_activity": isoformat(term.last_activity), |
| 97 | + } |
| 98 | + return model |
| 99 | + |
| 100 | + def _check_terminal(self, name): |
| 101 | + """Check a that terminal 'name' exists and raise 404 if not.""" |
| 102 | + if name not in self.terminals: |
| 103 | + raise web.HTTPError(404, u'Terminal not found: %s' % name) |
| 104 | + |
| 105 | + def _initialize_culler(self): |
| 106 | + """Start culler if 'cull_inactive_timeout' is greater than zero. |
| 107 | + Regardless of that value, set flag that we've been here. |
| 108 | + """ |
| 109 | + if not self._initialized_culler and self.cull_inactive_timeout > 0: |
| 110 | + if self._culler_callback is None: |
| 111 | + loop = IOLoop.current() |
| 112 | + if self.cull_interval <= 0: # handle case where user set invalid value |
| 113 | + self.log.warning("Invalid value for 'cull_interval' detected (%s) - using default value (%s).", |
| 114 | + self.cull_interval, self.cull_interval_default) |
| 115 | + self.cull_interval = self.cull_interval_default |
| 116 | + self._culler_callback = PeriodicCallback( |
| 117 | + self._cull_terminals, 1000 * self.cull_interval) |
| 118 | + self.log.info("Culling terminals with inactivity > %s seconds at %s second intervals ...", |
| 119 | + self.cull_inactive_timeout, self.cull_interval) |
| 120 | + self._culler_callback.start() |
| 121 | + |
| 122 | + self._initialized_culler = True |
| 123 | + |
| 124 | + async def _cull_terminals(self): |
| 125 | + self.log.debug("Polling every %s seconds for terminals inactive for > %s seconds...", |
| 126 | + self.cull_interval, self.cull_inactive_timeout) |
| 127 | + # Create a separate list of terminals to avoid conflicting updates while iterating |
| 128 | + for name in list(self.terminals): |
| 129 | + try: |
| 130 | + await self._cull_inactive_terminal(name) |
| 131 | + except Exception as e: |
| 132 | + self.log.exception("The following exception was encountered while checking the " |
| 133 | + "activity of terminal {}: {}".format(name, e)) |
| 134 | + |
| 135 | + async def _cull_inactive_terminal(self, name): |
| 136 | + try: |
| 137 | + term = self.terminals[name] |
| 138 | + except KeyError: |
| 139 | + return # KeyErrors are somewhat expected since the terminal can be terminated as the culling check is made. |
| 140 | + |
| 141 | + self.log.debug("name=%s, last_activity=%s", name, term.last_activity) |
| 142 | + if hasattr(term, 'last_activity'): |
| 143 | + dt_now = utcnow() |
| 144 | + dt_inactive = dt_now - term.last_activity |
| 145 | + # Compute idle properties |
| 146 | + is_time = dt_inactive > timedelta(seconds=self.cull_inactive_timeout) |
| 147 | + # Cull the kernel if all three criteria are met |
| 148 | + if (is_time): |
| 149 | + inactivity = int(dt_inactive.total_seconds()) |
| 150 | + self.log.warning("Culling terminal '%s' due to %s seconds of inactivity.", name, inactivity) |
| 151 | + await self.terminate(name, force=True) |
0 commit comments