Add support for forced session termination (#89)

Author: achimnol

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changes
 =======
 
+19.09.6 (2020-03-xx)
+--------------------
+
+* NEW: Add ``-f`` / ``--forced`` option to ``backend.ai terminate`` command
+  and add an optional ``forced`` keyword-argument to SDK API (``Kernel.destroy()``). (#89)
+
 19.09.5 (2020-03-08)
 --------------------
 

src/ai/backend/client/cli/run.py

@@ -847,11 +847,13 @@ def start(image, session_id, owner,                                 # base args
 
 @main.command(aliases=['rm', 'kill'])
 @click.argument('session_ids', metavar='SESSID', nargs=-1)
+@click.option('-f', '--forced', is_flag=True,
+              help='Force-terminate the errored sessions (only allowed for admins)')
 @click.option('-o', '--owner', '--owner-access-key', metavar='ACCESS_KEY',
               help='Specify the owner of the target session explicitly.')
 @click.option('-s', '--stats', is_flag=True,
               help='Show resource usage statistics after termination')
-def terminate(session_ids, owner, stats):
+def terminate(session_ids, forced, owner, stats):
     '''
     Terminate the given session.
 
@@ -863,7 +865,7 @@ def terminate(session_ids, owner, stats):
         for session_id in session_ids:
             try:
                 kernel = session.Kernel(session_id, owner)
-                ret = kernel.destroy()
+                ret = kernel.destroy(forced=forced)
             except BackendAPIError as e:
                 print_error(e)
                 if e.status == 404:

src/ai/backend/client/kernel.py

@@ -195,7 +195,7 @@ def __init__(self, kernel_id: str, owner_access_key: str = None):
         self.owner_access_key = owner_access_key
 
     @api_function
-    async def destroy(self):
+    async def destroy(self, *, forced: bool = False):
         '''
         Destroys the compute session.
         Since the server literally kills the container(s), all ongoing executions are
@@ -204,6 +204,8 @@ async def destroy(self):
         params = {}
         if self.owner_access_key:
             params['owner_access_key'] = self.owner_access_key
+        if forced:
+            params['forced'] = 'true'
         rqst = Request(self.session,
                        'DELETE', '/kernel/{}'.format(self.kernel_id),
                        params=params)