Fix Windows compatibility and interrupted exits on all platforms (#93)

* repo: Update gitignore
* fix: Let asyncio_run_forever() rely on intrinsic interrupt mechanisms
  - To fix either hang-up or not-implemented-error for signal handlers,
    we need to use Python 3.8 for proper Windows + IOCP support.
* fix: "backend.ai --version" has been broken...
* fix(setup): Update trove classifiers and the CLI entrypoint.
* fix: Properly set the exit code for interruptions of all commands
  - We need to handle this correctly because our Windows customer
    is going to use "backend.ai" CLI inside *batch scripts*.
    See https://bugs.python.org/issue1054041 for details!

Author: achimnol

.gitignore

@@ -36,3 +36,4 @@ env_*.sh
 .ipynb_checkpoints
 
 .vscode/
+.mypy_cache/

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Changes
 =======
 
+19.09.7 (2020-03-31)
+--------------------
+
+* FIX: Not-implemented-error in ``backend.ai app`` command on Windows, due
+  to manually set event loop UNIX signal handlers. (#93)
+
+* FIX: Now *all* CLI commands set exit codes correctly for interrupts
+  (Ctrl+C on Windows or SIGINT on POSIX systems) so that batch/shell
+  scripts that use ``backend.ai`` commands get interrupted properly.
+  (#93)
+
 19.09.6 (2020-03-16)
 --------------------
 

setup.py

@@ -77,8 +77,10 @@ def read_src_version():
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Operating System :: POSIX',
         'Operating System :: MacOS :: MacOS X',
+        'Operating System :: Windows',
         'Environment :: No Input/Output (Daemon)',
         'Topic :: Scientific/Engineering',
         'Topic :: Software Development',
@@ -97,7 +99,7 @@ def read_src_version():
     data_files=[],
     entry_points={
         'console_scripts': [
-            'backend.ai = ai.backend.client.cli:main',
+            'backend.ai = ai.backend.client.cli:run_main',
             'lcc = ai.backend.client.cli:run_alias',
             'lpython = ai.backend.client.cli:run_alias',
         ],

src/ai/backend/client/cli/__init__.py

@@ -1,8 +1,12 @@
 from pathlib import Path
+import os
+import signal
 import sys
 
 import click
+from click.exceptions import ClickException, Abort
 
+from .. import __version__
 from ..config import APIConfig, set_config
 
 
@@ -85,7 +89,7 @@ def format_commands(self, ctx, formatter):
 @click.option('--skip-sslcert-validation',
               help='Skip SSL certificate validation for all API requests.',
               is_flag=True)
-@click.version_option()
+@click.version_option(version=__version__)
 def main(skip_sslcert_validation):
     """
     Backend.AI command line interface.
@@ -109,7 +113,7 @@ def run_alias():
     sys.argv.insert(1, 'run')
     if help:
         sys.argv.append('--help')
-    main.main(prog_name='backend.ai')
+    run_main()
 
 
 def _attach_command():
@@ -118,3 +122,48 @@ def _attach_command():
 
 
 _attach_command()
+
+
+def run_main():
+    try:
+        _interrupted = False
+        main.main(
+            standalone_mode=False,
+            prog_name='backend.ai',
+        )
+    except KeyboardInterrupt:
+        # For interruptions outside the Click's exception handling block.
+        print("Interrupted!", end="", file=sys.stderr)
+        sys.stderr.flush()
+        _interrupted = True
+    except Abort as e:
+        # Click wraps unhandled KeyboardInterrupt with a plain
+        # sys.exit(1) call and prints "Aborted!" message
+        # (which would look non-sense to users).
+        # This is *NOT* what we want.
+        # Instead of relying on Click, mark the _interrupted
+        # flag to perform our own exit routines.
+        if isinstance(e.__context__, KeyboardInterrupt):
+            print("Interrupted!", end="", file=sys.stderr)
+            sys.stderr.flush()
+            _interrupted = True
+        else:
+            print("Aborted!", end="", file=sys.stderr)
+            sys.stderr.flush()
+            sys.exit(1)
+    except ClickException as e:
+        e.show()
+        sys.exit(e.exit_code)
+    finally:
+        if _interrupted:
+            # Override the exit code when it's interrupted,
+            # referring https://github.com/python/cpython/pull/11862
+            if sys.platform.startswith('win'):
+                # Use STATUS_CONTROL_C_EXIT to notify cmd.exe
+                # for interrupted exit
+                sys.exit(-1073741510)
+            else:
+                # Use the default signal handler to set the exit
+                # code properly for interruption.
+                signal.signal(signal.SIGINT, signal.SIG_DFL)
+                os.kill(0, signal.SIGINT)

src/ai/backend/client/cli/__main__.py

@@ -1,3 +1,4 @@
-from . import main
+from . import run_main
 
-main()
+
+run_main()

src/ai/backend/client/compat.py

@@ -55,11 +55,11 @@ def _cancel_all_tasks(loop):
             })
 
 
-def _asyncio_run(coro, *, debug=False):
+def asyncio_run(coro, *, debug=False):
     loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.set_debug(debug)
     try:
-        asyncio.set_event_loop(loop)
-        loop.set_debug(debug)
         return loop.run_until_complete(coro)
     finally:
         try:
@@ -68,15 +68,10 @@ def _asyncio_run(coro, *, debug=False):
                 loop.run_until_complete(loop.shutdown_asyncgens())
         finally:
             loop.stop()
+            loop.close()
             asyncio.set_event_loop(None)
 
 
-if hasattr(asyncio, 'run'):  # Python 3.7+
-    asyncio_run = asyncio.run
-else:
-    asyncio_run = _asyncio_run
-
-
 def asyncio_run_forever(setup_coro, shutdown_coro, *,
                         stop_signals={signal.SIGINT}, debug=False):
     '''
@@ -87,17 +82,15 @@ def asyncio_run_forever(setup_coro, shutdown_coro, *,
     async def wait_for_stop():
         loop = current_loop()
         future = loop.create_future()
-        for stop_sig in stop_signals:
-            loop.add_signal_handler(stop_sig, future.set_result, stop_sig)
         try:
-            recv_sig = await future
-        finally:
-            loop.remove_signal_handler(recv_sig)
+            await future
+        except asyncio.CancelledError:
+            pass
 
     loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.set_debug(debug)
     try:
-        asyncio.set_event_loop(loop)
-        loop.set_debug(debug)
         loop.run_until_complete(setup_coro)
         loop.run_until_complete(wait_for_stop())
     finally:
@@ -108,4 +101,5 @@ async def wait_for_stop():
                 loop.run_until_complete(loop.shutdown_asyncgens())
         finally:
             loop.stop()
+            loop.close()
             asyncio.set_event_loop(None)