refactor: Make CLI commands consistent (#163)

* refactor: Make all cli.admin module names singular
* refactor: Reorganize admin command hierarchy
  - Move individual "entity" and "entities" commands
    into "entity info" and "entity list" commands.
  - No longer duplicate arguments and options
  - Just "generate" the command functions multiple times and place
    it under different Click groups.
  - Avoid circular imports by making client.cli.main as a module
    when doing so.

* wip: Add `client.cli.output` subpkg to provide common interface for output
  - All GraphQL/output fields are now defined as FieldSpec.
  - All CLI/functional API implementations refer the central FieldSet
    defined in the `ai.backend.client.output.fields` module.
  - All FieldSpec instances have their own formatters.

* setup: Update backend.ai-cli
* fix: remove support for servers with API ver older than v5.20191215
* maintenance: Remove Python 3.7 support and now require 3.8 or later.
* fix: Improve handling of JSON-string CLI args better

* Overrided the CLA check due to git client misconfiguration to specify author emails

Co-authored-by: Chrysan Angela Piarso <[email protected]>

Author: 2 people

.github/workflows/default.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -73,7 +73,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

changes/163.breaking

@@ -0,0 +1,6 @@
+Multiple breaking changes related to pagination and CLI outputs:
+- The CLI command hierarchy and arguments are revamped to be much more consistent with enumeration and item queries, and to support global options such as `--output=json` and `-y`/`--yes` flags. This is not backward-compatible due to parsing ambiguity.
+- When invoking the functional APIs to retrieve the details and lists, you need to change your `fields` arguments to be a list of `FieldSpec` objects instead of strings.  You may refer `ai.backend.client.output.fields` for predefined field definitions and individual functional API modules for their default list/detail field sets.
+  You may also define and pass your own `FieldSpec` and `OutputFormatter` objects as well.
+- It requires the Backend.AI Manager API v5.20191215 (release 20.03) or later.  If used with older managers, it will show a warning.
+- It requires Python 3.8 or higher to run.

setup.cfg

@@ -10,7 +10,6 @@ exclude = .git, .cache, .idea, __pycache__, venv, build, dist, docs
 
 [tool:pytest]
 norecursedirs = venv virtualenv .git
-timeout = 5
 markers =
     integration: Test cases that require real manager (and agents) to be running on http://localhost:8081.
 

setup.py

@@ -22,7 +22,6 @@
     'rich~=10.5.0',
     'tabulate>=0.8.9',
     'tqdm>=4.61',
-    'typing-extensions>=3.10.0',
     'yarl>=1.6.3',
     'backend.ai-cli~=0.5.0.post1',
 ]
@@ -37,7 +36,6 @@
     'pytest-mock',
     'pytest-asyncio>=0.15.1',
     'aioresponses>=0.7.2',
-    'asynctest>=0.13; python_version<"3.8"',
     'codecov',
 ]
 lint_requires = [
@@ -90,7 +88,6 @@ def read_src_version():
         'Intended Audience :: Developers',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Operating System :: POSIX',
@@ -102,7 +99,7 @@ def read_src_version():
     ],
     package_dir={'': 'src'},
     packages=find_namespace_packages(where='src', include='ai.backend.*'),
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     setup_requires=setup_requires,
     install_requires=install_requires,
     extras_require={
@@ -119,7 +116,7 @@ def read_src_version():
     },
     entry_points={
         'backendai_cli_v10': [
-            '_ = ai.backend.client.cli:main',
+            '_ = ai.backend.client.cli.main:main',
         ]
     },
 )

src/ai/backend/client/cli/__init__.py

@@ -1,44 +1,9 @@
-import warnings
-
-import click
-
-from .. import __version__
-from ..config import APIConfig, set_config
-from ai.backend.cli.extensions import ExtendedCommandGroup
-
-
-@click.group(
-    cls=ExtendedCommandGroup,
-    context_settings={
-        'help_option_names': ['-h', '--help'],
-    },
-)
-@click.option('--skip-sslcert-validation',
-              help='Skip SSL certificate validation for all API requests.',
-              is_flag=True)
-@click.version_option(version=__version__)
-def main(skip_sslcert_validation):
-    """
-    Backend.AI command line interface.
-    """
-    from .announcement import announce
-    config = APIConfig(
-        skip_sslcert_validation=skip_sslcert_validation,
-        announcement_handler=announce,
-    )
-    set_config(config)
-
-    from .pretty import show_warning
-    warnings.showwarning = show_warning
-
-
-def _attach_command():
-    from . import admin, config, app, files, logs, manager, proxy, ps, run # noqa
-    from . import ssh # noqa
-    from . import vfolder       # noqa
-    from . import session_template  # noqa
-    from . import dotfile  # noqa
-    from . import server_log  # noqa
-
-
-_attach_command()
+from . import main  # noqa
+from . import config  # noqa
+from . import session  # noqa
+from . import session_template  # noqa
+from . import vfolder  # noqa
+from . import dotfile  # noqa
+from . import server_log  # noqa
+from . import admin  # noqa
+from . import app, logs, proxy, run # noqa

src/ai/backend/client/cli/__main__.py

@@ -1,4 +1,4 @@
-from . import main
+from .main import main
 
 
 main()

src/ai/backend/client/cli/admin/__init__.py

@@ -1,30 +1,27 @@
-from .. import main
+from ..main import main
 
 
 @main.group()
 def admin():
-    '''
-    Provides the admin API access.
-    '''
+    """
+    Administrative command set
+    """
 
 
-def _attach_command():
-    from . import (  # noqa
-        agents,
-        domains,
-        etcd,
-        groups,
-        images,
-        keypairs,
-        license,
-        resources,
-        resource_policies,
-        scaling_groups,
-        sessions,
-        storage,
-        users,
-        vfolders,
-    )
-
-
-_attach_command()
+from . import (  # noqa
+    agent,
+    domain,
+    etcd,
+    group,
+    image,
+    keypair,
+    manager,
+    license,
+    resource,
+    resource_policy,
+    scaling_group,
+    session,
+    storage,
+    user,
+    vfolder,
+)

src/ai/backend/client/cli/admin/agent.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ai.backend.client.session import Session
+from ai.backend.client.output.fields import agent_fields
+from ..types import CLIContext
+from . import admin
+
+
+@admin.group()
+def agent():
+    """
+    Agent administration commands.
+    """
+
+
+@agent.command()
+@click.pass_obj
+@click.argument('agent_id')
+def info(ctx: CLIContext, agent_id: str) -> None:
+    """
+    Show the information about the given agent.
+    """
+    fields = [
+        agent_fields['id'],
+        agent_fields['status'],
+        agent_fields['region'],
+        agent_fields['first_contact'],
+        agent_fields['cpu_cur_pct'],
+        agent_fields['available_slots'],
+        agent_fields['occupied_slots'],
+        agent_fields['hardware_metadata'],
+        agent_fields['live_stat'],
+    ]
+    with Session() as session:
+        try:
+            item = session.Agent.detail(agent_id=agent_id, fields=fields)
+            ctx.output.print_item(item, fields)
+        except Exception as e:
+            ctx.output.print_error(e)
+            sys.exit(1)
+
+
+@agent.command()
+@click.pass_obj
+@click.option('-s', '--status', type=str, default='ALIVE',
+              help='Filter agents by the given status.')
+@click.option('--scaling-group', '--sgroup', type=str, default=None,
+              help='Filter agents by the scaling group.')
+@click.option('--filter', 'filter_', default=None,
+              help='Set the query filter expression.')
+@click.option('--order', default=None,
+              help='Set the query ordering expression.')
+@click.option('--offset', default=0,
+              help='The index of the current page start for pagination.')
+@click.option('--limit', default=None,
+              help='The page size for pagination.')
+def list(
+    ctx: CLIContext,
+    status: str,
+    scaling_group: str | None,
+    filter_: str | None,
+    order: str | None,
+    offset: int,
+    limit: int | None,
+) -> None:
+    """
+    List agents.
+    (super-admin privilege required)
+    """
+    fields = [
+        agent_fields['id'],
+        agent_fields['status'],
+        agent_fields['scaling_group'],
+        agent_fields['region'],
+        agent_fields['first_contact'],
+        agent_fields['cpu_cur_pct'],
+        agent_fields['mem_cur_bytes'],
+        agent_fields['available_slots'],
+        agent_fields['occupied_slots'],
+    ]
+    try:
+        with Session() as session:
+            fetch_func = lambda pg_offset, pg_size: session.Agent.paginated_list(
+                status,
+                scaling_group,
+                fields=fields,
+                page_offset=pg_offset,
+                page_size=pg_size,
+                filter=filter_,
+                order=order,
+            )
+            ctx.output.print_paginated_list(
+                fetch_func,
+                initial_page_offset=offset,
+                page_size=limit,
+            )
+    except Exception as e:
+        ctx.output.print_error(e)
+        sys.exit(1)
+
+
+@admin.group()
+def watcher():
+    """
+    Agent watcher commands.
+
+    Available only for Linux-based agents.
+    """
+
+
+@watcher.command()
+@click.pass_obj
+@click.argument('agent', type=str)
+def status(ctx: CLIContext, agent: str) -> None:
+    """
+    Get agent and watcher status.
+    (superadmin privilege required)
+
+    \b
+    AGENT: Agent id.
+    """
+    with Session() as session:
+        try:
+            status = session.AgentWatcher.get_status(agent)
+            print(status)
+        except Exception as e:
+            ctx.output.print_error(e)
+            sys.exit(1)
+
+
+@watcher.command()
+@click.pass_obj
+@click.argument('agent', type=str)
+def agent_start(ctx: CLIContext, agent: str) -> None:
+    """
+    Start agent service.
+    (superadmin privilege required)
+
+    \b
+    AGENT: Agent id.
+    """
+    with Session() as session:
+        try:
+            status = session.AgentWatcher.agent_start(agent)
+            print(status)
+        except Exception as e:
+            ctx.output.print_error(e)
+            sys.exit(1)
+
+
+@watcher.command()
+@click.pass_obj
+@click.argument('agent', type=str)
+def agent_stop(ctx: CLIContext, agent: str) -> None:
+    """
+    Stop agent service.
+    (superadmin privilege required)
+
+    \b
+    AGENT: Agent id.
+    """
+    with Session() as session:
+        try:
+            status = session.AgentWatcher.agent_stop(agent)
+            print(status)
+        except Exception as e:
+            ctx.output.print_error(e)
+            sys.exit(1)
+
+
+@watcher.command()
+@click.pass_obj
+@click.argument('agent', type=str)
+def agent_restart(ctx: CLIContext, agent: str) -> None:
+    """
+    Restart agent service.
+    (superadmin privilege required)
+
+    \b
+    AGENT: Agent id.
+    """
+    with Session() as session:
+        try:
+            status = session.AgentWatcher.agent_restart(agent)
+            print(status)
+        except Exception as e:
+            ctx.output.print_error(e)
+            sys.exit(1)