Add cli pathing application (#422)

* Add a cli path application

* Add a custom cli config variable for the .pth file directory

* Add commands for adding / removing hub-parsers paths from sys.path

* Add decorators module ...

Moves the operations_mode decorator into the new module. Adds another
decorator for checking the python system path and adding any paths
specified in our designated .pth file. This applies for any file that
might need access to the hub-parsers

* Improve async handling in decorators

* Fix the hub parsers paths to the parent to correctly import hub

* Cleanup documentation for cli pathing application

Author: ctrl-schaff

biothings/cli/__init__.py

@@ -14,6 +14,7 @@
 from biothings.cli.commands.admin import build_admin_application
 from biothings.cli.commands.config import config_application, load_configuration
 from biothings.cli.commands.dataplugin import dataplugin_application
+from biothings.cli.commands.pathing import path_application
 
 
 def setup_logging_configuration(logging_level: Literal[10, 20, 30, 40, 50]) -> None:
@@ -63,4 +64,5 @@ def main():
 
     admin_application.add_typer(dataplugin_application, name="dataplugin")
     admin_application.add_typer(config_application, name="config")
+    admin_application.add_typer(path_application, name="path")
     return admin_application()

biothings/cli/commands/config.py

@@ -268,6 +268,13 @@ def default_biothings_configuration() -> dict:
         "HUB_MAX_WORKERS": os.cpu_count(),
         "MAX_QUEUED_JOBS": 1000
     }
+
+    # specific attributes to the biothings-cli application
+    cli_configuration = {
+        "BIOTHINGS_CLI_PATH": "biothings_hub/path",
+    }
+    configuration.update(cli_configuration)
+
     return configuration
 
 

biothings/cli/commands/decorators.py

@@ -0,0 +1,136 @@
+"""
+Collection of decorators for usage within the biothings-cli
+
+These are often method we want associated with many of the plugin methods we
+use, but don't directly impact the logic of the actual operation. Typically things
+related to paths and configurations that apply to large swaths of the cli
+would make sense as a decorator
+"""
+
+import functools
+import inspect
+import logging
+import pathlib
+import sys
+from typing import Callable
+
+from biothings.cli.exceptions import MissingPluginName
+
+
+logger = logging.getLogger(name="biothings-cli")
+
+
+def operation_mode(operation: Callable):
+    """
+    Based off the directory structure for where the biothings-cli
+    was invoked we set the "mode" to one of two states:
+
+    0) singular
+    The current working directory contains a singular data-plugin
+
+    In this case we don't require a plugin_name argument to be passed
+    at the command-line
+
+    1) hub
+    The current working directory contains N directories operating as a
+    "hub" or collection of data-plugins under one umbrella
+
+    In this case we do require a plugin_name argument to be passed
+    at the command-line. Otherwise we have no idea which data-plugin to
+    refer to
+
+    We attempt to load the plugin from this working directory. If we sucessfully load
+    either a manifest or advanced plugin, then we can safely say this is a singular
+    dataplugin
+
+    If we cannot load either a manifest or advanced plugin then we default assume that
+    the mode is hub
+    """
+
+    @functools.wraps(operation)
+    def determine_operation_mode(*args, **kwargs):
+
+        def determine_hub_mode():
+            working_directory = pathlib.Path.cwd()
+            working_directory_files = {file.name for file in working_directory.iterdir()}
+
+            mode = None
+            if "manifest.json" in working_directory_files or "manifest.yaml" in working_directory_files:
+                logger.debug("Inferring singular manifest plugin from directory structure")
+                mode = "SINGULAR"
+            elif "__init__.py" in working_directory_files:
+                logger.debug("Inferring singular advanced plugin from directory structure")
+                mode = "SINGULAR"
+            else:
+                logger.debug("Inferring multiple plugins from directory structure")
+                mode = "HUB"
+
+            if mode == "SINGULAR":
+                if kwargs.get("plugin_name", None) is not None:
+                    kwargs["plugin_name"] = None
+            elif mode == "HUB":
+                if kwargs.get("plugin_name", None) is None:
+                    raise MissingPluginName(working_directory)
+
+        @functools.wraps(operation)
+        def handle_function(*args, **kwargs):
+            operation_result = operation(*args, **kwargs)
+            return operation_result
+
+        @functools.wraps(operation)
+        async def handle_corountine(*args, **kwargs):
+            operation_result = await operation(*args, **kwargs)
+            return operation_result
+
+        determine_hub_mode()
+
+        if inspect.iscoroutinefunction(operation):
+            return handle_corountine(*args, **kwargs)
+        else:
+            return handle_function(*args, **kwargs)
+
+    return determine_operation_mode
+
+
+def cli_system_path(operation: Callable):  # pylint: disable=unused-argument
+    """
+    Used for ensuring that if we've appended files to biothings-cli
+    path file (stored under config.BIOTHINGS_CLI_PATH), then we need to update
+    the system path so we can discover the modules at runtime
+    """
+
+    @functools.wraps(operation)
+    def update_system_path(*args, **kwargs):
+
+        def update_system_path_from_file():
+            from biothings import config
+
+            discovery_path = pathlib.Path(config.BIOTHINGS_CLI_PATH).resolve().absolute()
+            path_file = discovery_path.joinpath("biothings_cli.pth")
+
+            if path_file.exists():
+                with open(path_file, "r", encoding="utf-8") as handle:
+                    path_entries = handle.readlines()
+                    path_entries = [entry.strip("\n") for entry in path_entries]
+                    sys.path.extend(path_entries)
+                    for path in path_entries:
+                        logger.debug("Adding %s to system path", path)
+
+        @functools.wraps(operation)
+        def handle_function(*args, **kwargs):
+            operation_result = operation(*args, **kwargs)
+            return operation_result
+
+        @functools.wraps(operation)
+        async def handle_corountine(*args, **kwargs):
+            operation_result = await operation(*args, **kwargs)
+            return operation_result
+
+        update_system_path_from_file()
+
+        if inspect.iscoroutinefunction(operation):
+            return handle_corountine(*args, **kwargs)
+        else:
+            return handle_function(*args, **kwargs)
+
+    return update_system_path

biothings/cli/commands/operations.py

@@ -47,7 +47,6 @@
 """
 
 import asyncio
-import functools
 import logging
 import multiprocessing
 import os
@@ -57,7 +56,7 @@
 import shutil
 import sys
 import uuid
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import jsonschema
 import rich
@@ -67,8 +66,9 @@
 from rich.console import Console
 from rich.panel import Panel
 
+from biothings.cli.commands.decorators import cli_system_path, operation_mode
 from biothings.cli.structure import TEMPLATE_DIRECTORY
-from biothings.cli.exceptions import MissingPluginName, UnknownUploaderSource
+from biothings.cli.exceptions import UnknownUploaderSource
 from biothings.cli.utils import (
     clean_dumped_files,
     clean_uploaded_sources,
@@ -87,65 +87,8 @@
 logger = logging.getLogger(name="biothings-cli")
 
 
-def operation_mode(operation_method: Callable):
-    """
-    Based off the directory structure for where the biothings-cli
-    was invoked we set the "mode" to one of two states:
-
-    0) singular
-    The current working directory contains a singular data-plugin
-
-    In this case we don't require a plugin_name argument to be passed
-    at the command-line
-
-    1) hub
-    The current working directory contains N directories operating as a
-    "hub" or collection of data-plugins under one umbrella
-
-    In this case we do require a plugin_name argument to be passed
-    at the command-line. Otherwise we have no idea which data-plugin to
-    refer to
-
-    We attempt to load the plugin from this working directory. If we sucessfully load
-    either a manifest or advanced plugin, then we can safely say this is a singular
-    dataplugin
-
-    If we cannot load either a manifest or advanced plugin then we default assume that
-    the mode is hub
-    """
-
-    @functools.wraps(operation_method)
-    def determine_operation_mode(*args, **kwargs):
-        working_directory = pathlib.Path.cwd()
-        working_directory_files = {file.name for file in working_directory.iterdir()}
-
-        mode = None
-        if "manifest.json" in working_directory_files or "manifest.yaml" in working_directory_files:
-            logger.debug("Inferring singular manifest plugin from directory structure")
-            mode = "SINGULAR"
-        elif "__init__.py" in working_directory_files:
-            logger.debug("Inferring singular advanced plugin from directory structure")
-            mode = "SINGULAR"
-        else:
-            logger.debug("Inferring multiple plugins from directory structure")
-            mode = "HUB"
-
-        if mode == "SINGULAR":
-            if kwargs.get("plugin_name", None) is not None:
-                kwargs["plugin_name"] = None
-        elif mode == "HUB":
-            if kwargs.get("plugin_name", None) is None:
-                raise MissingPluginName(working_directory)
-
-        operation_result = operation_method(*args, **kwargs)
-        return operation_result
-
-    return determine_operation_mode
-
-
 # do not apply operation_mode decorator since this operation means to create a new plugin
 # regardless what the current working directory has
-# @operation_mode
 def do_create(plugin_name: str, multi_uploaders: bool = False, parallelizer: bool = False):
     """
     Create a new data plugin from the template
@@ -178,6 +121,7 @@ def do_create(plugin_name: str, multi_uploaders: bool = False, parallelizer: boo
     logger.info("Successfully created data plugin template at: %s\n", new_plugin_directory)
 
 
+@cli_system_path
 @operation_mode
 async def do_dump(plugin_name: Optional[str] = None, show_dumped: bool = True) -> None:
     """
@@ -223,6 +167,7 @@ async def do_dump(plugin_name: Optional[str] = None, show_dumped: bool = True) -
         show_dumped_files(data_folder, assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_upload(plugin_name: Optional[str] = None, batch_limit: int = 10000, show_uploaded: bool = True) -> None:
     """
@@ -277,6 +222,7 @@ async def do_upload(plugin_name: Optional[str] = None, batch_limit: int = 10000,
         show_uploaded_sources(pathlib.Path(assistant_instance.plugin_directory), assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_parallel_upload(
     plugin_name: Optional[str] = None, batch_limit: int = 10000, show_uploaded: bool = True
@@ -344,6 +290,7 @@ async def do_parallel_upload(
         show_uploaded_sources(pathlib.Path(assistant_instance.plugin_directory), assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_dump_and_upload(plugin_name: str) -> None:
     """
@@ -354,6 +301,7 @@ async def do_dump_and_upload(plugin_name: str) -> None:
     logger.info("[green]Success![/green] :rocket:", extra={"markup": True})
 
 
+@cli_system_path
 @operation_mode
 async def do_index(plugin_name: Optional[str] = None, sub_source_name: Optional[str] = None) -> None:
     """
@@ -540,6 +488,7 @@ async def do_index(plugin_name: Optional[str] = None, sub_source_name: Optional[
     await show_source_index(index_name, assistant_instance.index_manager, elasticsearch_mapping)
 
 
+@cli_system_path
 @operation_mode
 async def do_list(
     plugin_name: Optional[str] = None, dump: bool = True, upload: bool = True, hubdb: bool = False
@@ -569,6 +518,7 @@ async def do_list(
         show_hubdb_content()
 
 
+@cli_system_path
 @operation_mode
 async def do_inspect(
     plugin_name: Optional[str] = None,
@@ -633,6 +583,7 @@ async def do_inspect(
                 write_mapping_to_file(sub_output, inspection_mapping)
 
 
+@cli_system_path
 @operation_mode
 async def do_serve(plugin_name: Optional[str] = None, host: str = "localhost", port: int = 9999):
     """
@@ -651,6 +602,7 @@ async def do_serve(plugin_name: Optional[str] = None, host: str = "localhost", p
     await main(host=host, port=port, db=src_db, table_space=table_space)
 
 
+@cli_system_path
 @operation_mode
 async def do_clean(
     plugin_name: Optional[str] = None, dump: bool = False, upload: bool = False, clean_all: bool = False
@@ -714,6 +666,7 @@ async def display_schema():
     console.print(panel)
 
 
+@cli_system_path
 @operation_mode
 async def validate_manifest(plugin_name: Optional[str] = None):
     """