feat(cli-import): make CLI import more robust (#960)

- Support `--app_dir` and add current dir to it by default
- Improve error message
- Update docs
- Cleanup unrelated comments

Author: georgeh0

docs/docs/core/cli.mdx

@@ -17,8 +17,13 @@ Once CocoIndex is installed, you can invoke the CLI directly using the `cocoinde
 ### APP_TARGET Format
 
 The `APP_TARGET` can be:
-1.  A **path to a Python file** defining your flows (e.g., `main.py`, `path/to/my_flows.py`).
-2.  An **installed Python module name** that contains your flow definitions (e.g., `my_package.flows`).
+1.  An **Python module name** that contains your flow definitions (e.g., `main`, `my_package.flows`).
+    You can also use `--app-dir <path>` to specify the base directory to load the module from.
+
+2.  A **path to a Python file** defining your flows (e.g., `main.py`, `path/to/my_flows.py`).
+
+    The file will be loaded as a top-level Python module, e.g. relative imports will not work as its parent package is not defined (similar to how `python main.py` works).
+
 3.  For commands that operate on a *specific flow* (like `show`, `update`, `evaluate`), you can combine the application reference with a flow name:
     * `path/to/my_flows.py:MyFlow`
     * `my_package.flows:MyFlow`
@@ -44,6 +49,7 @@ If no file is found, only existing system environment variables are used.
 CocoIndex CLI supports the following global options:
 
 * `--env-file <path>`: Load environment variables from a specified `.env` file. If not provided, `.env` in the current directory is loaded if it exists.
+* `--app-dir <path>`: Load apps from the specified directory. It will be treated as part of `PYTHONPATH`. Default to the current directory.
 * `--version`: Show the CocoIndex version and exit.
 * `--help`: Show the main help message and exit.
 

python/cocoindex/cli.py

@@ -5,6 +5,7 @@
 import os
 import signal
 import threading
+import sys
 from types import FrameType
 from typing import Any, Iterable
 
@@ -19,9 +20,8 @@
 from .setup import flow_names_with_setup
 from .runtime import execution_context
 from .subprocess_exec import add_user_app
-from .user_app_loader import load_user_app
+from .user_app_loader import load_user_app, Error as UserAppLoaderError
 
-# Create ServerSettings lazily upon first call, as environment variables may be loaded from files, etc.
 COCOINDEX_HOST = "https://cocoindex.io"
 
 
@@ -77,7 +77,16 @@ def _get_app_ref_from_specifier(
 
 
 def _load_user_app(app_target: str) -> None:
-    load_user_app(app_target)
+    if not app_target:
+        raise click.ClickException("Application target not provided.")
+
+    try:
+        load_user_app(app_target)
+    except UserAppLoaderError as e:
+        raise click.ClickException(
+            f"Failed to load APP_TARGET '{app_target}': {e}"
+        ) from e
+
     add_user_app(app_target)
 
 
@@ -99,7 +108,13 @@ def _initialize_cocoindex_in_process() -> None:
     default=None,
     show_default=False,
 )
-def cli(env_file: str | None = None) -> None:
+@click.option(
+    "--app-dir",
+    help="Load apps from the specified directory. Default to the current directory.",
+    default="",
+    show_default=True,
+)
+def cli(env_file: str | None = None, app_dir: str | None = "") -> None:
     """
     CLI for Cocoindex.
     """
@@ -109,6 +124,9 @@ def cli(env_file: str | None = None) -> None:
         loaded_env_path = os.path.abspath(dotenv_path)
         click.echo(f"Loaded environment variables from: {loaded_env_path}\n", err=True)
 
+    if app_dir is not None:
+        sys.path.insert(0, app_dir)
+
     try:
         _initialize_cocoindex_in_process()
     except Exception as e:

python/cocoindex/user_app_loader.py

@@ -1,23 +1,27 @@
 import os
 import sys
 import importlib
-import click
 import types
 
 
+class Error(Exception):
+    """
+    Exception raised when a user app target is invalid or cannot be loaded.
+    """
+
+    pass
+
+
 def load_user_app(app_target: str) -> types.ModuleType:
     """
     Loads the user's application, which can be a file path or an installed module name.
     Exits on failure.
     """
-    if not app_target:
-        raise click.ClickException("Application target not provided.")
-
     looks_like_path = os.sep in app_target or app_target.lower().endswith(".py")
 
     if looks_like_path:
         if not os.path.isfile(app_target):
-            raise click.ClickException(f"Application file path not found: {app_target}")
+            raise Error(f"Application file path not found: {app_target}")
         app_path = os.path.abspath(app_target)
         app_dir = os.path.dirname(app_path)
         module_name = os.path.splitext(os.path.basename(app_path))[0]
@@ -35,7 +39,7 @@ def load_user_app(app_target: str) -> types.ModuleType:
             spec.loader.exec_module(module)
             return module
         except (ImportError, FileNotFoundError, PermissionError) as e:
-            raise click.ClickException(f"Failed importing file '{app_path}': {e}")
+            raise Error(f"Failed importing file '{app_path}': {e}") from e
         finally:
             if app_dir in sys.path and sys.path[0] == app_dir:
                 sys.path.pop(0)
@@ -44,8 +48,6 @@ def load_user_app(app_target: str) -> types.ModuleType:
     try:
         return importlib.import_module(app_target)
     except ImportError as e:
-        raise click.ClickException(f"Failed to load module '{app_target}': {e}")
+        raise Error(f"Failed to load module '{app_target}': {e}") from e
     except Exception as e:
-        raise click.ClickException(
-            f"Unexpected error importing module '{app_target}': {e}"
-        )
+        raise Error(f"Unexpected error importing module '{app_target}': {e}") from e