feat!: overhaul CLI (#230)

* implement GenomicMedLab/software-templates#83
* update docs

Author: jsstevenson

docs/source/conf.py

@@ -21,6 +21,7 @@
     "sphinx_copybutton",
     "sphinx.ext.autosummary",
     "sphinx_github_changelog",
+    "sphinx_click",
 ]
 
 templates_path = ["_templates"]
@@ -77,3 +78,106 @@ def linkcode_resolve(domain, info):
 # -- code block style --------------------------------------------------------
 pygments_style = "default"
 pygements_dark_style = "monokai"
+
+
+# -- sphinx-click ------------------------------------------------------------
+# These functions let us write descriptions/docstrings in a way that doesn't look
+# weird in the Click CLI, but get additional formatting in the sphinx-click autodocs for
+# better readability.
+import re
+
+from click.core import Context
+from sphinx.application import Sphinx
+from sphinx_click.ext import _get_usage, _indent
+
+
+CMD_PATTERN = r"--[^ ]+"
+STR_PATTERN = r"\"[^ ]+\""
+SNAKE_PATTERN = r"[A-Z]+_[A-Z_]*[A-Z][., ]"
+
+
+def _add_formatting_to_string(line: str) -> str:
+    """Add fixed-width code formatting to span sections in lines:
+
+    * shell options, eg `--update_all`
+    * double-quoted strings, eg `"DO"`
+    * all caps SNAKE_CASE env vars, eg `DISEASE_NORM_REMOTE_DB_URL`
+    """
+    for pattern in (CMD_PATTERN, STR_PATTERN, SNAKE_PATTERN):
+        line = re.sub(pattern, lambda x: f"``{x.group()}``", line)
+    return line
+
+
+def process_description(app: Sphinx, ctx: Context, lines: list[str]):
+    """Add custom formatting to sphinx-click autodoc descriptions.
+
+    * remove :param: :return: etc
+    * add fixed-width (code) font to certain words
+    * add code block formatting to example shell commands
+    * move primary usage example to the top of the description
+
+    Because we have to modify the lines list in place, we have to make multiple passes
+    through it to format everything correctly.
+    """
+    if not lines:
+        return
+
+    # chop off params
+    param_boundary = None
+    for i, line in enumerate(lines):
+        if ":param" in line:
+            param_boundary = i
+            break
+    if param_boundary is not None:
+        del lines[param_boundary:]
+        lines[-1] = ""
+
+    # add code formatting to strings, commands, and env vars
+    lines_to_fmt = []
+    for i, line in enumerate(lines):
+        if line.startswith(("   ", ">>> ", "|")):
+            continue  # skip example code blocks
+        if any(
+            [
+                re.findall(CMD_PATTERN, line),
+                re.findall(STR_PATTERN, line),
+                re.findall(SNAKE_PATTERN, line),
+            ]
+        ):
+            lines_to_fmt.append(i)
+    for line_num in lines_to_fmt:
+        lines[line_num] = _add_formatting_to_string(lines[line_num])
+
+    # add code block formatting to example console commands
+    for i in range(len(lines) - 1, -1, -1):
+        if lines[i].startswith(("    ", "|     ")):
+            if lines[i].startswith("|     "):
+                lines[i] = lines[i][3:]
+            if (i == 0 or lines[i - 1] == "\b" or lines[i - 1] == ""):
+                lines.insert(i, "")
+                lines.insert(i, ".. code-block:: console")
+
+    # put usage at the top of the description
+    lines.insert(0, "")
+    for usage_line in _get_usage(ctx).splitlines()[::-1]:
+        lines.insert(0, _indent(usage_line))
+    lines.insert(0, "")
+    lines.insert(0, ".. code-block:: shell")
+
+
+def process_option(app: Sphinx, ctx: Context, lines: list[str]):
+    """Add fixed-width formatting to strings in sphinx-click autodoc options."""
+    for i, line in enumerate(lines):
+        if re.findall(STR_PATTERN, line):
+            lines[i] = re.sub(STR_PATTERN, lambda x: f"``{x.group()}``", line)
+
+
+def setup(app):
+    """Used to hook format customization into sphinx-click build.
+
+    In particular, since we move usage to the top of the command description, we need
+    an extra hook here to silence the built-in usage section.
+    """
+    app.connect("sphinx-click-process-description", process_description)
+    app.connect("sphinx-click-process-options", process_option)
+    app.connect("sphinx-click-process-usage", lambda app, ctx, lines: lines.clear())

docs/source/managing_data/loading_and_updating_data.rst

@@ -3,55 +3,12 @@
 Loading and updating data
 =========================
 
+The primary means of managing Disease Normalizer data is via the included command-line interface.
+
 .. note::
 
     See the :ref:`ETL API documentation<etl-api>` for information on programmatic access to the data loader classes.
 
-Full load/reload
-----------------
-
-Calling the Disease Normalizer update command with the ``--update_all`` and ``--update_merged`` flags will delete all existing data, fetch new source data if available, and then perform a complete reload of the database (including merged records):
-
-.. code-block:: shell
-
-    disease_norm_update --update_all --update_merged
-
-
-Reload individual source
-------------------------
-
-To update specific sources, call the ``--sources`` option with one or more source name(s) quoted and separated by spaces. While it is possible to update individual source data without also updating the normalized record data, that may affect the proper function of the normalized query endpoints, so it is recommended to include the ``--update_merged`` flag as well.
-
-.. code-block:: shell
-
-    disease_norm_update --sources="HGNC NCBI" --update_merged
-
-
-Use local data
---------------
-
-The Disease Normalizer will fetch the latest available data from all sources if local data is out-of-date. To suppress this and force usage of local files, use the `--use_existing` flag:
-
-.. code-block:: shell
-
-    disease_norm_update --update_all --use_existing
-
-
-Check DB health
----------------
-
-The shell command ``disease_norm_check_db`` performs a basic check on the database status. It first confirms that the database's schema exists, and then identifies whether metadata is available for each source, and whether disease record and normalized concept tables are non-empty. Check the process's exit code for the result (per the UNIX standard, ``0`` means success, and any other return code means failure).
-
-.. code-block:: console
-
-    $ disease_norm_check_db
-    $ echo $?
-    1  # indicates failure
-
-This command is equivalent to the combination of the database classes' ``check_schema_initialized`` and ``check_tables_populated`` methods:
-
-.. code-block:: python
-
-   from disease.database import create_db
-   db = create_db()
-   db_is_healthy = db.check_schema_initialized() and db.check_tables_populated()
+.. click:: disease.cli:cli
+   :prog: disease-normalizer
+   :nested: full

docs/source/managing_data/postgresql.rst

@@ -24,18 +24,18 @@ Once created, set the environment variable ``DISEASE_NORM_DB_URL`` to a connecti
 Load from remote source
 --------------------------------
 
-The Disease Normalizer's PostgreSQL class provides the ``disease_norm_update_remote`` shell command to refresh its data directly from a remotely-stored SQL dump, instead of acquiring, transforming, and loading source data. This enables data loading on the order of seconds rather than hours. See the command description at ``disease_norm_update_remote --help`` for more information.
+The Disease Normalizer's PostgreSQL class supports a CLI command to refresh its data directly from a remotely-stored SQL dump, instead of acquiring, transforming, and loading source data. This enables data loading on the order of seconds rather than hours. See the command description at ``disease-normalizer update-from-remote --help`` for more information.
 
 By default, this command will fetch the `latest data dump <https://vicc-normalizers.s3.us-east-2.amazonaws.com/disease_normalization/postgresql/disease_norm_latest.sql.tar.gz>`_ provided by the VICC. Alternative URLs can be set with the ``--data_url`` option: ::
 
-    disease_norm_update_remote --data_url=https://vicc-normalizers.s3.us-east-2.amazonaws.com/disease_normalization/postgresql/disease_norm_20230322163523.sql.tar.gz
+    disease-normalizer update-from-remote --data_url=https://vicc-normalizers.s3.us-east-2.amazonaws.com/disease_normalization/postgresql/disease_norm_20230322163523.sql.tar.gz
 
 
 Create SQL dump from database
 -----------------------------
 
-The Disease Normalizer's PostgreSQL class also provides the ``disease_norm_dump`` shell command to create a SQL dump of current data into a file. This command will create a file named ``disease_norm_YYYYMMDDHHmmss.sql`` in the current directory; the ``-o`` option can be used to specify an alternate location, like so: ::
+The Disease Normalizer's PostgreSQL class also supports a CLI command to create a SQL dump of current data into a file. This command will create a file named ``disease_norm_YYYYMMDDHHmmss.sql`` in the current directory; the ``-o`` option can be used to specify an alternate location, like so: ::
 
-    disease_norm_dump -o ~/.disease_data/
+    disease-normalizer dump-database -o ~/.disease_data/
 
-See ``disease_norm_dump --help`` for more information.
+See ``disease-normalizer dump-database --help`` for more information.

docs/source/reference/api/etl/disease.etl.update.rst

@@ -0,0 +1,9 @@
+disease.etl.update
+==================
+
+.. automodule:: disease.etl.update
+   :members:
+   :undoc-members:
+   :special-members: __init__
+   :inherited-members:
+   :exclude-members: model_fields, model_config

docs/source/reference/index.rst

@@ -37,6 +37,7 @@ ETL Modules
    :template: module_summary_inh.rst
 
    disease.etl.base
+   disease.etl.update
    disease.etl.do
    disease.etl.mondo
    disease.etl.ncit

pyproject.toml

@@ -51,7 +51,8 @@ docs = [
     "sphinxext-opengraph==0.8.2",
     "furo==2023.3.27",
     "gravis==0.1.0",
-    "sphinx-github-changelog==1.2.1"
+    "sphinx-github-changelog==1.2.1",
+    "sphinx-click==5.0.1",
 ]
 
 [project.urls]
@@ -62,10 +63,7 @@ Source = "https://github.com/cancervariants/disease-normalization"
 "Bug Tracker" = "https://github.com/cancervariants/disease-normalization/issues"
 
 [project.scripts]
-disease_norm_update = "disease.cli:update_db"
-disease_norm_update_remote = "disease.cli:update_from_remote"
-disease_norm_dump = "disease.cli:dump_database"
-disease_norm_check_db = "disease.cli:check_db"
+disease-normalizer = "disease.cli:cli"
 
 [build-system]
 requires = ["setuptools>=64", "setuptools_scm>=8"]
@@ -88,7 +86,7 @@ branch = true
 
 [tool.ruff]
 src = ["src"]
-exclude = ["docs/source/conf.py", "analysis/"]
+extend-exclude = ["docs/source/conf.py", "analysis/"]
 
 [tool.ruff.lint]
 select = [