Merge pull request #103 from jeromekelleher/cli-refinements

Cli refinements

Author: jeromekelleher

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.0.3 2024-03-28
+
+- Various refinements to the CLI
+
 # 0.0.2 2024-03-27
 
 - Merged 1D and 2D encode steps into one, and change rate reporting to bytes

README.md

@@ -31,6 +31,7 @@ and will always work.
 
 ## vcf2zarr
 
+
 Convert a VCF to zarr format:
 
 ```
@@ -72,6 +73,21 @@ vcf2zarr encode tmp/sample.exploded tmp/sample.zarr -s sample.schema.json
 Use the ``-p, --worker-processes`` argument to control the number of workers used
 in the ``explode`` and ``encode`` phases.
 
+### Shell completion
+
+To enable shell completion for a particular session in Bash do:
+
+```
+eval "$(_VCF2ZARR_COMPLETE=bash_source vcf2zarr)" 
+```
+
+If you add this to your ``.bashrc`` vcf2zarr shell completion should available
+in all new shell sessions.
+
+See the [Click documentation](https://click.palletsprojects.com/en/8.1.x/shell-completion/#enabling-completion)
+for instructions on how to enable completion in other shells.
+a
+
 ## plink2zarr
 
 Convert a plink ``.bed`` file to zarr format. **This is incomplete**

bio2zarr/cli.py

@@ -1,3 +1,8 @@
+import logging
+import os
+import pathlib
+import shutil
+
 import click
 import tabulate
 import coloredlogs
@@ -8,6 +13,9 @@
 from . import provenance
 
 
+logger = logging.getLogger(__name__)
+
+
 class NaturalOrderGroup(click.Group):
     """
     List commands in the order they are provided in the help text.
@@ -18,8 +26,32 @@ def list_commands(self, ctx):
 
 
 # Common arguments/options
+vcfs = click.argument(
+    "vcfs", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False)
+)
+
+new_icf_path = click.argument(
+    "icf_path", type=click.Path(file_okay=False, dir_okay=True)
+)
+
+icf_path = click.argument(
+    "icf_path", type=click.Path(exists=True, file_okay=False, dir_okay=True)
+)
+
+new_zarr_path = click.argument(
+    "zarr_path", type=click.Path(file_okay=False, dir_okay=True)
+)
+
 verbose = click.option("-v", "--verbose", count=True, help="Increase verbosity")
 
+force = click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    flag_value=True,
+    help="Force overwriting of existing directories",
+)
+
 version = click.version_option(version=f"{provenance.__version__}")
 
 worker_processes = click.option(
@@ -64,41 +96,62 @@ def setup_logging(verbosity):
     coloredlogs.install(level=level)
 
 
+def check_overwrite_dir(path, force):
+    path = pathlib.Path(path)
+    if path.exists():
+        if not force:
+            click.confirm(
+                f"Do you want to overwrite {path}? (use --force to skip this check)",
+                abort=True,
+            )
+        # These trees can be mondo-big and on slow file systems, so it's entirely
+        # feasible that the delete would fail or be killed. This makes it less likely
+        # that partially deleted paths are mistaken for good paths.
+        tmp_delete_path = path.with_suffix(f"{path.suffix}.{os.getpid()}.DELETING")
+        logger.info(f"Deleting {path} (renamed to {tmp_delete_path} while in progress)")
+        os.rename(path, tmp_delete_path)
+        shutil.rmtree(tmp_delete_path)
+
+
 @click.command
-@click.argument("vcfs", nargs=-1, required=True)
-@click.argument("zarr_path", type=click.Path())
+@vcfs
+@new_icf_path
+@force
 @verbose
 @worker_processes
 @column_chunk_size
-def explode(vcfs, zarr_path, verbose, worker_processes, column_chunk_size):
+def explode(vcfs, icf_path, force, verbose, worker_processes, column_chunk_size):
     """
     Convert VCF(s) to intermediate columnar format
     """
     setup_logging(verbose)
+    check_overwrite_dir(icf_path, force)
     vcf.explode(
         vcfs,
-        zarr_path,
+        icf_path,
         worker_processes=worker_processes,
         column_chunk_size=column_chunk_size,
         show_progress=True,
     )
 
 
 @click.command
-@click.argument("vcfs", nargs=-1, required=True)
-@click.argument("icf_path", type=click.Path())
-@click.argument("num_partitions", type=int)
+@vcfs
+@new_icf_path
+@click.argument("num_partitions", type=click.IntRange(min=1))
+@force
 @column_chunk_size
 @verbose
 @worker_processes
 def dexplode_init(
-    vcfs, icf_path, num_partitions, column_chunk_size, verbose, worker_processes
+    vcfs, icf_path, num_partitions, force, column_chunk_size, verbose, worker_processes
 ):
     """
-    Initial step for parallel conversion of VCF(s) to intermediate columnar format
+    Initial step for distributed conversion of VCF(s) to intermediate columnar format
     over the requested number of paritions.
     """
     setup_logging(verbose)
+    check_overwrite_dir(icf_path, force)
     num_partitions = vcf.explode_init(
         icf_path,
         vcfs,
@@ -111,12 +164,12 @@ def dexplode_init(
 
 
 @click.command
-@click.argument("icf_path", type=click.Path())
-@click.argument("partition", type=int)
+@icf_path
+@click.argument("partition", type=click.IntRange(min=0))
 @verbose
 def dexplode_partition(icf_path, partition, verbose):
     """
-    Convert a VCF partition into intermediate columnar format. Must be called *after*
+    Convert a VCF partition to intermediate columnar format. Must be called *after*
     the ICF path has been initialised with dexplode_init. Partition indexes must be
     from 0 (inclusive) to the number of paritions returned by dexplode_init (exclusive).
     """
@@ -129,26 +182,26 @@ def dexplode_partition(icf_path, partition, verbose):
 @verbose
 def dexplode_finalise(path, verbose):
     """
-    Final step for parallel conversion of VCF(s) to intermediate columnar format
+    Final step for distributed conversion of VCF(s) to intermediate columnar format.
     """
     setup_logging(verbose)
     vcf.explode_finalise(path)
 
 
 @click.command
-@click.argument("icf_path", type=click.Path())
+@click.argument("path", type=click.Path())
 @verbose
-def inspect(icf_path, verbose):
+def inspect(path, verbose):
     """
-    Inspect an intermediate format or Zarr path.
+    Inspect an intermediate columnar format or Zarr path.
     """
     setup_logging(verbose)
-    data = vcf.inspect(icf_path)
+    data = vcf.inspect(path)
     click.echo(tabulate.tabulate(data, headers="keys"))
 
 
 @click.command
-@click.argument("icf_path", type=click.Path())
+@icf_path
 def mkschema(icf_path):
     """
     Generate a schema for zarr encoding
@@ -158,8 +211,9 @@ def mkschema(icf_path):
 
 
 @click.command
-@click.argument("icf_path", type=click.Path())
-@click.argument("zarr_path", type=click.Path())
+@icf_path
+@new_zarr_path
+@force
 @verbose
 @click.option("-s", "--schema", default=None, type=click.Path(exists=True))
 @variants_chunk_size
@@ -186,6 +240,7 @@ def mkschema(icf_path):
 def encode(
     icf_path,
     zarr_path,
+    force,
     verbose,
     schema,
     variants_chunk_size,
@@ -198,10 +253,11 @@ def encode(
     Encode intermediate columnar format (see explode) to vcfzarr.
     """
     setup_logging(verbose)
+    check_overwrite_dir(zarr_path, force)
     vcf.encode(
         icf_path,
         zarr_path,
-        schema,
+        schema_path=schema,
         variants_chunk_size=variants_chunk_size,
         samples_chunk_size=samples_chunk_size,
         max_v_chunks=max_variant_chunks,
@@ -212,8 +268,8 @@ def encode(
 
 
 @click.command(name="convert")
-@click.argument("vcfs", nargs=-1, required=True)
-@click.argument("zarr_path", type=click.Path())
+@vcfs
+@new_zarr_path
 @variants_chunk_size
 @samples_chunk_size
 @verbose
@@ -235,17 +291,6 @@ def convert_vcf(
     )
 
 
-@click.command
-@click.argument("vcfs", nargs=-1, required=True)
-@click.argument("zarr_path", type=click.Path())
-def validate(vcfs, zarr_path):
-    """
-    Development only, do not use. Will be removed before release.
-    """
-    # FIXME! Will silently not look at remaining VCFs
-    vcf.validate(vcfs[0], zarr_path, show_progress=True)
-
-
 @version
 @click.group(cls=NaturalOrderGroup)
 def vcf2zarr():
@@ -309,7 +354,6 @@ def vcf2zarr():
 vcf2zarr.add_command(dexplode_init)
 vcf2zarr.add_command(dexplode_partition)
 vcf2zarr.add_command(dexplode_finalise)
-vcf2zarr.add_command(validate)
 
 
 @click.command(name="convert")