Basic documentation and polish

Author: jeromekelleher

bio2zarr/cli.py

@@ -332,7 +332,17 @@ def dencode_init(
     verbose,
 ):
     """
-    TODO DOCUMENT
+    Initialise conversion of intermediate format to VCF Zarr. This will
+    set up the specified ZARR_PATH to perform this conversion over
+    NUM_PARTITIONS.
+
+    The output of this commmand is the actual number of partitions generated
+    (which may be less then the requested number, if there is not sufficient
+    chunks in the variants dimension) and a rough lower-bound on the amount
+    of memory required to encode a partition.
+
+    NOTE: the format of this output will likely change in subsequent releases;
+    it should not be considered machine-readable for now.
     """
     setup_logging(verbose)
     check_overwrite_dir(zarr_path, force)
@@ -363,7 +373,10 @@ def dencode_init(
 @verbose
 def dencode_partition(zarr_path, partition, verbose):
     """
-    TODO DOCUMENT
+    Convert a partition from intermediate columnar format to VCF Zarr.
+    Must be called *after* the Zarr path has been initialised with dencode_init.
+    Partition indexes must be from 0 (inclusive) to the number of paritions
+    returned by dencode_init (exclusive).
     """
     setup_logging(verbose)
     vcf.encode_partition(zarr_path, partition)
@@ -374,10 +387,10 @@ def dencode_partition(zarr_path, partition, verbose):
 @verbose
 def dencode_finalise(zarr_path, verbose):
     """
-    TODO DOCUMENT
+    Final step for distributed conversion of ICF to VCF Zarr.
     """
     setup_logging(verbose)
-    vcf.encode_finalise(zarr_path)
+    vcf.encode_finalise(zarr_path, show_progress=True)
 
 
 @click.command(name="convert")

bio2zarr/vcf.py

@@ -2026,7 +2026,7 @@ def encode_all_partitions(
     ):
         max_memory = parse_max_memory(max_memory)
         self.load_metadata()
-        num_partitions = len(self.num_partitions)
+        num_partitions = self.num_partitions
         per_worker_memory = self.get_max_encoding_memory()
         logger.info(
             f"Encoding Zarr over {num_partitions} partitions with "

tests/test_cli.py

@@ -396,7 +396,7 @@ def test_encode(self, mocked, tmp_path):
             **DEFAULT_ENCODE_ARGS,
         )
 
-    @mock.patch("bio2zarr.vcf.encode_init", return_value=10)
+    @mock.patch("bio2zarr.vcf.encode_init", return_value=(10, 1024))
     def test_dencode_init(self, mocked, tmp_path):
         icf_path = tmp_path / "icf"
         icf_path.mkdir()
@@ -408,7 +408,7 @@ def test_dencode_init(self, mocked, tmp_path):
             catch_exceptions=False,
         )
         assert result.exit_code == 0
-        assert result.stdout == "10\n"
+        assert result.stdout == "10\t1 KiB\n"
         assert len(result.stderr) == 0
         mocked.assert_called_once_with(
             str(icf_path),
@@ -564,7 +564,7 @@ def test_dencode(self, tmp_path):
             catch_exceptions=False,
         )
         assert result.exit_code == 0
-        assert result.stdout.strip() == "3"
+        assert result.stdout.split()[0] == "3"
 
         for j in range(3):
             result = runner.invoke(

tests/test_vcf.py

@@ -64,7 +64,7 @@ def test_not_enough_memory(self, tmp_path, icf_path, max_memory):
         with pytest.raises(ValueError, match="Insufficient memory"):
             vcf.encode(icf_path, zarr_path, max_memory=max_memory)
 
-    @pytest.mark.parametrize("max_memory", ["20MiB"])
+    @pytest.mark.parametrize("max_memory", ["150KiB", "200KiB"])
     def test_not_enough_memory_for_two(
         self, tmp_path, icf_path, zarr_path, caplog, max_memory
     ):