Merge pull request #37 from jeromekelleher/renaming-things

Renaming things

Author: jeromekelleher

README.md

@@ -29,22 +29,22 @@ python3 -m bio2zarr vcf2zarr explode tests/data/vcf/sample.vcf.gz tmp/sample.exp
 
 Then, (optionally) inspect this representation to get a feel for your dataset
 ```
-python3 -m bio2zarr vcf2zarr summarise tmp/sample.exploded
+python3 -m bio2zarr vcf2zarr inspec tmp/sample.exploded
 ```
 
 Then, (optionally) generate a conversion schema to describe the corresponding
 Zarr arrays:
 
 ```
-python3 -m bio2zarr vcf2zarr genspec tmp/sample.exploded > sample.schema.json
+python3 -m bio2zarr vcf2zarr mkschema tmp/sample.exploded > sample.schema.json
 ```
 
 View and edit the schema, deleting any columns you don't want.
 
 Finally, convert to Zarr
 
 ```
-python3 -m bio2zarr vcf2zarr to-zarr tmp/sample.exploded tmp/sample.zarr -s sample.schema.json
+python3 -m bio2zarr vcf2zarr encode tmp/sample.exploded tmp/sample.zarr -s sample.schema.json
 ```
 
 Use the ``-p, --worker-processes`` argument to control the number of workers used

bio2zarr/cli.py

@@ -35,6 +35,9 @@ def setup_logging(verbosity):
 @worker_processes
 @click.option("-c", "--column-chunk-size", type=int, default=64)
 def explode(vcfs, out_path, verbose, worker_processes, column_chunk_size):
+    """
+    Convert VCF(s) to columnar intermediate format
+    """
     setup_logging(verbose)
     vcf.explode(
         vcfs,
@@ -46,34 +49,42 @@ def explode(vcfs, out_path, verbose, worker_processes, column_chunk_size):
 
 
 @click.command
-@click.argument("columnarised", type=click.Path())
+@click.argument("if_path", type=click.Path())
 @verbose
-def summarise(columnarised, verbose):
+def inspect(if_path, verbose):
+    """
+    Inspect an intermediate format file
+    """
     setup_logging(verbose)
-    data = vcf.summarise(columnarised)
+    data = vcf.inspect(if_path)
     click.echo(tabulate.tabulate(data, headers="keys"))
 
 
 @click.command
-@click.argument("columnarised", type=click.Path())
-# @click.argument("specfile", type=click.Path())
-def genspec(columnarised):
+@click.argument("if_path", type=click.Path())
+def mkschema(if_path):
+    """
+    Generate a schema for zarr encoding
+    """
     stream = click.get_text_stream("stdout")
-    vcf.generate_spec(columnarised, stream)
+    vcf.mkschema(if_path, stream)
 
 
 @click.command
-@click.argument("columnarised", type=click.Path())
+@click.argument("if_path", type=click.Path())
 @click.argument("zarr_path", type=click.Path())
 @verbose
-@click.option("-s", "--conversion-spec", default=None)
+@click.option("-s", "--schema", default=None)
 @worker_processes
-def to_zarr(columnarised, zarr_path, verbose, conversion_spec, worker_processes):
+def encode(if_path, zarr_path, verbose, schema, worker_processes):
+    """
+    Encode intermediate format (see explode) to vcfzarr
+    """
     setup_logging(verbose)
-    vcf.to_zarr(
-        columnarised,
+    vcf.encode(
+        if_path,
         zarr_path,
-        conversion_spec,
+        schema,
         worker_processes=worker_processes,
         show_progress=True,
     )
@@ -85,16 +96,18 @@ def to_zarr(columnarised, zarr_path, verbose, conversion_spec, worker_processes)
 @verbose
 @worker_processes
 def convert_vcf(vcfs, out_path, verbose, worker_processes):
+    """
+    Convert input VCF(s) directly to vcfzarr (not recommended for large files)
+    """
     setup_logging(verbose)
-    vcf.convert_vcf(
-        vcfs, out_path, show_progress=True, worker_processes=worker_processes
-    )
+    vcf.convert(vcfs, out_path, show_progress=True, worker_processes=worker_processes)
 
 
 @click.command
 @click.argument("vcfs", nargs=-1, required=True)
 @click.argument("out_path", type=click.Path())
 def validate(vcfs, out_path):
+    # FIXME! Will silently not look at remaining VCFs
     vcf.validate(vcfs[0], out_path, show_progress=True)
 
 
@@ -103,10 +116,11 @@ def vcf2zarr():
     pass
 
 
+# TODO figure out how to get click to list these in the given order.
 vcf2zarr.add_command(explode)
-vcf2zarr.add_command(summarise)
-vcf2zarr.add_command(genspec)
-vcf2zarr.add_command(to_zarr)
+vcf2zarr.add_command(inspect)
+vcf2zarr.add_command(mkschema)
+vcf2zarr.add_command(encode)
 vcf2zarr.add_command(convert_vcf)
 vcf2zarr.add_command(validate)
 
@@ -118,6 +132,9 @@ def vcf2zarr():
 @click.option("--chunk-width", type=int, default=None)
 @click.option("--chunk-length", type=int, default=None)
 def convert_plink(in_path, out_path, worker_processes, chunk_width, chunk_length):
+    """
+    In development; DO NOT USE!
+    """
     plink.convert(
         in_path,
         out_path,

bio2zarr/vcf.py

@@ -879,14 +879,14 @@ def explode(
     )
 
 
-def summarise(columnarised):
-    pcvcf = vcf.PickleChunkedVcf.load(columnarised)
+def inspect(if_path):
+    # TODO add support for the Zarr format also
+    pcvcf = PickleChunkedVcf.load(if_path)
     return pcvcf.summary_table()
 
 
 @dataclasses.dataclass
 class ZarrColumnSpec:
-    # TODO change to "variable_name"
     name: str
     dtype: str
     shape: tuple
@@ -897,6 +897,11 @@ class ZarrColumnSpec:
     compressor: dict
     # TODO add filters
 
+    def __post_init__(self):
+        self.shape = tuple(self.shape)
+        self.chunks = tuple(self.chunks)
+        self.dimensions = tuple(self.dimensions)
+
 
 @dataclasses.dataclass
 class ZarrConversionSpec:
@@ -907,17 +912,26 @@ class ZarrConversionSpec:
     contig_id: list
     contig_length: list
     filter_id: list
-    variables: list
+    columns: dict
 
     def asdict(self):
         return dataclasses.asdict(self)
 
+    def asjson(self):
+        return json.dumps(self.asdict(), indent=4)
+
     @staticmethod
     def fromdict(d):
         ret = ZarrConversionSpec(**d)
-        ret.variables = [ZarrColumnSpec(**cd) for cd in d["variables"]]
+        ret.columns = {
+            key: ZarrColumnSpec(**value) for key, value in d["columns"].items()
+        }
         return ret
 
+    @staticmethod
+    def fromjson(s):
+        return ZarrConversionSpec.fromdict(json.loads(s))
+
     @staticmethod
     def generate(pcvcf, chunk_length=None, chunk_width=None):
         m = pcvcf.num_records
@@ -1069,7 +1083,7 @@ def fixed_field_spec(
         return ZarrConversionSpec(
             chunk_width=chunk_width,
             chunk_length=chunk_length,
-            variables=colspecs,
+            columns={col.name: col for col in colspecs},
             dimensions=["variants", "samples", "ploidy", "alleles", "filters"],
             sample_id=pcvcf.metadata.samples,
             contig_id=pcvcf.metadata.contig_names,
@@ -1260,8 +1274,8 @@ def convert(
         logger.info(f"Create zarr at {write_path}")
         sgvcf = SgvcfZarr(write_path)
         sgvcf.root = zarr.group(store=store, overwrite=True)
-        for variable in conversion_spec.variables[:]:
-            sgvcf.create_array(variable)
+        for column in conversion_spec.columns.values():
+            sgvcf.create_array(column)
 
         progress_config = core.ProgressConfig(
             total=pcvcf.total_uncompressed_bytes,
@@ -1286,7 +1300,7 @@ def convert(
             )
             pwm.submit(sgvcf.encode_filters, pcvcf, conversion_spec.filter_id)
             has_gt = False
-            for variable in conversion_spec.variables[:]:
+            for variable in conversion_spec.columns.values():
                 if variable.vcf_field is not None:
                     # print("Encode", variable.name)
                     # TODO for large columns it's probably worth splitting up
@@ -1308,32 +1322,29 @@ def convert(
         os.rename(write_path, path)
 
 
-def generate_spec(columnarised, out):
-    pcvcf = PickleChunkedVcf.load(columnarised)
+def mkschema(if_path, out):
+    pcvcf = PickleChunkedVcf.load(if_path)
     spec = ZarrConversionSpec.generate(pcvcf)
-    json.dump(spec.asdict(), out, indent=4)
+    out.write(spec.asjson())
 
 
-def to_zarr(
-    columnarised, zarr_path, conversion_spec, worker_processes=1, show_progress=False
-):
-    pcvcf = PickleChunkedVcf.load(columnarised)
-    if conversion_spec is None:
-        spec = ZarrConversionSpec.generate(pcvcf)
+def encode(if_path, zarr_path, schema_path, worker_processes=1, show_progress=False):
+    pcvcf = PickleChunkedVcf.load(if_path)
+    if schema_path is None:
+        schema = ZarrConversionSpec.generate(pcvcf)
     else:
-        with open(conversion_spec, "r") as f:
-            d = json.load(f)
-            spec = ZarrConversionSpec.fromdict(d)
+        with open(schema_path, "r") as f:
+            schema = ZarrConversionSpec.fromjson(f.read())
     SgvcfZarr.convert(
         pcvcf,
         zarr_path,
-        conversion_spec=spec,
+        conversion_spec=schema,
         worker_processes=worker_processes,
         show_progress=show_progress,
     )
 
 
-def convert_vcf(
+def convert(
     vcfs,
     out_path,
     *,

tests/test_cli.py

@@ -4,14 +4,13 @@
 
 from bio2zarr import cli
 
-class TestWithMocks:
 
+class TestWithMocks:
     def test_vcf_explode(self):
         runner = ct.CliRunner(mix_stderr=False)
         with mock.patch("bio2zarr.vcf.explode") as mocked:
             result = runner.invoke(
                 cli.vcf2zarr, ["explode", "source", "dest"], catch_exceptions=False
-
             )
             assert result.exit_code == 0
             assert len(result.stdout) == 0
@@ -24,15 +23,97 @@ def test_vcf_explode(self):
                 show_progress=True,
             )
 
-    def test_summarise(self):
+    def test_inspect(self):
         runner = ct.CliRunner(mix_stderr=False)
-        with mock.patch("bio2zarr.vcf.summarise", return_value={}) as mocked:
+        with mock.patch("bio2zarr.vcf.inspect", return_value={}) as mocked:
             result = runner.invoke(
-                cli.vcf2zarr, ["summarise", "path"], catch_exceptions=False
-
+                cli.vcf2zarr, ["inspect", "path"], catch_exceptions=False
             )
             assert result.exit_code == 0
             assert result.stdout == "\n"
             assert len(result.stderr) == 0
             mocked.assert_called_once_with("path")
 
+    def test_mkschema(self):
+        runner = ct.CliRunner(mix_stderr=False)
+        with mock.patch("bio2zarr.vcf.mkschema") as mocked:
+            result = runner.invoke(
+                cli.vcf2zarr, ["mkschema", "path"], catch_exceptions=False
+            )
+            assert result.exit_code == 0
+            assert len(result.stdout) == 0
+            assert len(result.stderr) == 0
+            # TODO figure out how to test that we call it with stdout from
+            # the CliRunner
+            # mocked.assert_called_once_with("path", stdout)
+            mocked.assert_called_once()
+
+    def test_encode(self):
+        runner = ct.CliRunner(mix_stderr=False)
+        with mock.patch("bio2zarr.vcf.encode") as mocked:
+            result = runner.invoke(
+                cli.vcf2zarr, ["encode", "if_path", "zarr_path"], catch_exceptions=False
+            )
+            assert result.exit_code == 0
+            assert len(result.stdout) == 0
+            assert len(result.stderr) == 0
+            mocked.assert_called_once_with(
+                "if_path",
+                "zarr_path",
+                None,
+                worker_processes=1,
+                show_progress=True,
+            )
+
+    def test_convert_vcf(self):
+        runner = ct.CliRunner(mix_stderr=False)
+        with mock.patch("bio2zarr.vcf.convert") as mocked:
+            result = runner.invoke(
+                cli.vcf2zarr,
+                ["convert", "vcf_path", "zarr_path"],
+                catch_exceptions=False,
+            )
+            assert result.exit_code == 0
+            assert len(result.stdout) == 0
+            assert len(result.stderr) == 0
+            mocked.assert_called_once_with(
+                ("vcf_path",),
+                "zarr_path",
+                worker_processes=1,
+                show_progress=True,
+            )
+
+    def test_validate(self):
+        runner = ct.CliRunner(mix_stderr=False)
+        with mock.patch("bio2zarr.vcf.validate") as mocked:
+            result = runner.invoke(
+                cli.vcf2zarr,
+                ["validate", "vcf_path", "zarr_path"],
+                catch_exceptions=False,
+            )
+            assert result.exit_code == 0
+            assert len(result.stdout) == 0
+            assert len(result.stderr) == 0
+            mocked.assert_called_once_with(
+                "vcf_path",
+                "zarr_path",
+                show_progress=True,
+            )
+
+    def test_convert_plink(self):
+        runner = ct.CliRunner(mix_stderr=False)
+        with mock.patch("bio2zarr.plink.convert") as mocked:
+            result = runner.invoke(
+                cli.plink2zarr, ["convert", "in", "out"], catch_exceptions=False
+            )
+            assert result.exit_code == 0
+            assert len(result.stdout) == 0
+            assert len(result.stderr) == 0
+            mocked.assert_called_once_with(
+                "in",
+                "out",
+                worker_processes=1,
+                chunk_width=None,
+                chunk_length=None,
+                show_progress=True,
+            )