Merge pull request #43 from jeromekelleher/provenance-and-metadata

Provenance and metadata

Author: jeromekelleher

bio2zarr/__init__.py

@@ -1,2 +1 @@
-
-from .vcf import explode as explode_vcf
+from . provenance import __version__

bio2zarr/__main__.py

@@ -2,17 +2,18 @@
 
 from . import cli
 
+@cli.version
 @click.group()
-def top_level():
+def bio2zarr():
     pass
 
 # Provide a single top-level interface to all of the functionality.
 # This probably isn't the recommended way of interacting, as we
 # install individual commands as console scripts. However, this
 # is handy for development and for those whose PATHs aren't set
 # up in the right way.
-top_level.add_command(cli.vcf2zarr)
-top_level.add_command(cli.plink2zarr)
+bio2zarr.add_command(cli.vcf2zarr)
+bio2zarr.add_command(cli.plink2zarr)
 
 if __name__ == "__main__":
-    top_level()
+    bio2zarr()

bio2zarr/cli.py

@@ -4,6 +4,7 @@
 
 from . import vcf
 from . import plink
+from . import provenance
 
 # Common arguments/options
 verbose = click.option("-v", "--verbose", count=True, help="Increase verbosity")
@@ -12,6 +13,7 @@
     "-p", "--worker-processes", type=int, default=1, help="Number of worker processes"
 )
 
+version = click.version_option(version=provenance.__version__)
 
 # Note: logging hasn't been implemented in the code at all, this is just
 # a first pass to try out some ways of doing things to see what works.
@@ -111,6 +113,7 @@ def validate(vcfs, out_path):
     vcf.validate(vcfs[0], out_path, show_progress=True)
 
 
+@version
 @click.group()
 def vcf2zarr():
     pass
@@ -145,6 +148,7 @@ def convert_plink(in_path, out_path, worker_processes, chunk_width, chunk_length
     )
 
 
+@version
 @click.group()
 def plink2zarr():
     pass

bio2zarr/provenance.py

@@ -0,0 +1,7 @@
+__version__ = "undefined"
+try:
+    from . import _version
+
+    __version__ = _version.version
+except ImportError:  # pragma: nocover
+    pass

bio2zarr/vcf.py

@@ -23,6 +23,7 @@
 import zarr
 
 from . import core
+from . import provenance
 
 logger = logging.getLogger(__name__)
 
@@ -178,6 +179,7 @@ def make_field_def(name, vcf_type, vcf_number):
 def scan_vcfs(paths, show_progress):
     partitions = []
     vcf_metadata = None
+    header = None
     logger.info(f"Scanning {len(paths)} VCFs")
     for path in tqdm.tqdm(paths, desc="Scan ", disable=not show_progress):
         vcf = cyvcf2.VCF(path)
@@ -215,6 +217,9 @@ def scan_vcfs(paths, show_progress):
 
         if vcf_metadata is None:
             vcf_metadata = metadata
+            # We just take the first header, assuming the others
+            # are compatible.
+            header = vcf.raw_header
         else:
             if metadata != vcf_metadata:
                 raise ValueError("Incompatible VCF chunks")
@@ -230,7 +235,7 @@ def scan_vcfs(paths, show_progress):
         )
     partitions.sort(key=lambda x: x.first_position)
     vcf_metadata.partitions = partitions
-    return vcf_metadata
+    return vcf_metadata, header
 
 
 def sanitise_value_bool(buff, j, value):
@@ -668,9 +673,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
 
 class PickleChunkedVcf(collections.abc.Mapping):
-    def __init__(self, path, metadata):
+    def __init__(self, path, metadata, vcf_header):
         self.path = path
         self.metadata = metadata
+        self.vcf_header = vcf_header
 
         self.columns = {}
         for field in self.metadata.fields:
@@ -753,7 +759,9 @@ def load(path):
         path = pathlib.Path(path)
         with open(path / "metadata.json") as f:
             metadata = VcfMetadata.fromdict(json.load(f))
-        return PickleChunkedVcf(path, metadata)
+        with open(path / "header.txt") as f:
+            header = f.read()
+        return PickleChunkedVcf(path, metadata, header)
 
     @staticmethod
     def convert_partition(
@@ -820,8 +828,8 @@ def convert(
     ):
         out_path = pathlib.Path(out_path)
         # TODO make scan work in parallel using general progress code too
-        vcf_metadata = scan_vcfs(vcfs, show_progress=show_progress)
-        pcvcf = PickleChunkedVcf(out_path, vcf_metadata)
+        vcf_metadata, header = scan_vcfs(vcfs, show_progress=show_progress)
+        pcvcf = PickleChunkedVcf(out_path, vcf_metadata, header)
         pcvcf.mkdirs()
 
         total_variants = sum(
@@ -855,6 +863,8 @@ def convert(
 
         with open(out_path / "metadata.json", "w") as f:
             json.dump(vcf_metadata.asdict(), f, indent=4)
+        with open(out_path / "header.txt", "w") as f:
+            f.write(header)
         return pcvcf
 
 
@@ -1214,7 +1224,6 @@ def encode_contig(self, pcvcf, contig_names, contig_lengths):
         logger.debug("Contig done")
 
     def encode_filters(self, pcvcf, filter_names):
-        self.root.attrs["filters"] = filter_names
         array = self.root.array(
             "filter_id",
             filter_names,
@@ -1277,6 +1286,10 @@ def convert(
         for column in conversion_spec.columns.values():
             sgvcf.create_array(column)
 
+        sgvcf.root.attrs["vcf_zarr_version"] = "0.2"
+        sgvcf.root.attrs["vcf_header"] = pcvcf.vcf_header
+        sgvcf.root.attrs["source"] = f"bio2zarr-{provenance.__version__}"
+
         progress_config = core.ProgressConfig(
             total=pcvcf.total_uncompressed_bytes,
             title="Encode",

setup.py

@@ -5,5 +5,5 @@
     # The package name along with all the other metadata is specified in setup.cfg
     # However, GitHub's dependency graph can't see the package unless we put this here.
     name="bio2zarr",
-    use_scm_version=True,
+    use_scm_version={"write_to": "bio2zarr/_version.py"},
 )

tests/test_cli.py

@@ -1,8 +1,11 @@
 from unittest import mock
 
+import pytest
 import click.testing as ct
 
 from bio2zarr import cli
+from bio2zarr import __main__ as main
+from bio2zarr import provenance
 
 
 class TestWithMocks:
@@ -117,3 +120,11 @@ def test_convert_plink(self):
                 chunk_length=None,
                 show_progress=True,
             )
+
+
+@pytest.mark.parametrize("cmd", [main.bio2zarr, cli.vcf2zarr, cli.plink2zarr])
+def test_version(cmd):
+    runner = ct.CliRunner(mix_stderr=False)
+    result = runner.invoke(cmd, ["--version"], catch_exceptions=False)
+    s = f"version {provenance.__version__}\n"
+    assert result.stdout.endswith(s)

tests/test_vcf_examples.py

@@ -3,8 +3,10 @@
 import xarray.testing as xt
 import pytest
 import sgkit as sg
+import cyvcf2
 
 from bio2zarr import vcf
+from bio2zarr import provenance
 
 
 class TestSmallExample:
@@ -33,6 +35,13 @@ def test_filters(self, ds):
             ],
         )
 
+    def test_header(self, ds):
+        vcf = cyvcf2.VCF(self.data_path)
+        assert ds.attrs["vcf_header"] == vcf.raw_header
+
+    def test_source(self, ds):
+        assert ds.attrs["source"] == f"bio2zarr-{provenance.__version__}"
+
     def test_contigs(self, ds):
         nt.assert_array_equal(ds["contig_id"], ["19", "20", "X"])
         assert "contig_length" not in ds