Merge pull request #85 from jeromekelleher/check-json-format-versions

Add checking for metadata format versions

Author: jeromekelleher

CHANGELOG.md

@@ -2,3 +2,4 @@
 
 - Merged 1D and 2D encode steps into one, and change rate reporting to bytes
 - Add --max-memory for encode
+- Change `chunk_width` to `samples_chunk_size` and `chunk_length` to `variants_chunk_size`

bio2zarr/vcf.py

@@ -142,6 +142,9 @@ class VcfPartition:
     num_records: int = -1
 
 
+VCF_METADATA_FORMAT_VERSION = "0.1"
+
+
 @dataclasses.dataclass
 class VcfMetadata:
     format_version: str
@@ -175,6 +178,11 @@ def num_records(self):
 
     @staticmethod
     def fromdict(d):
+        if d["format_version"] != VCF_METADATA_FORMAT_VERSION:
+            raise ValueError(
+                "Exploded metadata format version mismatch: "
+                f"{d['format_version']} != {VCF_METADATA_FORMAT_VERSION}"
+            )
         fields = [VcfField.fromdict(fd) for fd in d["fields"]]
         partitions = [VcfPartition(**pd) for pd in d["partitions"]]
         d = d.copy()
@@ -239,8 +247,7 @@ def scan_vcf(path, target_num_partitions):
             # TODO use the mapping dictionary
             fields=fields,
             partitions=[],
-            # FIXME do something systematic with this
-            format_version="0.1",
+            format_version=VCF_METADATA_FORMAT_VERSION,
         )
         try:
             metadata.contig_lengths = vcf.seqlens
@@ -1060,6 +1067,9 @@ def __post_init__(self):
         self.dimensions = tuple(self.dimensions)
 
 
+ZARR_SCHEMA_FORMAT_VERSION = "0.2"
+
+
 @dataclasses.dataclass
 class ZarrConversionSpec:
     format_version: str
@@ -1080,6 +1090,11 @@ def asjson(self):
 
     @staticmethod
     def fromdict(d):
+        if d["format_version"] != ZARR_SCHEMA_FORMAT_VERSION:
+            raise ValueError(
+                "Zarr schema format version mismatch: "
+                f"{d['format_version']} != {ZARR_SCHEMA_FORMAT_VERSION}"
+            )
         ret = ZarrConversionSpec(**d)
         ret.columns = {
             key: ZarrColumnSpec(**value) for key, value in d["columns"].items()
@@ -1241,8 +1256,7 @@ def fixed_field_spec(
             )
 
         return ZarrConversionSpec(
-            # TODO do something systematic
-            format_version="0.1",
+            format_version=ZARR_SCHEMA_FORMAT_VERSION,
             samples_chunk_size=samples_chunk_size,
             variants_chunk_size=variants_chunk_size,
             columns={col.name: col for col in colspecs},

tests/test_vcf.py

@@ -0,0 +1,55 @@
+import json
+
+import pytest
+
+
+from bio2zarr import vcf
+
+
+@pytest.fixture(scope="module")
+def vcf_file():
+    return "tests/data/vcf/sample.vcf.gz"
+
+
+@pytest.fixture(scope="module")
+def exploded_path(vcf_file, tmp_path_factory):
+    out = tmp_path_factory.mktemp("data") / "example.exploded"
+    vcf.explode([vcf_file], out)
+    return out
+
+
+@pytest.fixture(scope="module")
+def schema_path(exploded_path, tmp_path_factory):
+    out = tmp_path_factory.mktemp("data") / "example.schema.json"
+    with open(out, "w") as f:
+        vcf.mkschema(exploded_path, f)
+    return out
+
+
+@pytest.fixture(scope="module")
+def zarr_path(exploded_path, tmp_path_factory):
+    out = tmp_path_factory.mktemp("data") / "example.zarr"
+    vcf.encode(exploded_path, out)
+    return out
+
+
+class TestJsonVersions:
+    @pytest.mark.parametrize("version", ["0.1", "1.0", "xxxxx", 0.2])
+    def test_zarr_schema_mismatch(self, schema_path, version):
+        with open(schema_path) as f:
+            d = json.load(f)
+
+        d["format_version"] = version
+        with pytest.raises(ValueError, match="Zarr schema format version mismatch"):
+            vcf.ZarrConversionSpec.fromdict(d)
+
+    @pytest.mark.parametrize("version", ["0.0", "1.0", "xxxxx", 0.1])
+    def test_exploded_metadata_mismatch(self, tmpdir, exploded_path, version):
+        with open(exploded_path / "metadata.json", "r") as f:
+            d = json.load(f)
+
+        d["format_version"] = version
+        with pytest.raises(
+            ValueError, match="Exploded metadata format version mismatch"
+        ):
+            vcf.VcfMetadata.fromdict(d)