detect typesize for blosc with evolve_from_core_array_metadata

Author: brokkoli71

src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java

@@ -65,13 +65,19 @@ public ArrayMetadata(
     this.endianness = dataType.getEndianness();
     this.order = order;
     this.dimensionSeparator = dimensionSeparator;
-    this.filters = filters;
-    this.compressor = compressor;
     this.coreArrayMetadata =
-        new ArrayMetadata.CoreArrayMetadata(shape, chunks,
+        new CoreArrayMetadata(shape, chunks,
             this.dataType,
             this.parsedFillValue
         );
+    if (filters == null) this.filters = null;
+    else{
+      this.filters = new Codec[filters.length];
+      for(int i = 0; i < filters.length; i++) {
+        this.filters[i] = filters[i].evolve_from_core_array_metadata(this.coreArrayMetadata);
+      }
+    }
+    this.compressor = compressor == null ? null : compressor.evolve_from_core_array_metadata(this.coreArrayMetadata);
   }
 
 
@@ -90,7 +96,7 @@ public DataType dataType() {
 
   @Override
   public Array allocateFillValueChunk() {
-      ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunks);
+      Array outputArray = Array.factory(dataType.getMA2DataType(), chunks);
       if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue);
       return outputArray;
   }

src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java

@@ -1,7 +1,11 @@
 package dev.zarr.zarrjava.v2.codec;
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import dev.zarr.zarrjava.ZarrException;
+import dev.zarr.zarrjava.v2.ArrayMetadata;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id")
-public interface Codec extends dev.zarr.zarrjava.core.codec.Codec {}
+public interface Codec extends dev.zarr.zarrjava.core.codec.Codec {
+    Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException;
+}
 

src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java

@@ -12,6 +12,7 @@
 import com.fasterxml.jackson.databind.ser.std.StdSerializer;
 import com.scalableminds.bloscjava.Blosc;
 import dev.zarr.zarrjava.ZarrException;
+import dev.zarr.zarrjava.core.ArrayMetadata;
 import dev.zarr.zarrjava.utils.Utils;
 import dev.zarr.zarrjava.v2.codec.Codec;
 
@@ -44,9 +45,6 @@ public BloscCodec(
       @JsonProperty(value = "typesize", defaultValue = "0") int typesize,
       @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize
   ) throws ZarrException {
-    if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) {
-      typesize = 4; //todo: in v2 typesize is not a required parameter. default to correct value based on dtype
-    }
     if (clevel < 0 || clevel > 9) {
       throw new ZarrException("'clevel' needs to be between 0 and 9.");
     }
@@ -57,7 +55,6 @@ public BloscCodec(
     this.blocksize = blocksize;
   }
 
-
   @Override
   public ByteBuffer encode(ByteBuffer chunkBytes)
       throws ZarrException {
@@ -72,6 +69,20 @@ public ByteBuffer encode(ByteBuffer chunkBytes)
     }
   }
 
+  @Override
+  public BloscCodec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
+    if (typesize == 0) {
+        return new BloscCodec(
+            this.cname,
+            this.shuffle,
+            this.clevel,
+            arrayMetadata.dataType.getByteCount(),
+            this.blocksize
+        );
+    }
+    return this;
+  }
+
   public static final class CustomShuffleSerializer extends StdSerializer<Blosc.Shuffle> {
 
     public CustomShuffleSerializer() {

src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java

@@ -2,6 +2,8 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import dev.zarr.zarrjava.ZarrException;
+import dev.zarr.zarrjava.core.ArrayMetadata;
 import dev.zarr.zarrjava.v2.codec.Codec;
 
 import javax.annotation.Nonnull;
@@ -22,5 +24,10 @@ public BytesCodec(
     protected ByteOrder getByteOrder() {
         return endian.getByteOrder();
     }
+
+    @Override
+    public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
+        return this;
+    }
 }
 

src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import dev.zarr.zarrjava.ZarrException;
+import dev.zarr.zarrjava.core.ArrayMetadata;
 import dev.zarr.zarrjava.utils.Utils;
 import dev.zarr.zarrjava.v2.codec.Codec;
 import dev.zarr.zarrjava.core.codec.BytesBytesCodec;
@@ -52,4 +53,9 @@ public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException {
             throw new ZarrException("Error in encoding zlib.", ex);
         }
     }
+
+    @Override
+    public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
+        return this;
+    }
 }

src/main/java/dev/zarr/zarrjava/v3/Array.java

@@ -193,7 +193,6 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException
    * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or
    * deleted. This method returns a new instance of the Zarr array class and the old instance
    * becomes invalid.
-   * TODO: test
    *
    * @param newShape the new shape of the Zarr array
    * @throws ZarrException if the new metadata is invalid
@@ -232,7 +231,6 @@ public Array setAttributes(Map<String, Object> newAttributes) throws ZarrExcepti
    * attributes as input and needs to return the new set of attributes. The attributes in the
    * callback may be mutated. This method overwrites and removes any existing attributes. This
    * method returns a new instance of the Zarr array class and the old instance becomes invalid.
-   * TODO: test
    *
    * @param attributeMapper the callback that is used to construct the new attributes
    * @throws ZarrException throws ZarrException if the new metadata is invalid

src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java

@@ -113,18 +113,19 @@ public ArrayMetadata(
         shardingCodec = getShardingIndexedCodec(shardingConfig.codecs);
       }
     }
-    this.dataType = dataType;
     this.chunkGrid = chunkGrid;
+    this.dataType = dataType;
+    this.coreArrayMetadata =
+        new CoreArrayMetadata(this.shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape,
+            this.dataType,
+            this.parsedFillValue
+        );
+
     this.chunkKeyEncoding = chunkKeyEncoding;
     this.codecs = codecs;
     this.dimensionNames = dimensionNames;
     this.attributes = attributes;
     this.storageTransformers = storageTransformers;
-    this.coreArrayMetadata =
-        new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape,
-            this.dataType,
-            this.parsedFillValue
-        );
   }
 
 

src/test/java/dev/zarr/zarrjava/ZarrV2Test.java

@@ -22,7 +22,7 @@
 public class ZarrV2Test extends ZarrTest {
     @ParameterizedTest
     @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"})
-    public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException {
+    public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException {
         Array array = Array.create(
             new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel),
             Array.metadataBuilder()
@@ -40,6 +40,20 @@ public void testV2createBlosc(String cname, String shuffle, int clevel) throws I
         Assertions.assertEquals(0, outArray.getByte(0));
     }
 
+
+
+    @ParameterizedTest
+    @CsvSource({
+        "BOOL", "FLOAT64"
+    })
+    public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrException {
+        String arrayname = dt == DataType.BOOL ? "bool" : "double";
+        StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname);
+        Array array = Array.open(storeHandle);
+        ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5});
+        Assertions.assertEquals(dt, array.metadata.dataType);
+    }
+
     @Test
     public void testCreate() throws IOException, ZarrException {
         DataType dataType = DataType.UINT32;

src/test/java/dev/zarr/zarrjava/ZarrV3Test.java

@@ -527,7 +527,7 @@ public void testGroup() throws IOException, ZarrException {
         Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape());
     }
 
-        @Test
+    @Test
     public void testCreateArray() throws ZarrException, IOException {
         StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3");
         Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path");

testdata/v2_sample/double/.zarray

@@ -0,0 +1,25 @@
+{
+  "shape": [
+    16,
+    16,
+    16
+  ],
+  "chunks": [
+    2,
+    4,
+    8
+  ],
+  "dtype": "<f8",
+  "fill_value": 0.0,
+  "order": "C",
+  "filters": null,
+  "dimension_separator": ".",
+  "compressor": {
+    "id": "blosc",
+    "cname": "blosclz",
+    "clevel": 3,
+    "shuffle": 1,
+    "blocksize": 0
+  },
+  "zarr_format": 2
+}