Test zarrita compatibility (#3)

* add zarrita python script

* add testReadFromZarrita

* renamed zarrita_write

* add testWriteToZarrita

* parameterized codecs in testReadFromZarrita

* fixed camelcase in zarr.json

* add venv for executing zarrita in linux

* adapt test-setup for linux

* adapt test-setup for linux

* fix testReadFromZarrita for gzip

* parameterize codecs in testReadFromZarrita

* remove unused imports

* add testCodecsWriteRead

* reformat

* remove unnecessary argument

* add testCodecTranspose

* start remove constants "C" adn "F" from Transpose Codec's order

* add CoreArrayMetadata to codec object instead of passing as argument in encode and decode
remove transpose order "F" and "C"

* remove wrong dependency

* set fail-fast: false

* specify testSourceDirectory

* added property index_location to ShardingIndexedCodec

* add indexLocation in ShardingCodec.

* change junit version for TestUtils

* remove creation of dir testdata

* update dependencies for JUnit 5

* install zarrita in CI

* correct python version, maven no-transfer-progress

* add venv for executing zarrita to CI

* add /venv_zarrita to .gitignore

* remove deprecated zarrita venv

* test CI

* fix tests for windows

* update python path for windows in ci.yml

* add Development Start-Guide to Run Tests Locally

* correct Development Start-Guide to python3.11

* add support of shardingCodec indexLocation=start

* code cleanup

* add testZstdLibrary

* fix Zstd compression and decompression

* cleanup code

* cleanup code

* remove unused method

* include crc32c codec into tests

* incorporate feedback for java version and merge tests

* add more variation to codec-configuration in tests with zarrita

* fix BloscCodec with shuffle = "shuffle"

Author: brokkoli71

.github/workflows/ci.yml

@@ -11,29 +11,41 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ ubuntu, windows, macos ]
-    runs-on: ${{ matrix.os }}-latest
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
           java-version: '22'
           distribution: 'temurin'
           cache: maven
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install zarrita
+        run: |
+          python -m venv venv_zarrita
+          if [ "${{ runner.os }}" = "Windows" ]; then venv_zarrita/Scripts/pip install zarrita; else venv_zarrita/bin/pip install zarrita; fi
+
       - name: Download blosc jar
         run: |
           mkdir -p ../blosc-java/target
           curl https://static.webknossos.org/misc/blosc-java-0.1-1.21.4-SNAPSHOT.jar -o ../blosc-java/target/blosc-java-0.1-1.21.4-SNAPSHOT.jar
 
       - name: Download testdata
         run: |
-          mkdir testdata testoutput
+          mkdir testoutput
           curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
           cd testdata
           unzip l4_sample.zip
@@ -44,7 +56,7 @@ jobs:
       - name: Test
         env:
           MAVEN_OPTS: "-Xmx6g"
-        run: mvn test -DargLine="-Xmx6g"
+        run: mvn --no-transfer-progress test -DargLine="-Xmx6g"
 
       - name: Assemble JAR
         run: mvn package -DskipTests

.gitignore

@@ -36,5 +36,6 @@ build/
 
 
 ### Custom ###
-/testdata
+/testdata/l4_sample
 /testoutput
+/venv_zarrita

README.md

@@ -38,3 +38,19 @@ array.write(
     ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024})
 );
 ```
+## Development Start-Guide
+
+### Run Tests Locally
+To be able to run the tests locally, make sure to have `python3.11` installed. 
+Also, you need to set up a venv for zarrita at the root of the project:
+`python3.11 -m venv venv_zarrita`.
+
+Then install zarrita there with `venv_zarrita/Scripts/pip install zarrita` 
+for Windows and `venv_zarrita/bin/pip install zarrita` for Linux.
+
+Furthermore, you will need the `l4_sample` test data:
+
+`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
+&& cd testdata
+&& unzip l4_sample.zip
+`

pom.xml

@@ -16,9 +16,31 @@
         <aws.version>1.12.477</aws.version>
         <netcdfJavaVersion>5.5.3</netcdfJavaVersion>
         <zstdVersion>1.5.5-5</zstdVersion>
+        <junit-jupiter-version>5.10.2</junit-jupiter-version>
     </properties>
 
     <dependencies>
+        <!-- JUnit 5 dependencies -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit-jupiter-version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit-jupiter-version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>${junit-jupiter-version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Other dependencies -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
@@ -54,6 +76,7 @@
             <artifactId>okhttp</artifactId>
             <version>2.7.5</version>
         </dependency>
+        <!-- JUnit 4 dependency for backward compatibility if needed -->
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
@@ -70,4 +93,16 @@
         </repository>
     </repositories>
 
-</project>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.2.5</version>
+                <configuration>
+                    <useSystemClassLoader>false</useSystemClassLoader>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

src/main/java/dev/zarr/zarrjava/utils/Utils.java

@@ -77,4 +77,24 @@ public static <T> T[] concatArrays(T[] array1, T[]... arrays) {
     }
     return result;
   }
+
+  public static boolean isPermutation(int[] array) {
+    if (array.length==0){
+      return false;
+    }
+    int[] arange = new int[array.length];
+    Arrays.setAll(arange, i -> i);
+    int[] orderSorted = array.clone();
+    Arrays.sort(orderSorted);
+    return Arrays.equals(orderSorted, arange);
+  }
+
+  public static int[] inversePermutation(int[] origin){
+    assert isPermutation(origin);
+    int[] inverse = new int[origin.length];
+    for (int i = 0; i < origin.length; i++) {
+      inverse[origin[i]] = i;
+    }
+    return inverse;
+  }
 }

src/main/java/dev/zarr/zarrjava/v3/Array.java

@@ -27,7 +27,7 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata)
       throws IOException, ZarrException {
     super(storeHandle);
     this.metadata = arrayMetadata;
-    this.codecPipeline = new CodecPipeline(arrayMetadata.codecs);
+    this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata);
   }
 
   /**
@@ -171,8 +171,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
 
                 if (codecPipeline.supportsPartialDecode()) {
                   final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
-                      Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape,
-                      metadata.coreArrayMetadata);
+                      Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
                   MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
                       chunkProjection.outOffset, chunkProjection.shape
                   );
@@ -223,7 +222,7 @@ public ucar.ma2.Array readChunk(long[] chunkCoords)
       return metadata.allocateFillValueChunk();
     }
 
-    return codecPipeline.decode(chunkBytes, metadata.coreArrayMetadata);
+    return codecPipeline.decode(chunkBytes);
   }
 
   /**
@@ -299,7 +298,7 @@ public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Zar
     if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue)) {
       chunkHandle.delete();
     } else {
-      ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, metadata.coreArrayMetadata);
+      ByteBuffer chunkBytes = codecPipeline.encode(chunkArray);
       chunkHandle.set(chunkBytes);
     }
   }

src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java

@@ -1,15 +1,14 @@
 package dev.zarr.zarrjava.v3.codec;
 
 import dev.zarr.zarrjava.ZarrException;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
 import ucar.ma2.Array;
 
-public interface ArrayArrayCodec extends Codec {
+public abstract class ArrayArrayCodec extends Codec {
 
-  Array encode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+  protected abstract Array encode(Array chunkArray)
       throws ZarrException;
 
-  Array decode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+  protected abstract Array decode(Array chunkArray)
       throws ZarrException;
 
 }

src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java

@@ -2,23 +2,24 @@
 
 import dev.zarr.zarrjava.ZarrException;
 import dev.zarr.zarrjava.store.StoreHandle;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
 import java.nio.ByteBuffer;
 import ucar.ma2.Array;
 
-public interface ArrayBytesCodec extends Codec {
+public abstract class ArrayBytesCodec extends Codec {
 
-  ByteBuffer encode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+  protected abstract ByteBuffer encode(Array chunkArray)
       throws ZarrException;
 
-  Array decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
+  protected abstract Array decode(ByteBuffer chunkBytes)
       throws ZarrException;
 
-  interface WithPartialDecode extends ArrayBytesCodec {
+  public abstract static class WithPartialDecode extends ArrayBytesCodec {
 
-    Array decodePartial(
-        StoreHandle handle, long[] offset, int[] shape,
-        CoreArrayMetadata arrayMetadata
+    public abstract Array decode(ByteBuffer shardBytes) throws ZarrException;
+    public abstract ByteBuffer encode(Array shardArray) throws ZarrException;
+
+    protected abstract Array decodePartial(
+        StoreHandle handle, long[] offset, int[] shape
     ) throws ZarrException;
   }
 }

src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java

@@ -1,15 +1,13 @@
 package dev.zarr.zarrjava.v3.codec;
 
 import dev.zarr.zarrjava.ZarrException;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
+
 import java.nio.ByteBuffer;
 
-public interface BytesBytesCodec extends Codec {
+public abstract class BytesBytesCodec extends Codec {
 
-  ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
-      throws ZarrException;
+    protected abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException;
 
-  ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
-      throws ZarrException;
+    public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException;
 
 }

src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java

@@ -5,9 +5,22 @@
 import dev.zarr.zarrjava.v3.ArrayMetadata;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name")
-public interface Codec {
+public abstract class Codec {
 
-  long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata)
-      throws ZarrException;
+    protected ArrayMetadata.CoreArrayMetadata arrayMetadata;
+
+    protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException {
+        if (arrayMetadata == null) {
+            throw new ZarrException("arrayMetadata needs to get set in for every codec");
+        }
+        return this.arrayMetadata;
+    }
+
+    protected abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+            throws ZarrException;
+
+    public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{
+        this.arrayMetadata = arrayMetadata;
+    }
 }