Skip to content

Commit b93de23

Browse files
brokkoli71brokkoli71
committed
zarr-python tests for v2, v3
1 parent a8c6347 commit b93de23

File tree

13 files changed

+487
-246
lines changed

13 files changed

+487
-246
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ jobs:
3131
- name: Install uv
3232
uses: astral-sh/setup-uv@v6
3333

34-
- name: Set up zarrita
34+
- name: Set up zarr-python
3535
run: |
3636
uv venv && uv init
37-
uv add zarrita
37+
uv add zarr
3838
3939
- name: Download testdata
4040
run: |

‎src/main/java/dev/zarr/zarrjava/store/StoreHandle.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import dev.zarr.zarrjava.utils.Utils;
44
import java.nio.ByteBuffer;
55
import java.nio.file.NoSuchFileException;
6+
import java.nio.file.Path;
67
import java.util.stream.Stream;
78
import javax.annotation.Nonnull;
89
import javax.annotation.Nullable;
@@ -70,4 +71,11 @@ public String toString() {
7071
public StoreHandle resolve(String... subKeys) {
7172
return new StoreHandle(store, Utils.concatArrays(keys, subKeys));
7273
}
74+
75+
public Path toPath() {
76+
if (!(store instanceof FilesystemStore)) {
77+
throw new UnsupportedOperationException("The underlying store is not a filesystem store.");
78+
}
79+
return ((FilesystemStore) store).resolveKeys(keys);
80+
}
7381
}

src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java renamed to src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,32 @@
1-
package dev.zarr.zarrjava.v3.codec.core;
1+
package dev.zarr.zarrjava.v2.codec.core;
22

33
import com.fasterxml.jackson.annotation.JsonCreator;
44
import com.fasterxml.jackson.annotation.JsonProperty;
55
import dev.zarr.zarrjava.ZarrException;
66
import dev.zarr.zarrjava.utils.Utils;
7+
import dev.zarr.zarrjava.v2.codec.Codec;
78
import dev.zarr.zarrjava.v3.ArrayMetadata;
8-
import dev.zarr.zarrjava.v3.codec.BytesBytesCodec;
9+
import dev.zarr.zarrjava.codec.BytesBytesCodec;
910

1011
import java.io.ByteArrayInputStream;
1112
import java.io.ByteArrayOutputStream;
1213
import java.io.IOException;
1314
import java.nio.ByteBuffer;
1415
import java.util.zip.*;
15-
import javax.annotation.Nonnull;
1616

17-
public class ZlibCodec extends BytesBytesCodec {
17+
public class ZlibCodec extends Codec implements BytesBytesCodec {
18+
19+
public final String id = "zlib";
20+
public final int level;
1821

19-
public final String name = "zlib";
20-
@Nonnull
21-
public final Configuration configuration;
2222

2323
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
2424
public ZlibCodec(
25-
@Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) {
26-
this.configuration = configuration;
25+
@JsonProperty(value = "level", defaultValue = "1") int level) throws ZarrException {
26+
if (level < 0 || level > 9) {
27+
throw new ZarrException("'level' needs to be between 0 and 9.");
28+
}
29+
this.level = level;
2730
}
2831

2932

@@ -42,7 +45,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException {
4245
@Override
4346
public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException {
4447
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
45-
DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.configuration.level))) {
48+
DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.level))) {
4649
dos.write(Utils.toArray(chunkBytes));
4750
dos.close();
4851
return ByteBuffer.wrap(outputStream.toByteArray());
@@ -56,19 +59,4 @@ public long computeEncodedSize(long inputByteLength,
5659
ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
5760
throw new ZarrException("Not implemented for Zlib codec.");
5861
}
59-
60-
61-
public static final class Configuration {
62-
63-
public final int level;
64-
65-
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
66-
public Configuration(@JsonProperty(value = "level", defaultValue = "1") int level)
67-
throws ZarrException {
68-
if (level < 0 || level > 9) {
69-
throw new ZarrException("'level' needs to be between 0 and 9.");
70-
}
71-
this.level = level;
72-
}
73-
}
7462
}

‎src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
import com.github.luben.zstd.Zstd;
66
import com.github.luben.zstd.ZstdCompressCtx;
77
import dev.zarr.zarrjava.ZarrException;
8+
import dev.zarr.zarrjava.v3.codec.Codec;
89
import dev.zarr.zarrjava.v3.ArrayMetadata;
9-
import dev.zarr.zarrjava.v3.codec.BytesBytesCodec;
10+
import dev.zarr.zarrjava.codec.BytesBytesCodec;
1011

1112
import javax.annotation.Nonnull;
1213
import java.nio.ByteBuffer;
1314

14-
public class ZstdCodec extends BytesBytesCodec {
15+
public class ZstdCodec extends Codec implements BytesBytesCodec {
1516

1617
public final String name = "zstd";
1718
@Nonnull

‎src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
package dev.zarr.zarrjava;
2+
3+
import dev.zarr.zarrjava.store.FilesystemStore;
4+
import dev.zarr.zarrjava.store.StoreHandle;
5+
import dev.zarr.zarrjava.v3.Array;
6+
import dev.zarr.zarrjava.v3.ArrayMetadataBuilder;
7+
import dev.zarr.zarrjava.v3.DataType;
8+
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
9+
import org.junit.jupiter.api.Assertions;
10+
import org.junit.jupiter.api.BeforeAll;
11+
import org.junit.jupiter.params.ParameterizedTest;
12+
import org.junit.jupiter.params.provider.CsvSource;
13+
14+
import java.io.BufferedReader;
15+
import java.io.File;
16+
import java.io.IOException;
17+
import java.io.InputStreamReader;
18+
import java.nio.file.Files;
19+
import java.nio.file.Path;
20+
import java.nio.file.Paths;
21+
import java.util.Arrays;
22+
import java.util.Comparator;
23+
import java.util.HashMap;
24+
import java.util.Map;
25+
import java.util.stream.Stream;
26+
27+
public class ZarrPythonTests {
28+
29+
final static Path TESTOUTPUT = Paths.get("testoutput");
30+
final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/");
31+
32+
@BeforeAll
33+
public static void clearTestoutputFolder() throws IOException {
34+
if (Files.exists(TESTOUTPUT)) {
35+
try (Stream<Path> walk = Files.walk(TESTOUTPUT)) {
36+
walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
37+
}
38+
}
39+
Files.createDirectory(TESTOUTPUT);
40+
}
41+
42+
public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException {
43+
ProcessBuilder pb = new ProcessBuilder();
44+
pb.command().add("uv");
45+
pb.command().add("run");
46+
pb.command().add(PYTHON_TEST_PATH.resolve(scriptName).toString());
47+
pb.command().addAll(Arrays.asList(args));
48+
Process process = pb.start();
49+
50+
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
51+
String line;
52+
while ((line = reader.readLine()) != null) {
53+
System.out.println(line);
54+
}
55+
56+
BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream()));
57+
while ((line = readerErr.readLine()) != null) {
58+
System.err.println(line);
59+
}
60+
61+
int exitCode = process.waitFor();
62+
assert exitCode == 0;
63+
}
64+
65+
@ParameterizedTest
66+
@CsvSource({
67+
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
68+
"gzip,0", "gzip,5",
69+
"zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
70+
"bytes,BIG", "bytes,LITTLE",
71+
"transpose,_",
72+
"sharding,start", "sharding,end",
73+
"sharding_nested,_",
74+
"crc32c,_",
75+
})
76+
public void testReadFromZarrPythonV3(String codec, String codecParam) throws IOException, ZarrException, InterruptedException {
77+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python", codec, codecParam);
78+
run_python_script("zarr_python_write.py", codec, codecParam, storeHandle.toPath().toString());
79+
Array array = Array.open(storeHandle);
80+
ucar.ma2.Array result = array.read();
81+
82+
//for expected values see zarr_python_write.py
83+
Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape());
84+
Assertions.assertEquals(DataType.INT32, array.metadata.dataType);
85+
Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape());
86+
Assertions.assertEquals(42, array.metadata.attributes.get("answer"));
87+
88+
int[] expectedData = new int[16 * 16 * 16];
89+
Arrays.setAll(expectedData, p -> p);
90+
Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT));
91+
}
92+
93+
@ParameterizedTest
94+
@CsvSource({
95+
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
96+
"gzip,0", "gzip,5",
97+
"zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
98+
"bytes,BIG", "bytes,LITTLE",
99+
"transpose,_",
100+
"sharding,start", "sharding,end",
101+
"sharding_nested,_",
102+
"crc32c,_",
103+
})
104+
public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throws Exception {
105+
int[] testData = new int[16 * 16 * 16];
106+
Arrays.setAll(testData, p -> p);
107+
108+
Map<String, Object> attributes = new HashMap<>();
109+
attributes.put("test_key", "test_value");
110+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python", codec, codecParam);
111+
112+
ArrayMetadataBuilder builder = Array.metadataBuilder()
113+
.withShape(16, 16, 16)
114+
.withDataType(DataType.UINT32)
115+
.withChunkShape(2, 4, 8)
116+
.withFillValue(0)
117+
.withAttributes(attributes);
118+
119+
switch (codec) {
120+
case "blosc":
121+
String cname = codecParam.split("_")[0];
122+
String shuffle = codecParam.split("_")[1];
123+
int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]);
124+
builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc));
125+
break;
126+
case "gzip":
127+
builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam)));
128+
break;
129+
case "zstd":
130+
int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]);
131+
boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]);
132+
builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum));
133+
break;
134+
case "bytes":
135+
builder = builder.withCodecs(c -> c.withBytes(codecParam));
136+
break;
137+
case "transpose":
138+
builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2}));
139+
break;
140+
case "sharding":
141+
builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam));
142+
break;
143+
case "sharding_nested":
144+
builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE"))));
145+
break;
146+
case "crc32c":
147+
builder = builder.withCodecs(CodecBuilder::withCrc32c);
148+
break;
149+
default:
150+
throw new IllegalArgumentException("Invalid Codec: " + codec);
151+
}
152+
153+
Array writeArray = Array.create(storeHandle, builder.build());
154+
writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData));
155+
156+
//read in zarr-java
157+
Array readArray = Array.open(storeHandle);
158+
ucar.ma2.Array result = readArray.read();
159+
160+
Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape());
161+
Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType);
162+
Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape());
163+
Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key"));
164+
165+
Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
166+
167+
//read in zarr_python
168+
run_python_script("zarr_python_read.py", codec, codecParam, storeHandle.toPath().toString());
169+
}
170+
171+
172+
@ParameterizedTest
173+
@CsvSource({
174+
"zlib,0", "zlib,5",
175+
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
176+
})
177+
public void testReadFromZarrPythonV2(String compressor, String compressorParam) throws IOException, ZarrException, InterruptedException {
178+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python_v2", compressor, compressorParam);
179+
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, storeHandle.toPath().toString());
180+
181+
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle);
182+
ucar.ma2.Array result = array.read();
183+
184+
//for expected values see zarr_python_write.py
185+
Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape());
186+
Assertions.assertEquals(DataType.INT32, array.metadata.dataType);
187+
Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape());
188+
// Assertions.assertEquals(42, array.metadata.attributes.get("answer"));
189+
190+
int[] expectedData = new int[16 * 16 * 16];
191+
Arrays.setAll(expectedData, p -> p);
192+
Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT));
193+
}
194+
195+
196+
@ParameterizedTest
197+
@CsvSource({
198+
"zlib,0", "zlib,5",
199+
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
200+
})
201+
public void testWriteReadWithZarrPythonV2(String compressor, String compressorParam) throws Exception {
202+
int[] testData = new int[16 * 16 * 16];
203+
Arrays.setAll(testData, p -> p);
204+
205+
// Map<String, Object> attributes = new HashMap<>();
206+
// attributes.put("test_key", "test_value");
207+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python_v2", compressor, compressorParam);
208+
209+
dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder()
210+
.withShape(16, 16, 16)
211+
.withDataType(DataType.UINT32)
212+
.withChunks(2, 4, 8)
213+
// .withAttributes(attributes)
214+
.withFillValue(0);
215+
216+
switch (compressor) {
217+
case "blosc":
218+
String cname = compressorParam.split("_")[0];
219+
String shuffle = compressorParam.split("_")[1];
220+
int clevel_blosc = Integer.parseInt(compressorParam.split("_")[2]);
221+
builder = builder.withBloscCompressor(cname, shuffle, clevel_blosc);
222+
break;
223+
case "zlib":
224+
builder = builder.withZlibCompressor(Integer.parseInt(compressorParam));
225+
break;
226+
default:
227+
throw new IllegalArgumentException("Invalid compressor: " + compressor);
228+
}
229+
230+
dev.zarr.zarrjava.v2.Array writeArray = dev.zarr.zarrjava.v2.Array.create(storeHandle, builder.build());
231+
writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData));
232+
233+
//read in zarr-java
234+
dev.zarr.zarrjava.v2.Array readArray = dev.zarr.zarrjava.v2.Array.open(storeHandle);
235+
ucar.ma2.Array result = readArray.read();
236+
237+
Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape());
238+
Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType);
239+
Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape());
240+
// Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key"));
241+
242+
Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
243+
244+
//read in zarr_python
245+
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, storeHandle.toPath().toString());
246+
}
247+
}

0 commit comments

Comments
 (0)