Skip to content

Commit b938c8d

Browse files
brokkoli71brokkoli71
committed
Merge branch 'main' into v2-attributes
# Conflicts: # src/main/java/dev/zarr/zarrjava/v2/Array.java # src/main/java/dev/zarr/zarrjava/v3/Array.java # src/test/java/dev/zarr/zarrjava/ZarrV2Test.java # src/test/java/dev/zarr/zarrjava/ZarrV3Test.java
2 parents f459a52 + 43ebb38 commit b938c8d

29 files changed

+247
-70
lines changed

‎README.md‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
# Early preview of `zarr-java`
1+
# zarr-java
22

3-
This repository contains an early preview of a Java implementation of the Zarr specification.
4-
It is intended for collecting feedback from the community and not for use. The API is subject to changes.
5-
6-
Refer to [JZarr](https://github.com/zarr-developers/jzarr) for a stable implementation of Zarr version 2.
3+
This repository contains a Java implementation of Zarr version 2 and 3.
74

85
## Usage
96
```java
@@ -50,4 +47,4 @@ Furthermore, you will need the `l4_sample` test data:
5047
`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
5148
&& cd testdata
5249
&& unzip l4_sample.zip
53-
`
50+
`

‎src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava.core;
22

33
import com.fasterxml.jackson.annotation.JsonIgnore;
4+
import com.fasterxml.jackson.annotation.JsonInclude;
45
import com.fasterxml.jackson.annotation.JsonProperty;
56
import dev.zarr.zarrjava.ZarrException;
67
import dev.zarr.zarrjava.utils.MultiArrayUtils;
@@ -16,6 +17,7 @@ public abstract class ArrayMetadata {
1617

1718
public final long[] shape;
1819

20+
@JsonInclude(JsonInclude.Include.ALWAYS)
1921
@JsonProperty("fill_value")
2022
public final Object fillValue;
2123
@JsonIgnore

‎src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,44 @@
11
package dev.zarr.zarrjava.core.codec.core;
22

33
import com.fasterxml.jackson.annotation.JsonValue;
4+
import dev.zarr.zarrjava.ZarrException;
45
import dev.zarr.zarrjava.core.codec.ArrayBytesCodec;
56
import ucar.ma2.*;
67

78
import java.nio.ByteBuffer;
89
import java.nio.ByteOrder;
910

1011
public abstract class BytesCodec extends ArrayBytesCodec {
11-
protected abstract ByteOrder getByteOrder();
12+
protected abstract ByteOrder getByteOrder() throws ZarrException;
1213

1314
@Override
14-
public Array decode(ByteBuffer chunkBytes) {
15-
chunkBytes.order(getByteOrder());
16-
DataType dtype = arrayMetadata.dataType.getMA2DataType();
17-
int[] shape = arrayMetadata.chunkShape;
18-
19-
// Array.factory does not support boolean arrays directly from ByteBuffer
20-
if (dtype == DataType.BOOLEAN) {
21-
int size = chunkBytes.remaining();
22-
boolean[] bools = new boolean[size];
23-
for (int i = 0; i < size; i++) {
24-
bools[i] = chunkBytes.get(i) != 0;
25-
}
15+
public Array decode(ByteBuffer chunkBytes) throws ZarrException {
16+
ByteOrder order = ByteOrder.BIG_ENDIAN; // Default for 1-byte types
17+
if (arrayMetadata.dataType.getByteCount() > 1)
18+
order = getByteOrder();
19+
chunkBytes.order(order);
20+
DataType dtype = arrayMetadata.dataType.getMA2DataType();
21+
int[] shape = arrayMetadata.chunkShape;
22+
23+
// Array.factory does not support boolean arrays directly from ByteBuffer
24+
if (dtype == DataType.BOOLEAN) {
25+
int size = chunkBytes.remaining();
26+
boolean[] bools = new boolean[size];
27+
for (int i = 0; i < size; i++) {
28+
bools[i] = chunkBytes.get(i) != 0;
29+
}
2630

27-
Index index = Index.factory(shape);
28-
return Array.factory(DataType.BOOLEAN, index, bools);
31+
Index index = Index.factory(shape);
32+
return Array.factory(DataType.BOOLEAN, index, bools);
33+
}
34+
return Array.factory(dtype, shape, chunkBytes);
2935
}
3036

31-
return Array.factory(dtype, shape, chunkBytes);
32-
}
3337
@Override
34-
public ByteBuffer encode(Array chunkArray) {
35-
ByteOrder order = getByteOrder();
38+
public ByteBuffer encode(Array chunkArray) throws ZarrException {
39+
ByteOrder order = ByteOrder.BIG_ENDIAN; // Default for 1-byte types
40+
if (arrayMetadata.dataType.getByteCount() > 1)
41+
order = getByteOrder();
3642

3743
// Boolean
3844
if (chunkArray instanceof ArrayBoolean) {
@@ -94,6 +100,10 @@ public ByteOrder getByteOrder() {
94100
throw new RuntimeException("Unreachable");
95101
}
96102
}
103+
104+
public static Endian nativeOrder() {
105+
return ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? LITTLE : BIG;
106+
}
97107
}
98108

99109
}

‎src/main/java/dev/zarr/zarrjava/store/S3Store.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,7 @@
44
import software.amazon.awssdk.core.ResponseInputStream;
55
import software.amazon.awssdk.core.sync.RequestBody;
66
import software.amazon.awssdk.services.s3.S3Client;
7-
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
8-
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
9-
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
10-
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
11-
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
12-
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
13-
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
7+
import software.amazon.awssdk.services.s3.model.*;
148

159
import java.io.ByteArrayInputStream;
1610
import java.io.IOException;
@@ -57,7 +51,11 @@ ByteBuffer get(GetObjectRequest getObjectRequest) {
5751
@Override
5852
public boolean exists(String[] keys) {
5953
HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build();
60-
return s3client.headObject(req).sdkHttpResponse().statusCode() == 200;
54+
try {
55+
return s3client.headObject(req).sdkHttpResponse().statusCode() == 200;
56+
} catch (NoSuchKeyException e) {
57+
return false;
58+
}
6159
}
6260

6361
@Nullable
@@ -84,7 +82,7 @@ public ByteBuffer get(String[] keys, long start, long end) {
8482
GetObjectRequest req = GetObjectRequest.builder()
8583
.bucket(bucketName)
8684
.key(resolveKeys(keys))
87-
.range(String.valueOf(start)+"-"+String.valueOf(end))
85+
.range(start +"-"+ end)
8886
.build();
8987
return get(req);
9088
}

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava.v2;
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.fasterxml.jackson.databind.ObjectWriter;
45
import dev.zarr.zarrjava.ZarrException;
56
import dev.zarr.zarrjava.core.Attributes;
67
import dev.zarr.zarrjava.store.FilesystemStore;
@@ -19,6 +20,7 @@
1920
import java.util.function.Function;
2021
import java.util.stream.Collectors;
2122
import static dev.zarr.zarrjava.v2.Node.makeObjectMapper;
23+
import static dev.zarr.zarrjava.v2.Node.makeObjectWriter;
2224

2325
public class Array extends dev.zarr.zarrjava.core.Array implements Node {
2426

@@ -148,12 +150,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata,
148150
"Trying to create a new array in " + storeHandle + ". But " + metadataHandle
149151
+ " already exists.");
150152
}
151-
ObjectMapper objectMapper = makeObjectMapper();
152-
ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata));
153+
ObjectWriter objectWriter = makeObjectWriter();
154+
ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata));
153155
if (arrayMetadata.attributes != null) {
154156
StoreHandle attrsHandle = storeHandle.resolve(ZATTRS);
155157
ByteBuffer attrsBytes = ByteBuffer.wrap(
156-
objectMapper.writeValueAsBytes(arrayMetadata.attributes));
158+
objectWriter.writeValueAsBytes(arrayMetadata.attributes));
157159
attrsHandle.set(attrsBytes);
158160
}
159161
metadataHandle.set(metadataBytes);

‎src/main/java/dev/zarr/zarrjava/v2/Group.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava.v2;
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.fasterxml.jackson.databind.ObjectWriter;
45
import dev.zarr.zarrjava.ZarrException;
56
import dev.zarr.zarrjava.core.Attributes;
67
import dev.zarr.zarrjava.store.FilesystemStore;
@@ -15,6 +16,7 @@
1516
import java.nio.file.Paths;
1617
import java.util.function.Function;
1718
import static dev.zarr.zarrjava.v2.Node.makeObjectMapper;
19+
import static dev.zarr.zarrjava.v2.Node.makeObjectWriter;
1820

1921
public class Group extends dev.zarr.zarrjava.core.Group implements Node{
2022
public GroupMetadata metadata;
@@ -49,13 +51,13 @@ public static Group open(String path) throws IOException {
4951
public static Group create(
5052
@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata
5153
) throws IOException {
52-
ObjectMapper objectMapper = makeObjectMapper();
53-
ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata));
54+
ObjectWriter objectWriter = makeObjectWriter();
55+
ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata));
5456
storeHandle.resolve(ZGROUP).set(metadataBytes);
5557
if (groupMetadata.attributes != null) {
5658
StoreHandle attrsHandle = storeHandle.resolve(ZATTRS);
5759
ByteBuffer attrsBytes = ByteBuffer.wrap(
58-
objectMapper.writeValueAsBytes(groupMetadata.attributes));
60+
objectWriter.writeValueAsBytes(groupMetadata.attributes));
5961
attrsHandle.set(attrsBytes);
6062
}
6163
return new Group(storeHandle, groupMetadata);

‎src/main/java/dev/zarr/zarrjava/v2/Node.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package dev.zarr.zarrjava.v2;
22

3+
import com.fasterxml.jackson.annotation.JsonInclude;
34
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.databind.ObjectWriter;
46
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
57
import dev.zarr.zarrjava.ZarrException;
68
import dev.zarr.zarrjava.store.FilesystemStore;
@@ -21,7 +23,11 @@ static ObjectMapper makeObjectMapper() {
2123
return objectMapper;
2224
}
2325

24-
/**
26+
static ObjectWriter makeObjectWriter() {
27+
return makeObjectMapper().writerWithDefaultPrettyPrinter();
28+
}
29+
30+
/**
2531
* Opens an existing Zarr array or group at a specified storage location.
2632
*
2733
* @param storeHandle the storage location of the Zarr array

‎src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava.v2.codec.core;
22

33
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonIgnore;
45
import com.fasterxml.jackson.annotation.JsonProperty;
56
import com.fasterxml.jackson.core.JsonGenerator;
67
import com.fasterxml.jackson.core.JsonParser;
@@ -22,6 +23,7 @@
2223

2324
public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec {
2425

26+
@JsonIgnore
2527
public final String id = "blosc";
2628

2729
@Nonnull

‎src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
package dev.zarr.zarrjava.v2.codec.core;
22

3-
import com.fasterxml.jackson.annotation.JsonCreator;
4-
import com.fasterxml.jackson.annotation.JsonProperty;
5-
import dev.zarr.zarrjava.ZarrException;
63
import dev.zarr.zarrjava.core.ArrayMetadata;
74
import dev.zarr.zarrjava.v2.codec.Codec;
85

@@ -14,9 +11,7 @@ public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec imp
1411
@Nonnull
1512
public final Endian endian;
1613

17-
@JsonCreator
18-
public BytesCodec(
19-
@JsonProperty(value = "endian", defaultValue = "little") Endian endian) {
14+
public BytesCodec(Endian endian) {
2015
this.endian = endian;
2116
}
2217

‎src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava.v2.codec.core;
22

33
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonIgnore;
45
import com.fasterxml.jackson.annotation.JsonProperty;
56
import dev.zarr.zarrjava.ZarrException;
67
import dev.zarr.zarrjava.core.ArrayMetadata;
@@ -16,6 +17,7 @@
1617

1718
public class ZlibCodec extends BytesBytesCodec implements Codec {
1819

20+
@JsonIgnore
1921
public final String id = "zlib";
2022
public final int level;
2123

0 commit comments

Comments
 (0)