Skip to content

Commit 23164e0

Browse files
brokkoli71brokkoli71
committed
v2.Array create, arrayMetadata and codecPipeline
1 parent e758d54 commit 23164e0

File tree

9 files changed

+991
-689
lines changed

9 files changed

+991
-689
lines changed

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,76 @@
11
package dev.zarr.zarrjava.v2;
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
4-
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
4+
import dev.zarr.zarrjava.ZarrException;
55
import dev.zarr.zarrjava.store.StoreHandle;
66
import dev.zarr.zarrjava.utils.Utils;
7+
import dev.zarr.zarrjava.v3.codec.CodecPipeline;
8+
import dev.zarr.zarrjava.v3.codec.Codec;
9+
import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
10+
711
import java.io.IOException;
12+
import java.nio.ByteBuffer;
813
import java.util.Arrays;
14+
import java.util.function.Function;
915
import java.util.stream.Collectors;
1016

17+
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
18+
1119
public class Array {
1220

1321
static final String ZARRAY = ".zarray";
1422
public ArrayMetadata metadata;
1523
public StoreHandle storeHandle;
24+
CodecPipeline codecPipeline;
1625

17-
Array(StoreHandle storeHandle) throws IOException {
26+
protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException {
1827
this.storeHandle = storeHandle;
28+
this.metadata = arrayMetadata;
29+
this.codecPipeline = new CodecPipeline(Utils.concatArrays(
30+
metadata.filters,
31+
new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())},
32+
metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor}
33+
), metadata.coreArrayMetadata);
34+
}
1935

20-
ObjectMapper objectMapper = new ObjectMapper();
21-
objectMapper.registerModule(new Jdk8Module());
22-
this.metadata = objectMapper.readValue(
23-
Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()),
24-
ArrayMetadata.class
36+
public static Array open(StoreHandle storeHandle) throws IOException, ZarrException {
37+
return new Array(
38+
storeHandle,
39+
makeObjectMapper()
40+
.readValue(
41+
Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()),
42+
ArrayMetadata.class
43+
)
2544
);
2645
}
2746

28-
public static Array open(StoreHandle storeHandle) throws IOException {
29-
return new Array(storeHandle);
47+
public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata)
48+
throws IOException, ZarrException {
49+
return Array.create(storeHandle, arrayMetadata, false);
50+
}
51+
52+
public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk)
53+
throws IOException, ZarrException {
54+
StoreHandle metadataHandle = storeHandle.resolve(ZARRAY);
55+
if (!existsOk && metadataHandle.exists()) {
56+
throw new RuntimeException(
57+
"Trying to create a new array in " + storeHandle + ". But " + metadataHandle
58+
+ " already exists.");
59+
}
60+
ObjectMapper objectMapper = makeObjectMapper();
61+
ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata));
62+
metadataHandle.set(metadataBytes);
63+
return new Array(storeHandle, arrayMetadata);
64+
}
65+
66+
public static Array create(StoreHandle storeHandle,
67+
Function<ArrayMetadataBuilder, ArrayMetadataBuilder> arrayMetadataBuilderMapper,
68+
boolean existsOk) throws IOException, ZarrException {
69+
return create(storeHandle,
70+
arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk);
3071
}
3172

73+
3274
@Override
3375
public String toString() {
3476
return String.format("<v2.Array {%s} (%s) %s>", storeHandle,

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,87 @@
11
package dev.zarr.zarrjava.v2;
22

3+
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonIgnore;
35
import com.fasterxml.jackson.annotation.JsonProperty;
6+
import dev.zarr.zarrjava.ZarrException;
7+
import dev.zarr.zarrjava.v3.DataType;
48
import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator;
5-
import java.util.Optional;
9+
import dev.zarr.zarrjava.v3.codec.Codec;
10+
11+
import javax.annotation.Nullable;
12+
13+
import static dev.zarr.zarrjava.v3.ArrayMetadata.parseFillValue;
14+
615

716
public class ArrayMetadata {
17+
static final int ZARR_FORMAT = 2;
818

919
@JsonProperty("zarr_format")
10-
public final int zarrFormat = 2;
20+
public final int zarrFormat = ZARR_FORMAT;
1121

1222
public long[] shape;
13-
public long[] chunks;
23+
public int[] chunks;
1424

1525
@JsonProperty("dtype")
16-
public DataType dataType;
26+
public DataTypeV2 dataTypeV2;
27+
@JsonIgnore
28+
public final DataType dataType;
29+
@JsonIgnore
30+
public final Endianness endianness;
1731

32+
@JsonProperty("order")
1833
public Order order;
1934

2035
@JsonProperty("dimension_separator")
2136
public Separator dimensionSeparator;
2237

2338
@JsonProperty("fill_value")
2439
public Object fillValue;
40+
@JsonIgnore
41+
public final Object parsedFillValue;
42+
43+
public Codec[] filters;
44+
@Nullable
45+
public Codec compressor;
46+
47+
@JsonIgnore
48+
public dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata coreArrayMetadata;
49+
50+
51+
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
52+
public ArrayMetadata(
53+
@JsonProperty(value = "zarr_format", required = true) int zarrFormat,
54+
@JsonProperty(value = "shape", required = true) long[] shape,
55+
@JsonProperty(value = "chunks", required = true) int[] chunks,
56+
@JsonProperty(value = "dtype", required = true) DataTypeV2 dataTypeV2,
57+
@JsonProperty(value = "fill_value", required = true) Object fillValue, //todo can be "null"
58+
@JsonProperty(value = "order", required = true) Order order,
59+
@Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator,
60+
@Nullable @JsonProperty(value = "filters") Codec[] filters, //todo can be "null"
61+
@Nullable @JsonProperty(value = "compressor") Codec compressor //todo can be "null"
62+
) throws ZarrException {
63+
if (zarrFormat != this.zarrFormat) {
64+
throw new ZarrException(
65+
"Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'.");
66+
}
67+
68+
this.shape = shape;
69+
this.chunks = chunks;
70+
this.dataTypeV2 = dataTypeV2;
71+
this.endianness = dataTypeV2.getEndianness();
72+
this.dataType = dataTypeV2.toV3();
73+
this.fillValue = fillValue;
74+
this.parsedFillValue = parseFillValue(fillValue, this.dataType);
75+
this.order = order;
76+
this.dimensionSeparator = dimensionSeparator;
77+
this.filters = filters;
78+
this.compressor = compressor;
79+
this.coreArrayMetadata =
80+
new dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata(shape, chunks,
81+
this.dataType,
82+
parsedFillValue
83+
);
84+
}
85+
2586

26-
public Optional<Object[]> filters;
27-
public Optional<Object> compressor;
2887
}

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package dev.zarr.zarrjava.v2;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.v3.DataType;
5+
import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator;
6+
import dev.zarr.zarrjava.v3.codec.Codec;
7+
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
8+
9+
import java.util.function.Function;
10+
11+
public class ArrayMetadataBuilder {
12+
long[] shape = null;
13+
int[] chunks = null;
14+
DataTypeV2 dataTypeV2 = null;
15+
Order order = Order.C;
16+
Separator dimensionSeparator = Separator.DOT;
17+
Object fillValue = 0;
18+
Codec[] filters = new Codec[]{};
19+
Codec compressor = null;
20+
21+
22+
protected ArrayMetadataBuilder() {
23+
}
24+
25+
protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) {
26+
ArrayMetadataBuilder builder = new ArrayMetadataBuilder();
27+
builder.shape = arrayMetadata.shape;
28+
builder.chunks = arrayMetadata.chunks;
29+
builder.dataTypeV2 = arrayMetadata.dataTypeV2;
30+
builder.order = arrayMetadata.order;
31+
builder.dimensionSeparator = arrayMetadata.dimensionSeparator;
32+
builder.fillValue = arrayMetadata.parsedFillValue;
33+
builder.filters = arrayMetadata.filters;
34+
builder.compressor = arrayMetadata.compressor;
35+
return builder;
36+
}
37+
38+
public ArrayMetadataBuilder withShape(long... shape) {
39+
this.shape = shape;
40+
return this;
41+
}
42+
43+
public ArrayMetadataBuilder withChunks(int... chunks) {
44+
this.chunks = chunks;
45+
return this;
46+
}
47+
48+
public ArrayMetadataBuilder withDataType(DataTypeV2 dataTypeV2) {
49+
this.dataTypeV2 = dataTypeV2;
50+
return this;
51+
}
52+
53+
public ArrayMetadataBuilder withDataType(DataType dataType) {
54+
this.dataTypeV2 = DataTypeV2.fromDataType(dataType);
55+
return this;
56+
}
57+
58+
public ArrayMetadataBuilder withOrder(Order order) {
59+
this.order = order;
60+
return this;
61+
}
62+
63+
public ArrayMetadataBuilder withDimensionSeparator(Separator dimensionSeparator) {
64+
this.dimensionSeparator = dimensionSeparator;
65+
return this;
66+
}
67+
68+
public ArrayMetadataBuilder withFillValue(Object fillValue) {
69+
this.fillValue = fillValue;
70+
return this;
71+
}
72+
73+
public ArrayMetadataBuilder withFilters(Codec... filters) {
74+
this.filters = filters;
75+
return this;
76+
}
77+
78+
public ArrayMetadataBuilder withFilters(Function<CodecBuilder, CodecBuilder> codecBuilder) throws ZarrException {
79+
if (dataTypeV2 == null) {
80+
throw new IllegalStateException("Please call `withDataType` first.");
81+
}
82+
CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3());
83+
this.filters = codecBuilder.apply(nestedCodecBuilder)
84+
.build();
85+
return this;
86+
}
87+
88+
public ArrayMetadataBuilder withCompressor(Codec compressor) {
89+
this.compressor = compressor;
90+
return this;
91+
}
92+
93+
public ArrayMetadata build() throws ZarrException {
94+
if (shape == null) {
95+
throw new IllegalStateException("Please call `withShape` first.");
96+
}
97+
if (chunks == null) {
98+
throw new IllegalStateException("Please call `withChunks` first.");
99+
}
100+
if (dataTypeV2 == null) {
101+
throw new IllegalStateException("Please call `withDataType` first.");
102+
}
103+
return new ArrayMetadata(
104+
2,
105+
shape,
106+
chunks,
107+
dataTypeV2,
108+
fillValue,
109+
order,
110+
dimensionSeparator,
111+
filters,
112+
compressor
113+
);
114+
}
115+
}

‎src/main/java/dev/zarr/zarrjava/v2/DataType.java‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)