Skip to content

Commit 9e18db5

Browse files
normanrznormanrz
committed
codec builder and mutability methods
1 parent 490e250 commit 9e18db5

File tree

10 files changed

+231
-22
lines changed

10 files changed

+231
-22
lines changed

‎README.md‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ Java implementation of the Zarr Specification
66
import com.scalableminds.zarrjava.store.FilesystemStore;
77
import com.scalableminds.zarrjava.store.HttpStore;
88
import com.scalableminds.zarrjava.v3.Array;
9-
import com.scalableminds.zarrjava.v3.ArrayMetadata;
9+
import com.scalableminds.zarrjava.v3.DataType;
1010
import com.scalableminds.zarrjava.v3.Group;
1111

1212
Group hierarchy = Group.open(
@@ -21,12 +21,13 @@ ucar.ma2.Array outArray = array.read(
2121

2222
Array array = Array.create(
2323
new FilesystemStore("/path/to/zarr").resolve("array"),
24-
ArrayMetadata
25-
.builder()
24+
Array.metadataBuilder()
2625
.withShape(1, 4096, 4096, 1536)
27-
.withDataType("uint32")
28-
.withChunkShape(1, 128, 128, 182)
29-
.build()
26+
.withDataType(DataType.UINT32)
27+
.withChunkShape(1, 1024, 1024, 1024)
28+
.withFillValue(0)
29+
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc(DataType.UINT32)))
30+
.build();
3031
);
3132
array.write(
3233
new long[]{0, 0, 0, 0},

‎TODO‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,10 @@
77
+ Write
88
- Partial write
99
- List iterator
10-
- Codec Builder
10+
+ Codec Builder
11+
+ Mutability (e.g. editing attributes, resizing)
12+
- Tests and docs
13+
- Windows support (e.g. file paths)?
14+
- CI
15+
- Caching hooks
16+
- Zarr v2 support

‎src/main/java/com/scalableminds/zarrjava/v3/Array.java‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import java.io.IOException;
1414
import java.nio.ByteBuffer;
1515
import java.util.Arrays;
16+
import java.util.Map;
17+
import java.util.function.Function;
1618
import java.util.stream.Collectors;
1719

1820
public class Array extends Node {
@@ -126,8 +128,7 @@ public void write(long[] offset, ucar.ma2.Array array) {
126128

127129
ucar.ma2.Array chunkArray;
128130
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape, chunkShape)) {
129-
chunkArray = array.sectionNoReduce(chunkProjection.outOffset, chunkProjection.shape,
130-
null);
131+
chunkArray = array.sectionNoReduce(chunkProjection.outOffset, chunkProjection.shape, null);
131132
} else {
132133
chunkArray = readChunk(chunkCoords);
133134
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
@@ -152,11 +153,44 @@ public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Zar
152153
}
153154
}
154155

156+
private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException {
157+
ObjectMapper objectMapper = Node.makeObjectMapper();
158+
ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata));
159+
storeHandle.resolve(ZARR_JSON).set(metadataBytes);
160+
return new Array(storeHandle, newArrayMetadata);
161+
}
162+
163+
public Array resize(long[] newShape) throws ZarrException, IOException {
164+
if (newShape.length != metadata.ndim()) {
165+
throw new IllegalArgumentException("'newShape' needs to have rank '" + metadata.ndim() + "'.");
166+
}
167+
168+
ArrayMetadata newArrayMetadata =
169+
new ArrayMetadata(newShape, metadata.dataType, metadata.chunkGrid, metadata.chunkKeyEncoding,
170+
metadata.parsedFillValue, metadata.codecs, metadata.dimensionNames, metadata.attributes);
171+
return writeMetadata(newArrayMetadata);
172+
}
173+
174+
public Array setAttributes(Map<String, Object> newAttributes) throws ZarrException, IOException {
175+
ArrayMetadata newArrayMetadata =
176+
new ArrayMetadata(metadata.shape, metadata.dataType, metadata.chunkGrid, metadata.chunkKeyEncoding,
177+
metadata.parsedFillValue, metadata.codecs, metadata.dimensionNames, newAttributes);
178+
return writeMetadata(newArrayMetadata);
179+
}
180+
181+
public Array updateAttributes(Function<Map<String, Object>, Map<String, Object>> attributeMapper) throws ZarrException,
182+
IOException {
183+
return setAttributes(attributeMapper.apply(metadata.attributes));
184+
}
155185

156186
@Override
157187
public String toString() {
158188
return String.format("<v3.Array {%s} (%s) %s>", storeHandle,
159189
Arrays.stream(metadata.shape).mapToObj(Long::toString).collect(Collectors.joining(", ")),
160190
metadata.dataType);
161191
}
192+
193+
public static ArrayMetadata.Builder metadataBuilder() {
194+
return ArrayMetadata.builder();
195+
}
162196
}

‎src/main/java/com/scalableminds/zarrjava/v3/ArrayMetadata.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,25 @@
1313
import com.scalableminds.zarrjava.v3.chunkkeyencoding.Separator;
1414
import com.scalableminds.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding;
1515
import com.scalableminds.zarrjava.v3.codec.Codec;
16+
import com.scalableminds.zarrjava.v3.codec.CodecBuilder;
1617

1718
import javax.annotation.Nonnull;
1819
import javax.annotation.Nullable;
1920
import java.nio.ByteBuffer;
2021
import java.util.Arrays;
2122
import java.util.HashMap;
2223
import java.util.Map;
24+
import java.util.function.Function;
2325

2426

2527
public final class ArrayMetadata {
28+
static final String NODE_TYPE = "array";
29+
static final int ZARR_FORMAT = 3;
30+
2631
@JsonProperty("zarr_format")
27-
public final int zarrFormat = 3;
32+
public final int zarrFormat = ZARR_FORMAT;
2833
@JsonProperty("node_type")
29-
public final String nodeType = "array";
34+
public final String nodeType = NODE_TYPE;
3035

3136

3237
public final long[] shape;
@@ -58,6 +63,14 @@ public final class ArrayMetadata {
5863
@JsonIgnore
5964
public CoreArrayMetadata coreArrayMetadata;
6065

66+
public ArrayMetadata(long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding, Object fillValue,
67+
@Nullable Codec[] codecs,
68+
@Nullable String[] dimensionNames,
69+
@Nullable Map<String, Object> attributes) throws ZarrException {
70+
this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames,
71+
attributes);
72+
}
73+
6174
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
6275
public ArrayMetadata(
6376
@JsonProperty(value = "zarr_format", required = true) int zarrFormat,
@@ -76,6 +89,7 @@ public ArrayMetadata(
7689
if (!nodeType.equals(this.nodeType)) {
7790
throw new ZarrException("Expected node type '" + this.nodeType + "', got '" + nodeType + "'.");
7891
}
92+
7993
this.shape = shape;
8094
this.dataType = dataType;
8195
this.chunkGrid = chunkGrid;
@@ -323,6 +337,12 @@ public Builder withCodecs(Codec... codecs) {
323337
return this;
324338
}
325339

340+
public Builder withCodecs(Function<CodecBuilder, CodecBuilder> codecBuilder) {
341+
CodecBuilder nestedCodecBuilder = new CodecBuilder();
342+
this.codecs = codecBuilder.apply(nestedCodecBuilder).build();
343+
return this;
344+
}
345+
326346
public Builder withDimensionNames(String... dimensionNames) {
327347
this.dimensionNames = dimensionNames;
328348
return this;
@@ -351,8 +371,8 @@ public ArrayMetadata build() throws ZarrException {
351371
chunkGrid.configuration.chunkShape.length + ") need to have the same " +
352372
"number of dimensions.");
353373
}
354-
return new ArrayMetadata(3, "array", shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
355-
dimensionNames, attributes);
374+
return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames,
375+
attributes);
356376
}
357377
}
358378
}

‎src/main/java/com/scalableminds/zarrjava/v3/Group.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
import java.io.IOException;
1010
import java.nio.ByteBuffer;
1111
import java.util.Arrays;
12+
import java.util.Map;
1213
import java.util.Objects;
14+
import java.util.function.Function;
1315

1416
public class Group extends Node {
1517
public GroupMetadata metadata;
@@ -50,9 +52,9 @@ public Node get(String key) throws ZarrException {
5052
try {
5153
String nodeType = objectMapper.readTree(metadataBytes.array()).get("node_type").asText();
5254
switch (nodeType) {
53-
case "array":
55+
case ArrayMetadata.NODE_TYPE:
5456
return new Array(keyHandle, objectMapper.readValue(metadataBytearray, ArrayMetadata.class));
55-
case "group":
57+
case GroupMetadata.NODE_TYPE:
5658
return new Group(keyHandle, objectMapper.readValue(metadataBytearray, GroupMetadata.class));
5759
default:
5860
throw new ZarrException("Unsupported node_type '" + nodeType + "' in " + keyHandle);
@@ -84,6 +86,23 @@ public Node[] list() {
8486
}).filter(Objects::nonNull).toArray(Node[]::new);
8587
}
8688

89+
private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException {
90+
ObjectMapper objectMapper = Node.makeObjectMapper();
91+
ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newGroupMetadata));
92+
storeHandle.resolve(ZARR_JSON).set(metadataBytes);
93+
return new Group(storeHandle, newGroupMetadata);
94+
}
95+
96+
public Group setAttributes(Map<String, Object> newAttributes) throws ZarrException, IOException {
97+
GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes);
98+
return writeMetadata(newGroupMetadata);
99+
}
100+
101+
public Group updateAttributes(Function<Map<String, Object>, Map<String, Object>> attributeMapper) throws ZarrException,
102+
IOException {
103+
return setAttributes(attributeMapper.apply(metadata.attributes));
104+
}
105+
87106
@Override
88107
public String toString() {
89108
return String.format("<v3.Group {%s}>", storeHandle);

‎src/main/java/com/scalableminds/zarrjava/v3/GroupMetadata.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import java.util.Map;
1010

1111
public final class GroupMetadata {
12+
static final String NODE_TYPE = "group";
13+
static final int ZARR_FORMAT = 3;
1214
@JsonProperty("zarr_format")
1315
public final int zarrFormat = 3;
1416
@JsonProperty("node_type")
@@ -17,6 +19,10 @@ public final class GroupMetadata {
1719
@Nullable
1820
public final Map<String, Object> attributes;
1921

22+
public GroupMetadata(@Nullable Map<String, Object> attributes) throws ZarrException {
23+
this(ZARR_FORMAT, NODE_TYPE, attributes);
24+
}
25+
2026
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
2127
public GroupMetadata(
2228
@JsonProperty(value = "zarr_format", required = true) int zarrFormat,
@@ -36,7 +42,7 @@ public static Builder builder() {
3642
}
3743

3844
public static GroupMetadata defaultValue() throws ZarrException {
39-
return new GroupMetadata(3, "group", new HashMap<>());
45+
return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new HashMap<>());
4046
}
4147

4248
public static final class Builder {
@@ -48,7 +54,7 @@ public Builder withAttribute(String key, Object value) {
4854
}
4955

5056
public GroupMetadata build() throws ZarrException {
51-
return new GroupMetadata(3, "group", attributes);
57+
return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, attributes);
5258
}
5359
}
5460
}

‎src/main/java/com/scalableminds/zarrjava/v3/codec/CodecBuilder.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package com.scalableminds.zarrjava.v3.codec;
2+
3+
import com.scalableminds.bloscjava.Blosc;
4+
import com.scalableminds.zarrjava.ZarrException;
5+
import com.scalableminds.zarrjava.v3.DataType;
6+
import com.scalableminds.zarrjava.v3.codec.core.*;
7+
8+
import java.util.ArrayList;
9+
import java.util.List;
10+
import java.util.function.Function;
11+
12+
public class CodecBuilder {
13+
List<Codec> codecs;
14+
15+
public CodecBuilder() {
16+
codecs = new ArrayList<>();
17+
}
18+
19+
public CodecBuilder withBlosc(Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, int blockSize) {
20+
try {
21+
codecs.add(new BloscCodec(new BloscCodec.Configuration(cname, shuffle, clevel, typeSize, blockSize)));
22+
} catch (ZarrException e) {
23+
throw new RuntimeException(e);
24+
}
25+
return this;
26+
}
27+
28+
public CodecBuilder withBlosc(String cname, String shuffle, int clevel, DataType dataType, int blockSize) {
29+
return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel,
30+
dataType.getByteCount(), blockSize);
31+
}
32+
33+
public CodecBuilder withBlosc(String cname, String shuffle, int clevel, DataType dataType) {
34+
return withBlosc(cname, shuffle, clevel, dataType, 0);
35+
}
36+
37+
public CodecBuilder withBlosc(String cname, int clevel, DataType dataType) {
38+
return withBlosc(cname, "noshuffle", clevel, dataType);
39+
}
40+
41+
public CodecBuilder withBlosc(String cname, DataType dataType) {
42+
return withBlosc(cname, 5, dataType);
43+
}
44+
45+
public CodecBuilder withBlosc(DataType dataType) {
46+
return withBlosc("zstd", dataType);
47+
}
48+
49+
public CodecBuilder withTranspose(String order) {
50+
try {
51+
codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order)));
52+
} catch (ZarrException e) {
53+
throw new RuntimeException(e);
54+
}
55+
return this;
56+
}
57+
58+
public CodecBuilder withEndian(EndianCodec.Endian endian) {
59+
codecs.add(new EndianCodec(new EndianCodec.Configuration(endian)));
60+
return this;
61+
}
62+
63+
public CodecBuilder withEndian(String endian) {
64+
return withEndian(EndianCodec.Endian.valueOf(endian));
65+
}
66+
67+
public CodecBuilder withGzip(int clevel) {
68+
try {
69+
codecs.add(new GzipCodec(new GzipCodec.Configuration(clevel)));
70+
} catch (ZarrException e) {
71+
throw new RuntimeException(e);
72+
}
73+
return this;
74+
}
75+
76+
public CodecBuilder withGzip() {
77+
return withGzip(5);
78+
}
79+
80+
public CodecBuilder withSharding(int[] chunkShape) {
81+
try {
82+
codecs.add(new ShardingIndexedCodec(new ShardingIndexedCodec.Configuration(chunkShape, null)));
83+
} catch (ZarrException e) {
84+
throw new RuntimeException(e);
85+
}
86+
return this;
87+
}
88+
89+
public CodecBuilder withSharding(int[] chunkShape, Function<CodecBuilder, CodecBuilder> codecBuilder) {
90+
CodecBuilder nestedBuilder = new CodecBuilder();
91+
try {
92+
codecs.add(new ShardingIndexedCodec(
93+
new ShardingIndexedCodec.Configuration(chunkShape, codecBuilder.apply(nestedBuilder).build())));
94+
} catch (ZarrException e) {
95+
throw new RuntimeException(e);
96+
}
97+
return this;
98+
}
99+
100+
public Codec[] build() {
101+
return codecs.toArray(new Codec[0]);
102+
}
103+
}

0 commit comments

Comments
 (0)