Skip to content

Commit 8ba6946

Browse files
brokkoli71brokkoli71
committed
implement and test v2 Group with metadata handling
1 parent c091e60 commit 8ba6946

File tree

17 files changed

+331
-130
lines changed

17 files changed

+331
-130
lines changed

‎src/main/java/dev/zarr/zarrjava/core/Array.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,15 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
4343
int[] shape = array.getShape();
4444

4545
final int[] chunkShape = metadata.chunkShape();
46-
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape));
46+
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
4747
if (parallel) {
4848
chunkStream = chunkStream.parallel();
4949
}
5050
chunkStream.forEach(
5151
chunkCoords -> {
5252
try {
5353
final IndexingUtils.ChunkProjection chunkProjection =
54-
IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset,
54+
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
5555
shape
5656
);
5757

@@ -169,7 +169,7 @@ public void write(ucar.ma2.Array array, boolean parallel) {
169169
*/
170170
@Nonnull
171171
public ucar.ma2.Array read() throws ZarrException {
172-
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()));
172+
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape));
173173
}
174174

175175
/**
@@ -193,14 +193,14 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
193193
*/
194194
@Nonnull
195195
public ucar.ma2.Array read(final boolean parallel) throws ZarrException {
196-
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()), parallel);
196+
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel);
197197
}
198198

199199
boolean chunkIsInArray(long[] chunkCoords) {
200200
final int[] chunkShape = metadata().chunkShape();
201201
for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) {
202202
if (chunkCoords[dimIdx] < 0
203-
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape()[dimIdx]) {
203+
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
204204
return false;
205205
}
206206
}
@@ -225,7 +225,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
225225
throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'.");
226226
}
227227
for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) {
228-
if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape()[dimIdx]) {
228+
if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) {
229229
throw new ZarrException("Requested data is outside of the array's domain.");
230230
}
231231
}
@@ -237,15 +237,15 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
237237

238238
final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
239239
shape);
240-
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape));
240+
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
241241
if (parallel) {
242242
chunkStream = chunkStream.parallel();
243243
}
244244
chunkStream.forEach(
245245
chunkCoords -> {
246246
try {
247247
final IndexingUtils.ChunkProjection chunkProjection =
248-
IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset,
248+
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
249249
shape
250250
);
251251

‎src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package dev.zarr.zarrjava.core;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnore;
4+
import com.fasterxml.jackson.annotation.JsonProperty;
35
import dev.zarr.zarrjava.ZarrException;
46
import dev.zarr.zarrjava.utils.MultiArrayUtils;
57
import dev.zarr.zarrjava.utils.Utils;
@@ -10,23 +12,40 @@
1012
import java.nio.ByteBuffer;
1113
import java.util.Arrays;
1214

13-
public interface ArrayMetadata {
14-
int ndim();
15+
public abstract class ArrayMetadata {
1516

16-
int[] chunkShape();
17+
public final long[] shape;
1718

18-
long[] shape();
19+
@JsonProperty("fill_value")
20+
public final Object fillValue;
21+
@JsonIgnore
22+
public final Object parsedFillValue;
1923

20-
DataType dataType();
24+
public ArrayMetadata(long[] shape, Object fillValue, DataType dataType) throws ZarrException {
25+
this.shape = shape;
26+
this.fillValue = fillValue;
27+
this.parsedFillValue = parseFillValue(fillValue, dataType);
28+
}
2129

22-
Array allocateFillValueChunk();
30+
public int ndim() {
31+
return shape.length;
32+
}
2333

24-
ChunkKeyEncoding chunkKeyEncoding();
34+
public abstract int[] chunkShape();
2535

26-
Object parsedFillValue();
36+
public abstract DataType dataType();
2737

28-
static Object parseFillValue(Object fillValue, @Nonnull DataType dataType)
38+
public abstract Array allocateFillValueChunk();
39+
40+
public abstract ChunkKeyEncoding chunkKeyEncoding();
41+
42+
public abstract Object parsedFillValue();
43+
44+
public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType)
2945
throws ZarrException {
46+
if (fillValue == null) {
47+
return null;
48+
}
3049
boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL;
3150
boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8;
3251
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16;
@@ -128,7 +147,7 @@ else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) {
128147
throw new ZarrException("Invalid fill value '" + fillValue + "'.");
129148
}
130149

131-
final class CoreArrayMetadata {
150+
public static final class CoreArrayMetadata {
132151

133152
public final long[] shape;
134153
public final int[] chunkShape;

‎src/main/java/dev/zarr/zarrjava/core/Group.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package dev.zarr.zarrjava.core;
2+
import dev.zarr.zarrjava.ZarrException;
3+
import dev.zarr.zarrjava.store.StoreHandle;
4+
5+
import javax.annotation.Nonnull;
6+
import javax.annotation.Nullable;
7+
import java.util.Objects;
8+
import java.util.stream.Stream;
9+
10+
public abstract class Group extends Node {
11+
12+
protected Group(@Nonnull StoreHandle storeHandle) {
13+
super(storeHandle);
14+
}
15+
16+
@Nullable
17+
public abstract Node get(String key) throws ZarrException;
18+
19+
public Stream<Node> list() {
20+
return storeHandle.list()
21+
.map(key -> {
22+
try {
23+
return get(key);
24+
} catch (ZarrException e) {
25+
throw new RuntimeException(e);
26+
}
27+
})
28+
.filter(Objects::nonNull);
29+
}
30+
31+
public Node[] listAsArray() {
32+
try (Stream<Node> nodeStream = list()) {
33+
return nodeStream.toArray(Node[]::new);
34+
}
35+
}
36+
}

‎src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package dev.zarr.zarrjava.core;
2+
3+
public abstract class GroupMetadata {}

‎src/main/java/dev/zarr/zarrjava/core/Node.java‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package dev.zarr.zarrjava.core;
22

3-
import com.fasterxml.jackson.databind.ObjectMapper;
4-
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
53
import dev.zarr.zarrjava.store.StoreHandle;
6-
import dev.zarr.zarrjava.v3.codec.CodecRegistry;
74
import javax.annotation.Nonnull;
85

96
public abstract class Node {
@@ -15,11 +12,4 @@ protected Node(@Nonnull StoreHandle storeHandle) {
1512
this.storeHandle = storeHandle;
1613
}
1714

18-
public static ObjectMapper makeObjectMapper() {
19-
ObjectMapper objectMapper = new ObjectMapper();
20-
objectMapper.registerModule(new Jdk8Module());
21-
objectMapper.registerSubtypes(CodecRegistry.getNamedTypes());
22-
return objectMapper;
23-
}
24-
2515
}

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
package dev.zarr.zarrjava.v2;
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
4-
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
54
import dev.zarr.zarrjava.ZarrException;
65
import dev.zarr.zarrjava.store.StoreHandle;
76
import dev.zarr.zarrjava.utils.Utils;
87
import dev.zarr.zarrjava.core.codec.CodecPipeline;
98
import dev.zarr.zarrjava.v2.codec.Codec;
10-
import dev.zarr.zarrjava.v2.codec.CodecRegistry;
119
import dev.zarr.zarrjava.v2.codec.core.BytesCodec;
1210

1311
import javax.annotation.Nonnull;
@@ -16,10 +14,10 @@
1614
import java.util.Arrays;
1715
import java.util.function.Function;
1816
import java.util.stream.Collectors;
17+
import static dev.zarr.zarrjava.v2.Node.makeObjectMapper;
1918

20-
public class Array extends dev.zarr.zarrjava.core.Array {
19+
public class Array extends dev.zarr.zarrjava.core.Array implements Node {
2120

22-
static final String ZARRAY = ".zarray";
2321
public ArrayMetadata metadata;
2422

2523
protected ArrayMetadata metadata() {
@@ -55,13 +53,6 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
5553
);
5654
}
5755

58-
public static ObjectMapper makeObjectMapper() {
59-
ObjectMapper objectMapper = new ObjectMapper();
60-
objectMapper.registerModule(new Jdk8Module());
61-
objectMapper.registerSubtypes(CodecRegistry.getNamedTypes());
62-
return objectMapper;
63-
}
64-
6556

6657
/**
6758
* Creates a new Zarr array with the provided metadata at a specified storage location. This

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java‎

Lines changed: 10 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -13,39 +13,31 @@
1313

1414
import javax.annotation.Nullable;
1515

16-
import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue;
1716

18-
19-
public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata {
17+
public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata {
2018
static final int ZARR_FORMAT = 2;
2119

2220
@JsonProperty("zarr_format")
2321
public final int zarrFormat = ZARR_FORMAT;
2422

25-
public long[] shape;
26-
public int[] chunks;
23+
public final int[] chunks;
2724

2825
@JsonProperty("dtype")
29-
public DataType dataType;
26+
public final DataType dataType;
3027

3128
@JsonIgnore
3229
public final Endianness endianness;
3330

3431
@JsonProperty("order")
35-
public Order order;
32+
public final Order order;
3633

3734
@JsonProperty("dimension_separator")
38-
public Separator dimensionSeparator;
39-
40-
@JsonProperty("fill_value")
41-
public Object fillValue;
42-
@JsonIgnore
43-
public final Object parsedFillValue;
35+
public final Separator dimensionSeparator;
4436

4537
@Nullable
46-
public Codec[] filters;
38+
public final Codec[] filters;
4739
@Nullable
48-
public Codec compressor;
40+
public final Codec compressor;
4941

5042
@JsonIgnore
5143
public CoreArrayMetadata coreArrayMetadata;
@@ -63,46 +55,33 @@ public ArrayMetadata(
6355
@Nullable @JsonProperty(value = "compressor", required = true) Codec compressor,
6456
@Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator
6557
) throws ZarrException {
66-
super();
58+
super(shape, fillValue, dataType);
6759
if (zarrFormat != this.zarrFormat) {
6860
throw new ZarrException(
6961
"Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'.");
7062
}
71-
this.shape = shape;
7263
this.chunks = chunks;
7364
this.dataType = dataType;
7465
this.endianness = dataType.getEndianness();
75-
this.fillValue = fillValue;
76-
if (fillValue == null) {
77-
this.parsedFillValue = null;
78-
} else {
79-
this.parsedFillValue = parseFillValue(fillValue, this.dataType);
80-
}
8166
this.order = order;
8267
this.dimensionSeparator = dimensionSeparator;
8368
this.filters = filters;
8469
this.compressor = compressor;
8570
this.coreArrayMetadata =
8671
new ArrayMetadata.CoreArrayMetadata(shape, chunks,
8772
this.dataType,
88-
parsedFillValue
73+
this.parsedFillValue
8974
);
9075
}
9176

9277

93-
public int ndim() {
94-
return shape.length;
95-
}
9678

9779
@Override
9880
public int[] chunkShape() {
9981
return chunks;
10082
}
10183

102-
@Override
103-
public long[] shape() {
104-
return shape;
105-
}
84+
10685

10786
@Override
10887
public DataType dataType() {

0 commit comments

Comments
 (0)