Skip to content

Commit c5cd02e

Browse files
brokkoli71dominiklbrokkoli71
authored
V2 attributes (#28)
* add support for attributes in Array and ArrayMetadata classes * change v3 attributes class to Attributes * implement setAttributes and updateAttributes for v2 * add test for resize array * fix array.setAttributes * rename attributes.add to set * Load v2 GroupMetadata attributes (#32) * Load v2 GroupMetadata attributes * Update src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java Co-authored-by: Hannes Spitz <[email protected]> * Update src/main/java/dev/zarr/zarrjava/v3/Group.java Co-authored-by: Hannes Spitz <[email protected]> --------- Co-authored-by: Hannes Spitz <[email protected]> * add Group.create with Attributes * add tests for v2 group attributes and * test and fix v3 Group Attributes * dont ignore all unknown group metadata but add consolidatedMetadata * rename zarr_python_group_v2.py to zarr_python_group.py for consistency * add group attributes unit tests * fix testoutput paths for testGroupReadWrite --------- Co-authored-by: Dominik Lindner <[email protected]> Co-authored-by: brokkoli71 <[email protected]>
1 parent 43ebb38 commit c5cd02e

21 files changed

+858
-76
lines changed

‎src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ public int ndim() {
4343

4444
public abstract Object parsedFillValue();
4545

46+
public @Nonnull abstract Attributes attributes() throws ZarrException;
47+
4648
public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType)
4749
throws ZarrException {
4850
if (fillValue == null) {

‎src/main/java/dev/zarr/zarrjava/core/Attributes.java‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
package dev.zarr.zarrjava.core;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
5+
import javax.annotation.Nonnull;
6+
import java.util.HashMap;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.function.Function;
10+
11+
public class Attributes extends HashMap<String, Object> {
12+
13+
public Attributes() {
14+
super();
15+
}
16+
17+
public Attributes (Function<Attributes, Attributes> attributeMapper) {
18+
super();
19+
attributeMapper.apply(this);
20+
}
21+
22+
23+
public Attributes(Map<String, Object> attributes) {
24+
super(attributes);
25+
}
26+
27+
public Attributes set(String s, Object o){
28+
this.put(s, o);
29+
return this;
30+
}
31+
32+
public Attributes delete(String s){
33+
this.remove(s);
34+
return this;
35+
}
36+
37+
public boolean getBoolean(String key) {
38+
Object value = this.get(key);
39+
if (value instanceof Boolean) {
40+
return (Boolean) value;
41+
}
42+
throw new IllegalArgumentException("Value for key " + key + " is not a Boolean");
43+
}
44+
45+
public int getInt(String key) {
46+
Object value = this.get(key);
47+
if (value instanceof Number) {
48+
return ((Number) value).intValue();
49+
}
50+
throw new IllegalArgumentException("Value for key " + key + " is not an Integer");
51+
}
52+
53+
public double getDouble(String key) {
54+
Object value = this.get(key);
55+
if (value instanceof Number) {
56+
return ((Number) value).doubleValue();
57+
}
58+
throw new IllegalArgumentException("Value for key " + key + " is not a Double");
59+
}
60+
61+
public float getFloat(String key) {
62+
Object value = this.get(key);
63+
if (value instanceof Number) {
64+
return ((Number) value).floatValue();
65+
}
66+
throw new IllegalArgumentException("Value for key " + key + " is not a Float");
67+
}
68+
69+
@Nonnull
70+
public String getString(String key) {
71+
Object value = this.get(key);
72+
if (value instanceof String) {
73+
return (String) value;
74+
}
75+
throw new IllegalArgumentException("Value for key " + key + " is not a String");
76+
}
77+
78+
@Nonnull
79+
public List<Object> getList(String key) {
80+
Object value = this.get(key);
81+
if (value instanceof List) {
82+
return (List<Object>) value;
83+
}
84+
throw new IllegalArgumentException("Value for key " + key + " is not a List");
85+
}
86+
87+
public Attributes getAttributes(String key) {
88+
Object value = this.get(key);
89+
if (value instanceof Attributes) {
90+
return (Attributes) value;
91+
}
92+
if (value instanceof Map) {
93+
return new Attributes((Map<String, Object>) value);
94+
}
95+
throw new IllegalArgumentException("Value for key " + key + " is not an Attributes object");
96+
}
97+
98+
public <T> T[] getArray(String key, Class<T> clazz) throws ZarrException {
99+
Object value = this.get(key);
100+
if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) {
101+
return (T[]) value;
102+
}
103+
if (value instanceof List) {
104+
List<?> list = (List<?>) value;
105+
@SuppressWarnings("unchecked")
106+
T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size());
107+
for (int i = 0; i < list.size(); i++) {
108+
Object elem = list.get(i);
109+
if (clazz.isInstance(elem)) {
110+
array[i] = clazz.cast(elem);
111+
} else {
112+
// Try to find a constructor that takes the element's class
113+
java.lang.reflect.Constructor<?> matched = null;
114+
for (java.lang.reflect.Constructor<?> c : clazz.getConstructors()) {
115+
Class<?>[] params = c.getParameterTypes();
116+
if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) {
117+
matched = c;
118+
break;
119+
}
120+
}
121+
if (matched != null) {
122+
try {
123+
array[i] = (T) matched.newInstance(elem);
124+
} catch (Exception e) {
125+
throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e);
126+
}
127+
} else {
128+
throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName());
129+
}
130+
}
131+
}
132+
return array;
133+
}
134+
throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName());
135+
}
136+
137+
public int[] getIntArray(String key) {
138+
Object value = this.get(key);
139+
if (value instanceof int[]) {
140+
return (int[]) value;
141+
}
142+
if (value instanceof List) {
143+
List<?> list = (List<?>) value;
144+
int[] array = new int[list.size()];
145+
for (int i = 0; i < list.size(); i++) {
146+
Object elem = list.get(i);
147+
if (elem instanceof Number) {
148+
array[i] = ((Number) elem).intValue();
149+
} else {
150+
throw new IllegalArgumentException("Element at index " + i + " is not a Number");
151+
}
152+
}
153+
return array;
154+
}
155+
throw new IllegalArgumentException("Value for key " + key + " is not an int array or List");
156+
}
157+
158+
public long[] getLongArray(String key) {
159+
Object value = this.get(key);
160+
if (value instanceof long[]) {
161+
return (long[]) value;
162+
}
163+
if (value instanceof List) {
164+
List<?> list = (List<?>) value;
165+
long[] array = new long[list.size()];
166+
for (int i = 0; i < list.size(); i++) {
167+
Object elem = list.get(i);
168+
if (elem instanceof Number) {
169+
array[i] = ((Number) elem).longValue();
170+
} else {
171+
throw new IllegalArgumentException("Element at index " + i + " is not a Number");
172+
}
173+
}
174+
return array;
175+
}
176+
throw new IllegalArgumentException("Value for key " + key + " is not a long array or List");
177+
}
178+
179+
public double[] getDoubleArray(String key) {
180+
Object value = this.get(key);
181+
if (value instanceof double[]) {
182+
return (double[]) value;
183+
}
184+
if (value instanceof List) {
185+
List<?> list = (List<?>) value;
186+
double[] array = new double[list.size()];
187+
for (int i = 0; i < list.size(); i++) {
188+
Object elem = list.get(i);
189+
if (elem instanceof Number) {
190+
array[i] = ((Number) elem).doubleValue();
191+
} else {
192+
throw new IllegalArgumentException("Element at index " + i + " is not a Number");
193+
}
194+
}
195+
return array;
196+
}
197+
throw new IllegalArgumentException("Value for key " + key + " is not a double array or List");
198+
}
199+
200+
public float[] getFloatArray(String key) {
201+
Object value = this.get(key);
202+
if (value instanceof float[]) {
203+
return (float[]) value;
204+
}
205+
if (value instanceof List) {
206+
List<?> list = (List<?>) value;
207+
float[] array = new float[list.size()];
208+
for (int i = 0; i < list.size(); i++) {
209+
Object elem = list.get(i);
210+
if (elem instanceof Number) {
211+
array[i] = ((Number) elem).floatValue();
212+
} else {
213+
throw new IllegalArgumentException("Element at index " + i + " is not a Number");
214+
}
215+
}
216+
return array;
217+
}
218+
throw new IllegalArgumentException("Value for key " + key + " is not a float array or List");
219+
}
220+
221+
public boolean[] getBooleanArray(String key) {
222+
Object value = this.get(key);
223+
if (value instanceof boolean[]) {
224+
return (boolean[]) value;
225+
}
226+
if (value instanceof List) {
227+
List<?> list = (List<?>) value;
228+
boolean[] array = new boolean[list.size()];
229+
for (int i = 0; i < list.size(); i++) {
230+
Object elem = list.get(i);
231+
if (elem instanceof Boolean) {
232+
array[i] = (Boolean) elem;
233+
} else {
234+
throw new IllegalArgumentException("Element at index " + i + " is not a Boolean");
235+
}
236+
}
237+
return array;
238+
}
239+
throw new IllegalArgumentException("Value for key " + key + " is not a boolean array or List");
240+
}
241+
}

‎src/main/java/dev/zarr/zarrjava/core/Group.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,4 +83,6 @@ public Node[] listAsArray() {
8383
return nodeStream.toArray(Node[]::new);
8484
}
8585
}
86+
87+
public abstract GroupMetadata metadata();
8688
}

‎src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
11
package dev.zarr.zarrjava.core;
22

3-
public abstract class GroupMetadata {}
3+
import javax.annotation.Nonnull;
4+
5+
import dev.zarr.zarrjava.ZarrException;
6+
7+
public abstract class GroupMetadata {
8+
9+
public @Nonnull abstract Attributes attributes() throws ZarrException;
10+
11+
}

‎src/main/java/dev/zarr/zarrjava/core/Node.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public interface Node {
1313

1414
String ZARR_JSON = "zarr.json";
1515
String ZARRAY = ".zarray";
16+
String ZATTRS = ".zattrs";
1617
String ZGROUP = ".zgroup";
1718

1819
/**

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package dev.zarr.zarrjava.v2;
22

3+
import com.fasterxml.jackson.databind.ObjectMapper;
34
import com.fasterxml.jackson.databind.ObjectWriter;
45
import dev.zarr.zarrjava.ZarrException;
6+
import dev.zarr.zarrjava.core.Attributes;
57
import dev.zarr.zarrjava.store.FilesystemStore;
68
import dev.zarr.zarrjava.store.StoreHandle;
79
import dev.zarr.zarrjava.utils.Utils;
@@ -47,13 +49,19 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE
4749
* @throws ZarrException throws ZarrException if the Zarr array cannot be opened
4850
*/
4951
public static Array open(StoreHandle storeHandle) throws IOException, ZarrException {
52+
ObjectMapper mapper = makeObjectMapper();
53+
ArrayMetadata metadata = mapper.readValue(
54+
Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()),
55+
ArrayMetadata.class
56+
);
57+
if (storeHandle.resolve(ZATTRS).exists())
58+
metadata.attributes = mapper.readValue(
59+
Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()),
60+
Attributes.class
61+
);
5062
return new Array(
5163
storeHandle,
52-
makeObjectMapper()
53-
.readValue(
54-
Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()),
55-
ArrayMetadata.class
56-
)
64+
metadata
5765
);
5866
}
5967

@@ -144,6 +152,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata,
144152
}
145153
ObjectWriter objectWriter = makeObjectWriter();
146154
ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata));
155+
if (arrayMetadata.attributes != null) {
156+
StoreHandle attrsHandle = storeHandle.resolve(ZATTRS);
157+
ByteBuffer attrsBytes = ByteBuffer.wrap(
158+
objectWriter.writeValueAsBytes(arrayMetadata.attributes));
159+
attrsHandle.set(attrsBytes);
160+
}
147161
metadataHandle.set(metadataBytes);
148162
return new Array(storeHandle, arrayMetadata);
149163
}
@@ -165,6 +179,61 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat
165179
return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata);
166180
}
167181

182+
private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException {
183+
return Array.create(storeHandle, newArrayMetadata, true);
184+
}
185+
186+
/**
187+
* Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or
188+
* deleted. This method returns a new instance of the Zarr array class and the old instance
189+
* becomes invalid.
190+
*
191+
* @param newShape the new shape of the Zarr array
192+
* @throws ZarrException if the new metadata is invalid
193+
* @throws IOException throws IOException if the new metadata cannot be serialized
194+
*/
195+
public Array resize(long[] newShape) throws ZarrException, IOException {
196+
if (newShape.length != metadata.ndim()) {
197+
throw new IllegalArgumentException(
198+
"'newShape' needs to have rank '" + metadata.ndim() + "'.");
199+
}
200+
201+
ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata)
202+
.withShape(newShape)
203+
.build();
204+
return writeMetadata(newArrayMetadata);
205+
}
206+
207+
/**
208+
* Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This
209+
* method returns a new instance of the Zarr array class and the old instance becomes invalid.
210+
*
211+
* @param newAttributes the new attributes of the Zarr array
212+
* @throws ZarrException throws ZarrException if the new metadata is invalid
213+
* @throws IOException throws IOException if the new metadata cannot be serialized
214+
*/
215+
public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException {
216+
ArrayMetadata newArrayMetadata =
217+
ArrayMetadataBuilder.fromArrayMetadata(metadata, false)
218+
.withAttributes(newAttributes)
219+
.build();
220+
return writeMetadata(newArrayMetadata);
221+
}
222+
223+
/**
224+
* Updates the attributes of the Zarr array. It provides a callback that gets the current
225+
* attributes as input and needs to return the new set of attributes. The attributes in the
226+
* callback may be mutated. This method overwrites and removes any existing attributes. This
227+
* method returns a new instance of the Zarr array class and the old instance becomes invalid.
228+
*
229+
* @param attributeMapper the callback that is used to construct the new attributes
230+
* @throws ZarrException throws ZarrException if the new metadata is invalid
231+
* @throws IOException throws IOException if the new metadata cannot be serialized
232+
*/
233+
public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
234+
return setAttributes(attributeMapper.apply(metadata.attributes));
235+
}
236+
168237
@Override
169238
public String toString() {
170239
return String.format("<v2.Array {%s} (%s) %s>", storeHandle,

0 commit comments

Comments
 (0)