Skip to content

Commit 6bc33a7

Browse files
normanrznormanrz
committed
wip
1 parent 42e891a commit 6bc33a7

File tree

20 files changed

+681
-172
lines changed

20 files changed

+681
-172
lines changed

‎TODO‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
- Cut out chunks
2+
- Transpose
3+
- Endian
4+
- Gzip

‎pom.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1515
<jackson.version>2.14.2</jackson.version>
1616
<aws.version>1.12.477</aws.version>
17+
<netcdfJavaVersion>5.5.3</netcdfJavaVersion>
1718
</properties>
1819

1920
<dependencyManagement>
@@ -39,6 +40,11 @@
3940
<artifactId>jackson-datatype-jdk8</artifactId>
4041
<version>${jackson.version}</version>
4142
</dependency>
43+
<dependency>
44+
<groupId>edu.ucar</groupId>
45+
<artifactId>cdm-core</artifactId>
46+
<version>${netcdfJavaVersion}</version>
47+
</dependency>
4248
<dependency>
4349
<groupId>com.amazonaws</groupId>
4450
<artifactId>aws-java-sdk-s3</artifactId>
@@ -62,4 +68,12 @@
6268
</dependency>
6369
</dependencies>
6470

71+
<repositories>
72+
<repository>
73+
<id>unidata-all</id>
74+
<name>Unidata All</name>
75+
<url>https://artifacts.unidata.ucar.edu/repository/unidata-all/</url>
76+
</repository>
77+
</repositories>
78+
6579
</project>

‎src/main/java/com/scalableminds/zarrjava/indexing/Indexer.java‎

Lines changed: 147 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,163 @@
11
package com.scalableminds.zarrjava.indexing;
22

3+
import com.scalableminds.zarrjava.v3.Utils;
4+
import ucar.ma2.Array;
5+
import ucar.ma2.IndexIterator;
6+
import ucar.ma2.InvalidRangeException;
7+
import ucar.ma2.Range;
8+
9+
import java.util.ArrayList;
310
import java.util.Arrays;
411

512
public class Indexer {
6-
public static long[][] computeChunkCoords(long[] shape, int[] chunkShape, int[] bufferShape, long[] offset) {
7-
final int depth = shape.length;
8-
long[] start = new long[depth];
9-
long[] end = new long[depth];
13+
14+
public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape) {
15+
return computeChunkCoords(arrayShape, chunkShape, new long[arrayShape.length], Utils.toIntArray(arrayShape));
16+
}
17+
18+
public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) {
19+
return computeChunkCoords(Utils.toLongArray(arrayShape), chunkShape);
20+
}
21+
22+
public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, int[] selShape) {
23+
final int ndim = arrayShape.length;
24+
long[] start = new long[ndim];
25+
long[] end = new long[ndim];
1026
int numChunks = 1;
11-
for (int i = 0; i < depth; i++) {
12-
final int staIdx = (int)(offset[i] / chunkShape[i]);
13-
final int endIdx = (int)((offset[i] + bufferShape[i] - 1) / chunkShape[i]);
27+
for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
28+
final int staIdx = (int) (selOffset[dimIdx] / chunkShape[dimIdx]);
29+
final int endIdx = (int) ((selOffset[dimIdx] + selShape[dimIdx] - 1) / chunkShape[dimIdx]);
1430
numChunks *= (endIdx - staIdx + 1);
15-
start[i] = staIdx;
16-
end[i] = endIdx;
31+
start[dimIdx] = staIdx;
32+
end[dimIdx] = endIdx;
1733
}
1834

19-
final long[][] chunkIndices = new long[numChunks][];
35+
final long[][] chunkCoords = new long[numChunks][];
2036

21-
final long[] currentIdx = Arrays.copyOf(start, depth);
22-
for (int i = 0; i < chunkIndices.length; i++) {
23-
chunkIndices[i] = Arrays.copyOf(currentIdx, depth);
24-
int depthIdx = depth - 1;
25-
while (depthIdx >= 0) {
26-
if (currentIdx[depthIdx] >= end[depthIdx]) {
27-
currentIdx[depthIdx] = start[depthIdx];
28-
depthIdx--;
37+
final long[] currentIdx = Arrays.copyOf(start, ndim);
38+
for (int i = 0; i < chunkCoords.length; i++) {
39+
chunkCoords[i] = Arrays.copyOf(currentIdx, ndim);
40+
int dimIdx = ndim - 1;
41+
while (dimIdx >= 0) {
42+
if (currentIdx[dimIdx] >= end[dimIdx]) {
43+
currentIdx[dimIdx] = start[dimIdx];
44+
dimIdx--;
2945
} else {
30-
currentIdx[depthIdx]++;
31-
depthIdx = -1;
46+
currentIdx[dimIdx]++;
47+
dimIdx = -1;
3248
}
3349
}
3450
}
35-
return chunkIndices;
51+
return chunkCoords;
52+
}
53+
54+
public static ChunkProjection computeProjection(long[] chunkCoords, int[] arrayShape, int[] chunkShape) {
55+
return computeProjection(chunkCoords, Utils.toLongArray(arrayShape), chunkShape);
56+
}
57+
58+
public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, int[] chunkShape) {
59+
return computeProjection(chunkCoords, arrayShape, chunkShape, new long[chunkCoords.length],
60+
Utils.toIntArray(arrayShape));
61+
}
62+
63+
public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, int[] chunkShape, long[] selOffset, int[] selShape) {
64+
int ndim = chunkCoords.length;
65+
int[] chunkOffset = new int[ndim];
66+
int[] outOffset = new int[ndim];
67+
int[] shape = new int[ndim];
68+
69+
for (int dimIdx = 0; dimIdx < chunkCoords.length; dimIdx++) {
70+
// compute offsets for chunk within overall array
71+
long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx];
72+
long dimLimit = Math.min(arrayShape[dimIdx], (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]);
73+
// determine chunk length, accounting for trailing chunk
74+
long dimChunkLen = dimLimit - dimOffset;
75+
76+
if (selOffset[dimIdx] < dimOffset) {
77+
// selection starts before current chunk
78+
chunkOffset[dimIdx] = 0;
79+
// compute number of previous items, provides offset into output array
80+
outOffset[dimIdx] = (int) (dimOffset - selOffset[dimIdx]);
81+
} else {
82+
// selection starts within current chunk
83+
chunkOffset[dimIdx] = (int) (selOffset[dimIdx] - dimOffset);
84+
outOffset[dimIdx] = 0;
85+
}
86+
87+
if (selOffset[dimIdx] + selShape[dimIdx] > dimLimit) {
88+
// selection ends after current chunk
89+
shape[dimIdx] = (int) (chunkShape[dimIdx] - selOffset[dimIdx]);
90+
} else {
91+
// selection ends within current chunk
92+
shape[dimIdx] = (int) (selOffset[dimIdx] + selShape[dimIdx] - dimOffset - chunkOffset[dimIdx]);
93+
}
94+
}
95+
96+
return new ChunkProjection(chunkCoords, chunkOffset, outOffset, shape);
97+
}
98+
99+
public static void copyRegion(Array source, int[] sourceOffset, Array target, int[] targetOffset, int[] shape) {
100+
assert sourceOffset.length == targetOffset.length;
101+
assert source.getRank() == sourceOffset.length;
102+
assert target.getRank() == targetOffset.length;
103+
assert shape.length == sourceOffset.length;
104+
105+
try {
106+
final ArrayList<Range> sourceRanges = new ArrayList<>();
107+
final ArrayList<Range> targetRanges = new ArrayList<>();
108+
for (int dimIdx = 0; dimIdx < shape.length; dimIdx++) {
109+
assert sourceOffset[dimIdx] + shape[dimIdx] <= source.getShape()[dimIdx];
110+
assert targetOffset[dimIdx] + shape[dimIdx] <= target.getShape()[dimIdx];
111+
112+
sourceRanges.add(new Range(sourceOffset[dimIdx], sourceOffset[dimIdx] + shape[dimIdx] - 1));
113+
targetRanges.add(new Range(targetOffset[dimIdx], targetOffset[dimIdx] + shape[dimIdx] - 1));
114+
}
115+
final IndexIterator sourceRangeIterator = source.getRangeIterator(sourceRanges);
116+
final IndexIterator targetRangeIterator = target.getRangeIterator(targetRanges);
117+
final Class elementType = source.getElementType();
118+
ValueSetter setter = createValueSetter(elementType);
119+
while (sourceRangeIterator.hasNext()) {
120+
setter.set(sourceRangeIterator, targetRangeIterator);
121+
}
122+
} catch (InvalidRangeException ex) {
123+
throw new RuntimeException("Unreachable");
124+
}
125+
}
126+
127+
private static ValueSetter createValueSetter(Class elementType) {
128+
if (elementType == double.class) {
129+
return (sourceIterator, targetIterator) -> targetIterator.setDoubleNext(sourceIterator.getDoubleNext());
130+
} else if (elementType == float.class) {
131+
return (sourceIterator, targetIterator) -> targetIterator.setFloatNext(sourceIterator.getFloatNext());
132+
} else if (elementType == long.class) {
133+
return (sourceIterator, targetIterator) -> targetIterator.setLongNext(sourceIterator.getLongNext());
134+
} else if (elementType == int.class) {
135+
return (sourceIterator, targetIterator) -> targetIterator.setIntNext(sourceIterator.getIntNext());
136+
} else if (elementType == short.class) {
137+
return (sourceIterator, targetIterator) -> targetIterator.setShortNext(sourceIterator.getShortNext());
138+
} else if (elementType == byte.class) {
139+
return (sourceIterator, targetIterator) -> targetIterator.setByteNext(sourceIterator.getByteNext());
140+
}
141+
return (sourceIterator, targetIterator) -> targetIterator.setObjectNext(sourceIterator.getObjectNext());
142+
}
143+
144+
private interface ValueSetter {
145+
146+
void set(IndexIterator sourceIterator, IndexIterator targetIterator);
147+
}
148+
149+
public static final class ChunkProjection {
150+
public long[] chunkCoords;
151+
public int[] chunkOffset;
152+
153+
public int[] outOffset;
154+
public int[] shape;
155+
156+
public ChunkProjection(long[] chunkCoords, int[] chunkOffset, int[] outOffset, int[] shape) {
157+
this.chunkCoords = chunkCoords;
158+
this.chunkOffset = chunkOffset;
159+
this.outOffset = outOffset;
160+
this.shape = shape;
161+
}
36162
}
37163
}

‎src/main/java/com/scalableminds/zarrjava/store/ArrayValueHandle.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.scalableminds.zarrjava.store;
2+
3+
import com.scalableminds.zarrjava.indexing.OpenSlice;
4+
import com.scalableminds.zarrjava.indexing.Selector;
5+
import com.scalableminds.zarrjava.v3.DataType;
6+
import ucar.ma2.Array;
7+
8+
import java.nio.ByteBuffer;
9+
import java.nio.ByteOrder;
10+
11+
public class ArrayValueHandle extends ValueHandle {
12+
13+
Array array;
14+
15+
public ArrayValueHandle(Array array) {
16+
this.array = array;
17+
}
18+
19+
@Override
20+
public ValueHandle get(OpenSlice slice) {
21+
return null;
22+
}
23+
24+
@Override
25+
public void set(OpenSlice slice, ValueHandle value) {
26+
27+
}
28+
29+
@Override
30+
public ByteBuffer toBytes() {
31+
return array.getDataAsByteBuffer(ByteOrder.LITTLE_ENDIAN);
32+
}
33+
34+
@Override
35+
public Array toArray() {
36+
return array;
37+
}
38+
39+
@Override
40+
public Array toArray(int[] shape, DataType dataType) {
41+
return array.reshapeNoCopy(shape);
42+
}
43+
}

‎src/main/java/com/scalableminds/zarrjava/store/BufferValueHandle.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.Selector;
3+
import com.scalableminds.zarrjava.indexing.OpenSlice;
4+
import com.scalableminds.zarrjava.v3.DataType;
5+
import ucar.ma2.Array;
46

57
import java.nio.ByteBuffer;
68

@@ -17,17 +19,27 @@ public BufferValueHandle(byte[] bytes) {
1719
}
1820

1921
@Override
20-
public ValueHandle get(Selector selector) {
22+
public ValueHandle get(OpenSlice slice) {
2123
return null;
2224
}
2325

2426
@Override
25-
public void set(Selector selector, ValueHandle value) {
27+
public void set(OpenSlice slice, ValueHandle value) {
2628

2729
}
2830

2931
@Override
3032
public ByteBuffer toBytes() {
3133
return bytes;
3234
}
35+
36+
@Override
37+
public Array toArray() {
38+
return null;
39+
}
40+
41+
@Override
42+
public Array toArray(int[] shape, DataType dataType) {
43+
return Array.factory(dataType.getMA2DataType(), shape, bytes);
44+
}
3345
}

‎src/main/java/com/scalableminds/zarrjava/store/FileValueHandle.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.Selector;
3+
import com.scalableminds.zarrjava.v3.DataType;
4+
import com.scalableminds.zarrjava.indexing.OpenSlice;
5+
import ucar.ma2.Array;
46

57
import java.nio.ByteBuffer;
68
import java.util.Optional;
@@ -15,23 +17,33 @@ public FileValueHandle(Store store, String key) {
1517
}
1618

1719
@Override
18-
public ValueHandle get(Selector selector) {
19-
if (selector.value.length != 1) throw new AssertionError();
20-
Optional<BufferValueHandle> valueHandle =
21-
store.get(key, selector.value[0]).map(BufferValueHandle::new);
20+
public ValueHandle get(OpenSlice slice) {
21+
Optional<BufferValueHandle> valueHandle = store.get(key, slice).map(BufferValueHandle::new);
2222
if (valueHandle.isPresent()) {
2323
return valueHandle.get();
2424
}
2525
return new NoneHandle();
2626
}
2727

2828
@Override
29-
public void set(Selector selector, ValueHandle value) {
29+
public void set(OpenSlice slice, ValueHandle value) {
3030

3131
}
3232

3333
@Override
3434
public ByteBuffer toBytes() {
3535
return store.get(key, null).orElse(null);
3636
}
37+
38+
@Override
39+
public Array toArray() {
40+
return null;
41+
}
42+
43+
@Override
44+
public Array toArray(int[] shape, DataType dataType) {
45+
ByteBuffer bytes = toBytes();
46+
if (bytes != null) return Array.factory(dataType.getMA2DataType(), shape, bytes);
47+
return null;
48+
}
3749
}

‎src/main/java/com/scalableminds/zarrjava/store/NoneHandle.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,33 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.Selector;
3+
import com.scalableminds.zarrjava.indexing.OpenSlice;
4+
import com.scalableminds.zarrjava.v3.DataType;
5+
import ucar.ma2.Array;
46

57
import java.nio.ByteBuffer;
68

7-
public class NoneHandle extends ValueHandle{
9+
public class NoneHandle extends ValueHandle {
810
@Override
9-
public ValueHandle get(Selector selector) {
11+
public ValueHandle get(OpenSlice slice) {
1012
return this;
1113
}
1214

1315
@Override
14-
public void set(Selector selector, ValueHandle value) {
16+
public void set(OpenSlice slice, ValueHandle value) {
1517
}
1618

1719
@Override
1820
public ByteBuffer toBytes() {
1921
return null;
2022
}
23+
24+
@Override
25+
public Array toArray() {
26+
return null;
27+
}
28+
29+
@Override
30+
public Array toArray(int[] shape, DataType dataType) {
31+
return null;
32+
}
2133
}

0 commit comments

Comments
 (0)