Skip to content

Commit a7d090a

Browse files
normanrznormanrz
committed
writing arrays
1 parent cf25ec3 commit a7d090a

File tree

18 files changed

+743
-260
lines changed

18 files changed

+743
-260
lines changed

‎TODO‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
- Cut out chunks
2-
- Transpose
3-
- Endian
4-
- Gzip
1+
+ Cut out chunks
2+
+ Transpose
3+
+ Endian
4+
+ Gzip
5+
+ S3Store
6+
- Partial read
7+
+ Write
8+
- Partial write

‎src/main/java/com/scalableminds/zarrjava/indexing/CRC32C.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package com.scalableminds.zarrjava.indexing;
2+
3+
import java.nio.ByteBuffer;
4+
import java.util.zip.Checksum;
5+
6+
// Copyright 2011 Google Inc. All rights reserved.
7+
public class CRC32C implements Checksum {
8+
9+
10+
/**
11+
* This class generates a CRC32C checksum, defined by rfc3720 section B.4.
12+
*/
13+
14+
private static final long[] CRC_TABLE =
15+
{0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf,
16+
0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f,
17+
0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0,
18+
0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede,
19+
0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611,
20+
0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1,
21+
0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, 0xba3a117e,
22+
0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc,
23+
0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, 0xcba24573,
24+
0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3,
25+
0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c,
26+
0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362,
27+
0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad,
28+
0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d,
29+
0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2,
30+
0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, 0x82f63b78,
31+
0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7,
32+
0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17,
33+
0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8,
34+
0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6,
35+
0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69,
36+
0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9,
37+
0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06,
38+
0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4,
39+
0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b,
40+
0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab,
41+
0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 0x590ab964,
42+
0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a,
43+
0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5,
44+
0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75,
45+
0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba,
46+
0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351};
47+
48+
private static final long LONG_MASK = 0xffffffffL;
49+
private static final long BYTE_MASK = 0xff;
50+
51+
private long crc;
52+
53+
public CRC32C() {
54+
crc = 0;
55+
}
56+
57+
/**
58+
* Updates the checksum with a new byte.
59+
*
60+
* @param b the new byte.
61+
*/
62+
@Override
63+
public void update(int b) {
64+
long newCrc = crc ^ LONG_MASK;
65+
newCrc = updateByte((byte) b, newCrc);
66+
crc = newCrc ^ LONG_MASK;
67+
}
68+
69+
/**
70+
* Updates the checksum with an array of bytes.
71+
*
72+
* @param bArray the array of bytes.
73+
* @param off the offset into the array where the update should begin.
74+
* @param len the length of data to examine.
75+
*/
76+
@Override
77+
public void update(byte[] bArray, int off, int len) {
78+
long newCrc = crc ^ LONG_MASK;
79+
for (int i = off; i < off + len; i++) {
80+
newCrc = updateByte(bArray[i], newCrc);
81+
}
82+
crc = newCrc ^ LONG_MASK;
83+
}
84+
85+
public void update(ByteBuffer buf) {
86+
long newCrc = crc ^ LONG_MASK;
87+
while (buf.remaining() > 0) {
88+
newCrc = updateByte(buf.get(), newCrc);
89+
}
90+
crc = newCrc ^ LONG_MASK;
91+
}
92+
93+
/**
94+
* Returns the value of the checksum.
95+
*
96+
* @return the long representation of the checksum (high bits set to zero).
97+
*/
98+
@Override
99+
public long getValue() {
100+
return crc;
101+
}
102+
103+
/**
104+
* Returns the value of the checksum.
105+
*
106+
* @return the 4-byte array representation of the checksum in network byte order (big endian).
107+
*/
108+
public byte[] getValueAsBytes() {
109+
long value = crc;
110+
byte[] result = new byte[4];
111+
for (int i = 3; i >= 0; i--) {
112+
result[i] = (byte) (value & 0xffL);
113+
value >>= 8;
114+
}
115+
return result;
116+
}
117+
118+
/**
119+
* Resets the crc.
120+
*/
121+
@Override
122+
public void reset() {
123+
crc = 0;
124+
}
125+
126+
private long updateByte(byte newByte, long crc) {
127+
byte b = (byte) (newByte & BYTE_MASK);
128+
int index = (int) ((crc ^ b) & BYTE_MASK);
129+
return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK;
130+
}
131+
}

‎src/main/java/com/scalableminds/zarrjava/indexing/Indexer.java‎

Lines changed: 22 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
package com.scalableminds.zarrjava.indexing;
22

33
import com.scalableminds.zarrjava.v3.Utils;
4-
import ucar.ma2.Array;
5-
import ucar.ma2.IndexIterator;
6-
import ucar.ma2.InvalidRangeException;
7-
import ucar.ma2.Range;
84

9-
import java.util.ArrayList;
105
import java.util.Arrays;
116

127
public class Indexer {
@@ -60,18 +55,16 @@ public static ChunkProjection computeProjection(long[] chunkCoords, long[] array
6055
Utils.toIntArray(arrayShape));
6156
}
6257

63-
public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, int[] chunkShape, long[] selOffset, int[] selShape) {
64-
int ndim = chunkCoords.length;
65-
int[] chunkOffset = new int[ndim];
66-
int[] outOffset = new int[ndim];
67-
int[] shape = new int[ndim];
58+
public static ChunkProjection computeProjection(final long[] chunkCoords, final long[] arrayShape, final int[] chunkShape, final long[] selOffset, final int[] selShape) {
59+
final int ndim = chunkCoords.length;
60+
final int[] chunkOffset = new int[ndim];
61+
final int[] outOffset = new int[ndim];
62+
final int[] shape = new int[ndim];
6863

6964
for (int dimIdx = 0; dimIdx < chunkCoords.length; dimIdx++) {
7065
// compute offsets for chunk within overall array
71-
long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx];
72-
long dimLimit = Math.min(arrayShape[dimIdx], (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]);
73-
// determine chunk length, accounting for trailing chunk
74-
long dimChunkLen = dimLimit - dimOffset;
66+
final long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx];
67+
final long dimLimit = Math.min(arrayShape[dimIdx], (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]);
7568

7669
if (selOffset[dimIdx] < dimOffset) {
7770
// selection starts before current chunk
@@ -97,7 +90,7 @@ public static ChunkProjection computeProjection(long[] chunkCoords, long[] array
9790
}
9891

9992

100-
public static long cOrderIndex(long[] chunkCoords, long[] arrayShape) {
93+
public static long cOrderIndex(final long[] chunkCoords, final long[] arrayShape) {
10194
long index = 0;
10295
long multiplier = 1;
10396

@@ -109,7 +102,7 @@ public static long cOrderIndex(long[] chunkCoords, long[] arrayShape) {
109102
return index;
110103
}
111104

112-
public static long fOrderIndex(long[] chunkCoords, long[] arrayShape) {
105+
public static long fOrderIndex(final long[] chunkCoords, final long[] arrayShape) {
113106
int index = 0;
114107
int multiplier = 1;
115108

@@ -121,64 +114,26 @@ public static long fOrderIndex(long[] chunkCoords, long[] arrayShape) {
121114
return index;
122115
}
123116

124-
public static void copyRegion(Array source, int[] sourceOffset, Array target, int[] targetOffset, int[] shape) {
125-
assert sourceOffset.length == targetOffset.length;
126-
assert source.getRank() == sourceOffset.length;
127-
assert target.getRank() == targetOffset.length;
128-
assert shape.length == sourceOffset.length;
129-
130-
try {
131-
final ArrayList<Range> sourceRanges = new ArrayList<>();
132-
final ArrayList<Range> targetRanges = new ArrayList<>();
133-
for (int dimIdx = 0; dimIdx < shape.length; dimIdx++) {
134-
assert sourceOffset[dimIdx] + shape[dimIdx] <= source.getShape()[dimIdx];
135-
assert targetOffset[dimIdx] + shape[dimIdx] <= target.getShape()[dimIdx];
136-
137-
sourceRanges.add(new Range(sourceOffset[dimIdx], sourceOffset[dimIdx] + shape[dimIdx] - 1));
138-
targetRanges.add(new Range(targetOffset[dimIdx], targetOffset[dimIdx] + shape[dimIdx] - 1));
139-
}
140-
final IndexIterator sourceRangeIterator = source.getRangeIterator(sourceRanges);
141-
final IndexIterator targetRangeIterator = target.getRangeIterator(targetRanges);
142-
final Class elementType = source.getElementType();
143-
ValueSetter setter = createValueSetter(elementType);
144-
while (sourceRangeIterator.hasNext()) {
145-
setter.set(sourceRangeIterator, targetRangeIterator);
146-
}
147-
} catch (InvalidRangeException ex) {
148-
throw new RuntimeException("Unreachable");
149-
}
150-
}
117+
public static boolean isFullChunk(final int[] selOffset, final int[] selShape, final int[] chunkShape) {
118+
assert selOffset.length == selShape.length;
119+
assert selOffset.length == chunkShape.length;
151120

152-
private static ValueSetter createValueSetter(Class elementType) {
153-
if (elementType == double.class) {
154-
return (sourceIterator, targetIterator) -> targetIterator.setDoubleNext(sourceIterator.getDoubleNext());
155-
} else if (elementType == float.class) {
156-
return (sourceIterator, targetIterator) -> targetIterator.setFloatNext(sourceIterator.getFloatNext());
157-
} else if (elementType == long.class) {
158-
return (sourceIterator, targetIterator) -> targetIterator.setLongNext(sourceIterator.getLongNext());
159-
} else if (elementType == int.class) {
160-
return (sourceIterator, targetIterator) -> targetIterator.setIntNext(sourceIterator.getIntNext());
161-
} else if (elementType == short.class) {
162-
return (sourceIterator, targetIterator) -> targetIterator.setShortNext(sourceIterator.getShortNext());
163-
} else if (elementType == byte.class) {
164-
return (sourceIterator, targetIterator) -> targetIterator.setByteNext(sourceIterator.getByteNext());
121+
for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) {
122+
if (selOffset[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) {
123+
return false;
124+
}
165125
}
166-
return (sourceIterator, targetIterator) -> targetIterator.setObjectNext(sourceIterator.getObjectNext());
167-
}
168-
169-
private interface ValueSetter {
170-
171-
void set(IndexIterator sourceIterator, IndexIterator targetIterator);
126+
return true;
172127
}
173128

174129
public static final class ChunkProjection {
175-
public long[] chunkCoords;
176-
public int[] chunkOffset;
130+
final public long[] chunkCoords;
131+
final public int[] chunkOffset;
177132

178-
public int[] outOffset;
179-
public int[] shape;
133+
final public int[] outOffset;
134+
final public int[] shape;
180135

181-
public ChunkProjection(long[] chunkCoords, int[] chunkOffset, int[] outOffset, int[] shape) {
136+
public ChunkProjection(final long[] chunkCoords, final int[] chunkOffset, final int[] outOffset, final int[] shape) {
182137
this.chunkCoords = chunkCoords;
183138
this.chunkOffset = chunkOffset;
184139
this.outOffset = outOffset;

0 commit comments

Comments
 (0)