Skip to content

Commit f650fd6

Browse files
brokkoli71brokkoli71
committed
add ZipStore tests
diff --git c/src/main/java/dev/zarr/zarrjava/store/ZipStore.java i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java new file mode 100644 index 0000000..054917f --- /dev/null +++ i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -0,0 +1,72 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; + +public class ZipStore implements Store, Store.ListableStore { + @nonnull + private final Path path; + + public ZipStore(@nonnull Path path) { + this.path = path; + } + + public ZipStore(@nonnull String path) { + this.path = Paths.get(path); + } + + + @OverRide + public Stream<String> list(String[] keys) { + return Stream.empty(); + } + + @OverRide + public boolean exists(String[] keys) { + return false; + } + + @nullable + @OverRide + public ByteBuffer get(String[] keys) { + return null; + } + + @nullable + @OverRide + public ByteBuffer get(String[] keys, long start) { + return null; + } + + @nullable + @OverRide + public ByteBuffer get(String[] keys, long start, long end) { + return null; + } + + @OverRide + public void set(String[] keys, ByteBuffer bytes) { + + } + + @OverRide + public void delete(String[] keys) { + + } + + @nonnull + @OverRide + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @OverRide + public String toString() { + return this.path.toUri().toString().replaceAll("\\/$", ""); + } + +} diff --git c/src/test/java/dev/zarr/zarrjava/Utils.java i/src/test/java/dev/zarr/zarrjava/Utils.java new file mode 100644 index 0000000..0026200 --- /dev/null +++ i/src/test/java/dev/zarr/zarrjava/Utils.java @@ -0,0 +1,40 @@ +package dev.zarr.zarrjava; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class Utils { + + static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { + if (fileToZip.isHidden()) { + return; + } + if (fileToZip.isDirectory()) { + if (fileName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(fileName)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(fileName + "/")); + zipOut.closeEntry(); + } + File[] children = fileToZip.listFiles(); + for (File childFile : children) { + zipFile(childFile, fileName + "/" + childFile.getName(), zipOut); + } + return; + } + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileName); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + } + +} diff --git c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4a369c9..c7d2ab4 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -7,16 +7,22 @@ import dev.zarr.zarrjava.v3.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.CsvSource; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; import java.util.stream.Stream; +import java.nio.file.Path; +import java.util.zip.ZipOutputStream; +import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { @@ -132,4 +138,72 @@ public class ZarrStoreTest extends ZarrTest { Assertions.assertEquals("test group", attrs.getString("description")); } + + @test + public void testZipStore() throws ZarrException, IOException { + Path sourceDir = TESTOUTPUT.resolve("testZipStore"); + Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); + FilesystemStore fsStore = new FilesystemStore(sourceDir); + writeTestGroupV3(fsStore, true); + + FileOutputStream fos = new FileOutputStream(targetDir.toFile()); + ZipOutputStream zipOut = new ZipOutputStream(fos); + + File fileToZip = new File(sourceDir.toUri()); + zipFile(fileToZip, fileToZip.getName(), zipOut); + zipOut.close(); + fos.close(); + + ZipStore zipStore = new ZipStore(targetDir); + assertIsTestGroupV3(Group.open(zipStore.resolve()), true); + } + + static Stream<Store> localStores() { + return Stream.of( +// new ConcurrentMemoryStore(), + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), + new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + ); + } + + @ParameterizedTest + @MethodSource("localStores") + public void testLocalStores(Store store) throws IOException, ZarrException { + boolean useParallel = true; + Group group = writeTestGroupV3(store, useParallel); + assertIsTestGroupV3(group, useParallel); + } + + int[] testData(){ + int[] testData = new int[1024 * 1024]; + Arrays.setAll(testData, p -> p); + return testData; + } + + Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException { + StoreHandle storeHandle = store.resolve(); + + Group group = Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + return group; + } + + void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException { + Stream<dev.zarr.zarrjava.core.Node> nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + Array array = (Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } }
1 parent a6512de commit f650fd6

File tree

3 files changed

+186
-0
lines changed

3 files changed

+186
-0
lines changed

‎src/main/java/dev/zarr/zarrjava/store/ZipStore.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package dev.zarr.zarrjava.store;
2+
3+
import javax.annotation.Nonnull;
4+
import javax.annotation.Nullable;
5+
import java.nio.ByteBuffer;
6+
import java.nio.file.Path;
7+
import java.nio.file.Paths;
8+
import java.util.stream.Stream;
9+
10+
public class ZipStore implements Store, Store.ListableStore {
11+
@Nonnull
12+
private final Path path;
13+
14+
public ZipStore(@Nonnull Path path) {
15+
this.path = path;
16+
}
17+
18+
public ZipStore(@Nonnull String path) {
19+
this.path = Paths.get(path);
20+
}
21+
22+
23+
@Override
24+
public Stream<String> list(String[] keys) {
25+
return Stream.empty();
26+
}
27+
28+
@Override
29+
public boolean exists(String[] keys) {
30+
return false;
31+
}
32+
33+
@Nullable
34+
@Override
35+
public ByteBuffer get(String[] keys) {
36+
return null;
37+
}
38+
39+
@Nullable
40+
@Override
41+
public ByteBuffer get(String[] keys, long start) {
42+
return null;
43+
}
44+
45+
@Nullable
46+
@Override
47+
public ByteBuffer get(String[] keys, long start, long end) {
48+
return null;
49+
}
50+
51+
@Override
52+
public void set(String[] keys, ByteBuffer bytes) {
53+
54+
}
55+
56+
@Override
57+
public void delete(String[] keys) {
58+
59+
}
60+
61+
@Nonnull
62+
@Override
63+
public StoreHandle resolve(String... keys) {
64+
return new StoreHandle(this, keys);
65+
}
66+
67+
@Override
68+
public String toString() {
69+
return this.path.toUri().toString().replaceAll("\\/$", "");
70+
}
71+
72+
}

‎src/test/java/dev/zarr/zarrjava/Utils.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package dev.zarr.zarrjava;
2+
3+
import java.io.File;
4+
import java.io.FileInputStream;
5+
import java.io.IOException;
6+
import java.util.zip.ZipEntry;
7+
import java.util.zip.ZipOutputStream;
8+
9+
public class Utils {
10+
11+
static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException {
12+
if (fileToZip.isHidden()) {
13+
return;
14+
}
15+
if (fileToZip.isDirectory()) {
16+
if (fileName.endsWith("/")) {
17+
zipOut.putNextEntry(new ZipEntry(fileName));
18+
zipOut.closeEntry();
19+
} else {
20+
zipOut.putNextEntry(new ZipEntry(fileName + "/"));
21+
zipOut.closeEntry();
22+
}
23+
File[] children = fileToZip.listFiles();
24+
for (File childFile : children) {
25+
zipFile(childFile, fileName + "/" + childFile.getName(), zipOut);
26+
}
27+
return;
28+
}
29+
FileInputStream fis = new FileInputStream(fileToZip);
30+
ZipEntry zipEntry = new ZipEntry(fileName);
31+
zipOut.putNextEntry(zipEntry);
32+
byte[] bytes = new byte[1024];
33+
int length;
34+
while ((length = fis.read(bytes)) >= 0) {
35+
zipOut.write(bytes, 0, length);
36+
}
37+
fis.close();
38+
}
39+
40+
}

‎src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,22 @@
77
import org.junit.jupiter.api.Assertions;
88
import org.junit.jupiter.api.Test;
99
import org.junit.jupiter.params.ParameterizedTest;
10+
import org.junit.jupiter.params.provider.MethodSource;
1011
import org.junit.jupiter.params.provider.CsvSource;
1112
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
1213
import software.amazon.awssdk.regions.Region;
1314
import software.amazon.awssdk.services.s3.S3Client;
1415

16+
import java.io.File;
17+
import java.io.FileOutputStream;
1518
import java.io.IOException;
1619
import java.nio.file.Files;
1720
import java.util.Arrays;
1821
import java.util.stream.Stream;
22+
import java.nio.file.Path;
23+
import java.util.zip.ZipOutputStream;
1924

25+
import static dev.zarr.zarrjava.Utils.zipFile;
2026
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
2127

2228
public class ZarrStoreTest extends ZarrTest {
@@ -132,4 +138,72 @@ public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOExcep
132138
Assertions.assertEquals("test group", attrs.getString("description"));
133139

134140
}
141+
142+
@Test
143+
public void testZipStore() throws ZarrException, IOException {
144+
Path sourceDir = TESTOUTPUT.resolve("testZipStore");
145+
Path targetDir = TESTOUTPUT.resolve("testZipStore.zip");
146+
FilesystemStore fsStore = new FilesystemStore(sourceDir);
147+
writeTestGroupV3(fsStore, true);
148+
149+
FileOutputStream fos = new FileOutputStream(targetDir.toFile());
150+
ZipOutputStream zipOut = new ZipOutputStream(fos);
151+
152+
File fileToZip = new File(sourceDir.toUri());
153+
zipFile(fileToZip, fileToZip.getName(), zipOut);
154+
zipOut.close();
155+
fos.close();
156+
157+
ZipStore zipStore = new ZipStore(targetDir);
158+
assertIsTestGroupV3(Group.open(zipStore.resolve()), true);
159+
}
160+
161+
static Stream<Store> localStores() {
162+
return Stream.of(
163+
// new ConcurrentMemoryStore(),
164+
new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")),
165+
new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"))
166+
);
167+
}
168+
169+
@ParameterizedTest
170+
@MethodSource("localStores")
171+
public void testLocalStores(Store store) throws IOException, ZarrException {
172+
boolean useParallel = true;
173+
Group group = writeTestGroupV3(store, useParallel);
174+
assertIsTestGroupV3(group, useParallel);
175+
}
176+
177+
int[] testData(){
178+
int[] testData = new int[1024 * 1024];
179+
Arrays.setAll(testData, p -> p);
180+
return testData;
181+
}
182+
183+
Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException {
184+
StoreHandle storeHandle = store.resolve();
185+
186+
Group group = Group.create(storeHandle);
187+
Array array = group.createArray("array", b -> b
188+
.withShape(1024, 1024)
189+
.withDataType(DataType.UINT32)
190+
.withChunkShape(5, 5)
191+
);
192+
array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel);
193+
group.createGroup("subgroup");
194+
group.setAttributes(new Attributes(b -> b.set("some", "value")));
195+
return group;
196+
}
197+
198+
void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException {
199+
Stream<dev.zarr.zarrjava.core.Node> nodes = group.list();
200+
Assertions.assertEquals(2, nodes.count());
201+
Array array = (Array) group.get("array");
202+
Assertions.assertNotNull(array);
203+
ucar.ma2.Array result = array.read(useParallel);
204+
Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
205+
Attributes attrs = group.metadata().attributes;
206+
Assertions.assertNotNull(attrs);
207+
Assertions.assertEquals("value", attrs.getString("some"));
208+
}
135209
}

0 commit comments

Comments
 (0)