Skip to content

Commit cf25ec3

Browse files
normanrznormanrz
committed
wip
1 parent fcf9afb commit cf25ec3

File tree

8 files changed

+119
-91
lines changed

8 files changed

+119
-91
lines changed

‎src/main/java/com/scalableminds/zarrjava/store/FilesystemStore.java‎

Lines changed: 50 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.OpenSlice;
43
import com.scalableminds.zarrjava.v3.Utils;
54

65
import javax.annotation.Nonnull;
6+
import javax.annotation.Nullable;
77
import java.io.IOException;
88
import java.nio.ByteBuffer;
99
import java.nio.channels.SeekableByteChannel;
1010
import java.nio.file.*;
1111
import java.util.Iterator;
12-
import java.util.Optional;
1312
import java.util.stream.Stream;
1413

1514
public class FilesystemStore implements Store, Store.ListableStore {
@@ -24,62 +23,76 @@ public FilesystemStore(@Nonnull String path) {
2423
this.path = path;
2524
}
2625

26+
@Nullable
2727
@Override
28-
public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
28+
public ByteBuffer get(String key) {
2929
Path keyPath = fileSystem.getPath(this.path, key);
30-
31-
System.out.println(keyPath);
32-
if (byteRange == null) {
33-
try {
34-
ByteBuffer bytes = ByteBuffer.wrap(Files.readAllBytes(keyPath));
35-
return Optional.of(bytes);
36-
} catch (IOException e) {
37-
return Optional.empty();
38-
}
30+
try {
31+
ByteBuffer bytes = ByteBuffer.wrap(Files.readAllBytes(keyPath));
32+
return bytes;
33+
} catch (IOException e) {
34+
return null;
3935
}
36+
}
37+
38+
@Nullable
39+
@Override
40+
public ByteBuffer get(String key, long start) {
41+
Path keyPath = fileSystem.getPath(this.path, key);
42+
4043
try (SeekableByteChannel byteChannel = Files.newByteChannel(keyPath)) {
4144
long startOffset = 0;
42-
if (byteRange.start != null) {
43-
if (byteRange.start >= 0) {
44-
startOffset = byteRange.start;
45-
} else {
46-
startOffset = byteChannel.size() + byteRange.start;
47-
}
45+
if (start >= 0) {
46+
startOffset = start;
47+
} else {
48+
startOffset = byteChannel.size() + start;
4849
}
4950
long endOffset = byteChannel.size();
50-
if (byteRange.end != null) {
51-
if (byteRange.end >= 0) {
52-
endOffset = byteRange.end;
53-
} else {
54-
endOffset = byteChannel.size() + byteRange.end;
55-
}
56-
}
57-
5851
ByteBuffer bytes = Utils.allocateNative((int) (endOffset - startOffset));
5952
byteChannel.position(startOffset);
6053
byteChannel.read(bytes);
61-
return Optional.of(bytes);
54+
bytes.rewind();
55+
return bytes;
6256
} catch (IOException e) {
63-
return Optional.empty();
57+
return null;
6458
}
6559
}
6660

61+
@Nullable
6762
@Override
68-
public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
63+
public ByteBuffer get(String key, long start, long end) {
64+
Path keyPath = fileSystem.getPath(this.path, key);
65+
66+
try (SeekableByteChannel byteChannel = Files.newByteChannel(keyPath)) {
67+
long startOffset = 0;
68+
if (start >= 0) {
69+
startOffset = start;
70+
} else {
71+
startOffset = byteChannel.size() + start;
72+
}
73+
ByteBuffer bytes = Utils.allocateNative((int) (end - startOffset));
74+
byteChannel.position(startOffset);
75+
byteChannel.read(bytes);
76+
bytes.rewind();
77+
return bytes;
78+
} catch (IOException e) {
79+
return null;
80+
}
81+
}
82+
83+
84+
@Override
85+
public void set(String key, ByteBuffer bytes) {
6986
Path keyPath = fileSystem.getPath(this.path, key);
7087
try {
7188
Files.createDirectories(keyPath.getParent());
7289
} catch (IOException e) {
7390
throw new RuntimeException(e);
7491
}
75-
if (byteRange == null) {
76-
try (SeekableByteChannel channel = Files.newByteChannel(keyPath, StandardOpenOption.WRITE)) {
77-
channel.write(bytes);
78-
} catch (IOException e) {
79-
throw new RuntimeException(e);
80-
}
81-
} else {
82-
throw new UnsupportedOperationException("`FilesystemStore::set` does not support range writes.");
92+
try (SeekableByteChannel channel = Files.newByteChannel(keyPath, StandardOpenOption.WRITE)) {
93+
channel.write(bytes);
94+
} catch (IOException e) {
95+
throw new RuntimeException(e);
8396
}
8497
}
8598

‎src/main/java/com/scalableminds/zarrjava/store/HttpStore.java‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.OpenSlice;
43
import com.squareup.okhttp.*;
54

65
import javax.annotation.Nonnull;
6+
import javax.annotation.Nullable;
77
import java.io.IOException;
88
import java.nio.ByteBuffer;
9-
import java.util.Optional;
109

1110
public class HttpStore implements Store {
1211

@@ -20,43 +19,46 @@ public HttpStore(@Nonnull String path) {
2019
this.path = path;
2120
}
2221

23-
private String getRangeHeader(OpenSlice byteRange) {
24-
if (byteRange.start != null) {
25-
if (byteRange.start < 0) {
26-
return String.format("bytes=-%d", -byteRange.start);
27-
} else if (byteRange.end != null && byteRange.end > 0) {
28-
return String.format("bytes=%d-%d", byteRange.start, byteRange.end + 1);
29-
} else {
30-
return String.format("bytes=%d", byteRange.start);
22+
@Nullable
23+
ByteBuffer get(Request request) {
24+
Call call = httpClient.newCall(request);
25+
try {
26+
Response response = call.execute();
27+
try (ResponseBody body = response.body()) {
28+
return ByteBuffer.wrap(body.bytes());
3129
}
30+
} catch (IOException e) {
31+
return null;
3232
}
33-
throw new UnsupportedOperationException("Unsupported range request");
3433
}
3534

35+
@Nullable
3636
@Override
37-
public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
38-
Request.Builder builder = new Request.Builder()
39-
.url(path + "/" + key);
37+
public ByteBuffer get(String key) {
38+
Request request = new Request.Builder().url(path + "/" + key).build();
39+
return get(request);
40+
}
4041

41-
if (byteRange != null) {
42-
builder.header("Range", getRangeHeader(byteRange));
43-
}
42+
@Nullable
43+
@Override
44+
public ByteBuffer get(String key, long start) {
45+
Request request = new Request.Builder().url(path + "/" + key).header("Range",
46+
start < 0 ? String.format("Bytes=%d", start) : String.format("Bytes=%d-", start)).build();
4447

45-
Request request = builder.build();
48+
return get(request);
49+
}
4650

47-
Call call = httpClient.newCall(request);
48-
try {
49-
Response response = call.execute();
50-
try (ResponseBody body = response.body()) {
51-
return Optional.of(ByteBuffer.wrap(body.bytes()));
52-
}
53-
} catch (IOException e) {
54-
return Optional.empty();
55-
}
51+
@Nullable
52+
@Override
53+
public ByteBuffer get(String key, long start, long end) {
54+
assert start >= 0;
55+
Request request = new Request.Builder().url(path + "/" + key).header("Range",
56+
String.format("Bytes=%d-%d", start, end + 1)).build();
57+
return get(request);
5658
}
5759

5860
@Override
59-
public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
61+
public void set(String key, ByteBuffer bytes) {
6062
throw new UnsupportedOperationException("Not implemented");
6163
}
6264

‎src/main/java/com/scalableminds/zarrjava/store/S3Store.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import com.amazonaws.services.s3.AmazonS3;
44
import com.amazonaws.services.s3.model.ObjectMetadata;
55
import com.amazonaws.services.s3.model.S3ObjectInputStream;
6-
import com.scalableminds.zarrjava.indexing.OpenSlice;
76
import com.scalableminds.zarrjava.v3.Utils;
87

98
import javax.annotation.Nonnull;
@@ -13,7 +12,6 @@
1312
import java.io.InputStream;
1413
import java.nio.ByteBuffer;
1514
import java.util.Iterator;
16-
import java.util.Optional;
1715

1816
public class S3Store implements Store, Store.ListableStore {
1917
@Nonnull
@@ -39,18 +37,31 @@ String dereferencePath(String key) {
3937
return prefix + "/" + key;
4038
}
4139

40+
@Nullable
4241
@Override
43-
public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
42+
public ByteBuffer get(String key) {
4443
try (S3ObjectInputStream inputStream = s3client.getObject(bucketName,
4544
dereferencePath(key)).getObjectContent()) {
46-
return Optional.of(Utils.asByteBuffer(inputStream));
45+
return Utils.asByteBuffer(inputStream);
4746
} catch (IOException e) {
48-
return Optional.empty();
47+
return null;
4948
}
5049
}
5150

51+
@Nullable
52+
@Override
53+
public ByteBuffer get(String key, long start) {
54+
return null; // TODO
55+
}
56+
57+
@Nullable
5258
@Override
53-
public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
59+
public ByteBuffer get(String key, long start, long end) {
60+
return null; // TODO
61+
}
62+
63+
@Override
64+
public void set(String key, ByteBuffer bytes) {
5465
try (InputStream byteStream = new ByteArrayInputStream(bytes.array())) {
5566
s3client.putObject(bucketName, dereferencePath(key), byteStream, new ObjectMetadata());
5667
} catch (IOException e) {
@@ -60,7 +71,7 @@ public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
6071

6172
@Override
6273
public void delete(String key) {
63-
74+
// TODO
6475
}
6576

6677
@Override

‎src/main/java/com/scalableminds/zarrjava/store/Store.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
package com.scalableminds.zarrjava.store;
22

3-
import com.scalableminds.zarrjava.indexing.OpenSlice;
4-
3+
import javax.annotation.Nullable;
54
import java.nio.ByteBuffer;
65
import java.util.Iterator;
7-
import java.util.Optional;
86

97
public interface Store {
8+
@Nullable
9+
ByteBuffer get(String key);
10+
11+
@Nullable
12+
ByteBuffer get(String key, long start);
1013

11-
Optional<ByteBuffer> get(String key, OpenSlice byteRange);
14+
@Nullable
15+
ByteBuffer get(String key, long start, long end);
1216

13-
void set(String key, ByteBuffer bytes, OpenSlice byteRange);
17+
void set(String key, ByteBuffer bytes);
1418

1519
void delete(String key);
1620

‎src/main/java/com/scalableminds/zarrjava/store/StoreHandle.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,18 @@ public StoreHandle(@Nonnull Store store, @Nonnull String key) {
1919

2020
@Nullable
2121
public ByteBuffer read() {
22-
return store.get(key, null).orElse(null);
22+
return store.get(key);
2323
}
24+
2425
@Nullable
2526
public ByteBuffer read(long start) {
26-
return store.get(key, new OpenSlice(start, null)).orElse(null);
27+
return store.get(key, start);
28+
} @Nullable
29+
public ByteBuffer read(long start,long end) {
30+
return store.get(key, start, end);
2731
}
2832

29-
public void set(OpenSlice slice, ByteBuffer bytes) {
30-
31-
}
33+
public void set(ByteBuffer bytes) {
3234

33-
@Override
34-
public ByteBuffer toBytes() {
35-
return store.get(key, null).orElse(null);
3635
}
3736
}

‎src/main/java/com/scalableminds/zarrjava/v2/Array.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public Array(Store store, String path) throws IOException {
1919

2020
ObjectMapper objectMapper = new ObjectMapper();
2121
objectMapper.registerModule(new Jdk8Module());
22-
this.metadata = objectMapper.readValue(store.get(path + "/.zarray", null).get().array(), ArrayMetadata.class);
22+
this.metadata = objectMapper.readValue(store.get(path + "/.zarray").array(), ArrayMetadata.class);
2323
}
2424

2525
@Override

‎src/main/java/com/scalableminds/zarrjava/v3/Array.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
44
import com.scalableminds.zarrjava.indexing.Indexer;
5-
import com.scalableminds.zarrjava.store.FileValueHandle;
65
import com.scalableminds.zarrjava.store.Store;
76
import com.scalableminds.zarrjava.store.StoreHandle;
87
import com.scalableminds.zarrjava.v3.codec.CodecPipeline;
@@ -19,7 +18,7 @@ public Array(Store store, String path) throws IOException {
1918
super(store, path);
2019

2120
ObjectMapper objectMapper = Utils.makeObjectMapper();
22-
this.metadata = objectMapper.readValue(store.get(path + "/zarr.json", null).get().array(), ArrayMetadata.class);
21+
this.metadata = objectMapper.readValue(store.get(path + "/zarr.json").array(), ArrayMetadata.class);
2322
}
2423

2524
public ucar.ma2.Array read(long[] offset, int[] shape) {
@@ -54,7 +53,7 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) {
5453
StoreHandle chunkHandle = new StoreHandle(store, path + "/" + chunkKey);
5554

5655
ucar.ma2.Array chunkArray =
57-
new CodecPipeline(metadata.codecs).decode(chunkHandle.toBytes(), metadata.getCoreMetadata());
56+
new CodecPipeline(metadata.codecs).decode(chunkHandle.read(), metadata.getCoreMetadata());
5857
return chunkArray;
5958
}
6059

‎src/main/java/com/scalableminds/zarrjava/v3/Group.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ public Group(Store store, String path) throws IOException {
1313
super(store, path);
1414

1515
ObjectMapper objectMapper = Utils.makeObjectMapper();
16-
this.metadata = objectMapper.readValue(store.get(path + "/zarr.json", null).get().array(), GroupMetadata.class);
16+
this.metadata = objectMapper.readValue(store.get(path + "/zarr.json").array(), GroupMetadata.class);
1717
}
1818

1919
public Node get(String key) {
2020
ObjectMapper objectMapper = Utils.makeObjectMapper();
2121

2222
try {
23-
String nodeType = objectMapper.readTree(store.get(path + "/" + key + "/zarr.json", null).get().array()).get(
23+
String nodeType = objectMapper.readTree(store.get(path + "/" + key + "/zarr.json").array()).get(
2424
"node_type").asText();
2525
switch (nodeType) {
2626
case "array":

0 commit comments

Comments
 (0)