Skip to content

Commit 42e891a

Browse files
normanrznormanrz
committed
blosc codec
1 parent 36612c5 commit 42e891a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+661
-370
lines changed

‎TODO‎

Whitespace-only changes.

‎pom.xml‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
55
<modelVersion>4.0.0</modelVersion>
66

7-
<groupId>dev.zarr.zarrjava</groupId>
7+
<groupId>com.scalableminds.zarrjava</groupId>
88
<artifactId>zarr-java</artifactId>
99
<version>1.0-SNAPSHOT</version>
1010

@@ -13,8 +13,21 @@
1313
<maven.compiler.target>8</maven.compiler.target>
1414
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1515
<jackson.version>2.14.2</jackson.version>
16+
<aws.version>1.12.477</aws.version>
1617
</properties>
1718

19+
<dependencyManagement>
20+
<dependencies>
21+
<dependency>
22+
<groupId>com.amazonaws</groupId>
23+
<artifactId>aws-java-sdk-bom</artifactId>
24+
<version>${aws.version}</version>
25+
<type>pom</type>
26+
<scope>import</scope>
27+
</dependency>
28+
</dependencies>
29+
</dependencyManagement>
30+
1831
<dependencies>
1932
<dependency>
2033
<groupId>com.fasterxml.jackson.core</groupId>
@@ -27,9 +40,14 @@
2740
<version>${jackson.version}</version>
2841
</dependency>
2942
<dependency>
30-
<groupId>org.lasersonlab</groupId>
31-
<artifactId>jblosc</artifactId>
32-
<version>1.0.1</version>
43+
<groupId>com.amazonaws</groupId>
44+
<artifactId>aws-java-sdk-s3</artifactId>
45+
</dependency>
46+
<dependency>
47+
<groupId>com.scalableminds</groupId>
48+
<artifactId>blosc-java</artifactId>
49+
<version>0.1-1.21.4-SNAPSHOT</version>
50+
<systemPath>${project.basedir}/../blosc-java/target/blosc-java-0.1-1.21.4-SNAPSHOT.jar</systemPath>
3351
</dependency>
3452
<dependency>
3553
<groupId>com.squareup.okhttp</groupId>

src/main/java/dev/zarr/zarrjava/indexing/Indexer.java renamed to src/main/java/com/scalableminds/zarrjava/indexing/Indexer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.indexing;
1+
package com.scalableminds.zarrjava.indexing;
22

33
import java.util.Arrays;
44

src/main/java/dev/zarr/zarrjava/indexing/OpenSlice.java renamed to src/main/java/com/scalableminds/zarrjava/indexing/OpenSlice.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.indexing;
1+
package com.scalableminds.zarrjava.indexing;
22

33
public class OpenSlice {
44
public Long start;

src/main/java/dev/zarr/zarrjava/indexing/Selector.java renamed to src/main/java/com/scalableminds/zarrjava/indexing/Selector.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.indexing;
1+
package com.scalableminds.zarrjava.indexing;
22

33
public class Selector {
44
public OpenSlice[] value;

src/main/java/dev/zarr/zarrjava/indexing/StrictSlice.java renamed to src/main/java/com/scalableminds/zarrjava/indexing/StrictSlice.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.indexing;
1+
package com.scalableminds.zarrjava.indexing;
22

33
public class StrictSlice {
44
public long start;

src/main/java/dev/zarr/zarrjava/store/BufferValueHandle.java renamed to src/main/java/com/scalableminds/zarrjava/store/BufferValueHandle.java

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,19 @@
1-
package dev.zarr.zarrjava.store;
1+
package com.scalableminds.zarrjava.store;
22

3-
import dev.zarr.zarrjava.indexing.Selector;
3+
import com.scalableminds.zarrjava.indexing.Selector;
4+
5+
import java.nio.ByteBuffer;
46

57
public class BufferValueHandle extends ValueHandle {
68

7-
byte[] bytes;
9+
ByteBuffer bytes;
10+
11+
public BufferValueHandle(ByteBuffer bytes) {
12+
this.bytes = bytes;
13+
}
14+
815
public BufferValueHandle(byte[] bytes) {
9-
this.bytes=bytes;
16+
this.bytes = ByteBuffer.wrap(bytes);
1017
}
1118

1219
@Override
@@ -20,7 +27,7 @@ public void set(Selector selector, ValueHandle value) {
2027
}
2128

2229
@Override
23-
public byte[] toBytes() {
30+
public ByteBuffer toBytes() {
2431
return bytes;
2532
}
2633
}

src/main/java/dev/zarr/zarrjava/store/FileValueHandle.java renamed to src/main/java/com/scalableminds/zarrjava/store/FileValueHandle.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
package dev.zarr.zarrjava.store;
1+
package com.scalableminds.zarrjava.store;
22

3-
import dev.zarr.zarrjava.indexing.Selector;
3+
import com.scalableminds.zarrjava.indexing.Selector;
44

5+
import java.nio.ByteBuffer;
56
import java.util.Optional;
67

78
public class FileValueHandle extends ValueHandle {
@@ -30,7 +31,7 @@ public void set(Selector selector, ValueHandle value) {
3031
}
3132

3233
@Override
33-
public byte[] toBytes() {
34+
public ByteBuffer toBytes() {
3435
return store.get(key, null).orElse(null);
3536
}
3637
}

src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java renamed to src/main/java/com/scalableminds/zarrjava/store/FilesystemStore.java

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
1-
package dev.zarr.zarrjava.store;
1+
package com.scalableminds.zarrjava.store;
22

3-
import dev.zarr.zarrjava.indexing.OpenSlice;
3+
import com.scalableminds.zarrjava.indexing.OpenSlice;
4+
import com.scalableminds.zarrjava.v3.Utils;
45

56
import java.io.IOException;
67
import java.nio.ByteBuffer;
78
import java.nio.channels.SeekableByteChannel;
8-
import java.nio.file.FileSystem;
9-
import java.nio.file.FileSystems;
10-
import java.nio.file.Files;
11-
import java.nio.file.Path;
9+
import java.nio.file.*;
1210
import java.util.List;
1311
import java.util.Optional;
1412
import java.util.stream.Collectors;
@@ -24,13 +22,13 @@ public FilesystemStore(String path) {
2422
}
2523

2624
@Override
27-
public Optional<byte[]> get(String key, OpenSlice byteRange) {
25+
public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
2826
Path keyPath = fileSystem.getPath(this.path, key);
2927

3028
System.out.println(keyPath);
3129
if (byteRange == null) {
3230
try {
33-
byte[] bytes = Files.readAllBytes(keyPath);
31+
ByteBuffer bytes = ByteBuffer.wrap(Files.readAllBytes(keyPath));
3432
return Optional.of(bytes);
3533
} catch (IOException e) {
3634
return Optional.empty();
@@ -54,26 +52,26 @@ public Optional<byte[]> get(String key, OpenSlice byteRange) {
5452
}
5553
}
5654

57-
ByteBuffer bytes = ByteBuffer.allocate((int) (endOffset - startOffset));
55+
ByteBuffer bytes = Utils.allocateNative((int) (endOffset - startOffset));
5856
byteChannel.position(startOffset);
5957
byteChannel.read(bytes);
60-
return Optional.of(bytes.array());
58+
return Optional.of(bytes);
6159
} catch (IOException e) {
6260
return Optional.empty();
6361
}
6462
}
6563

6664
@Override
67-
public void set(String key, byte[] bytes, OpenSlice byteRange) {
65+
public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
6866
Path keyPath = fileSystem.getPath(this.path, key);
6967
try {
7068
Files.createDirectories(keyPath.getParent());
7169
} catch (IOException e) {
7270
throw new RuntimeException(e);
7371
}
7472
if (byteRange == null) {
75-
try {
76-
Files.write(keyPath, bytes);
73+
try (SeekableByteChannel channel = Files.newByteChannel(keyPath, StandardOpenOption.WRITE)) {
74+
channel.write(bytes);
7775
} catch (IOException e) {
7876
throw new RuntimeException(e);
7977
}
@@ -92,7 +90,6 @@ public void delete(String key) {
9290
}
9391
}
9492

95-
@Override
9693
public List<String> list(String key) {
9794
Path keyPath = fileSystem.getPath(this.path, key);
9895
try {

src/main/java/dev/zarr/zarrjava/store/HttpStore.java renamed to src/main/java/com/scalableminds/zarrjava/store/HttpStore.java

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
package dev.zarr.zarrjava.store;
1+
package com.scalableminds.zarrjava.store;
22

3+
import com.scalableminds.zarrjava.indexing.OpenSlice;
34
import com.squareup.okhttp.Call;
45
import com.squareup.okhttp.OkHttpClient;
56
import com.squareup.okhttp.Request;
67
import com.squareup.okhttp.Response;
7-
import dev.zarr.zarrjava.indexing.OpenSlice;
88

99
import java.io.IOException;
10-
import java.util.List;
10+
import java.nio.ByteBuffer;
1111
import java.util.Optional;
1212

1313
public class HttpStore extends Store {
@@ -34,7 +34,7 @@ private String getRangeHeader(OpenSlice byteRange) {
3434
}
3535

3636
@Override
37-
public Optional<byte[]> get(String key, OpenSlice byteRange) {
37+
public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
3838
Request.Builder builder = new Request.Builder()
3939
.url(path + "/" + key);
4040

@@ -47,14 +47,14 @@ public Optional<byte[]> get(String key, OpenSlice byteRange) {
4747
Call call = httpClient.newCall(request);
4848
try {
4949
Response response = call.execute();
50-
return Optional.of(response.body().bytes());
50+
return Optional.of(ByteBuffer.wrap(response.body().bytes()));
5151
} catch (IOException e) {
5252
return Optional.empty();
5353
}
5454
}
5555

5656
@Override
57-
public void set(String key, byte[] bytes, OpenSlice byteRange) {
57+
public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
5858
throw new UnsupportedOperationException("Not implemented");
5959
}
6060

@@ -63,10 +63,6 @@ public void delete(String key) {
6363
throw new UnsupportedOperationException("Not implemented");
6464
}
6565

66-
@Override
67-
public List<String> list(String key) {
68-
throw new UnsupportedOperationException("Not implemented");
69-
}
7066

7167
@Override
7268
public String toString() {

0 commit comments

Comments
 (0)