wip

Author: normanrz

pom.xml

@@ -53,6 +53,7 @@
             <groupId>com.scalableminds</groupId>
             <artifactId>blosc-java</artifactId>
             <version>0.1-1.21.4-SNAPSHOT</version>
+            <scope>system</scope>
             <systemPath>${project.basedir}/../blosc-java/target/blosc-java-0.1-1.21.4-SNAPSHOT.jar</systemPath>
         </dependency>
         <dependency>

src/main/java/com/scalableminds/zarrjava/indexing/Indexer.java

@@ -96,6 +96,31 @@ public static ChunkProjection computeProjection(long[] chunkCoords, long[] array
         return new ChunkProjection(chunkCoords, chunkOffset, outOffset, shape);
     }
 
+
+    public static long cOrderIndex(long[] chunkCoords, long[] arrayShape) {
+        long index = 0;
+        long multiplier = 1;
+
+        for (int i = arrayShape.length - 1; i >= 0; i--) {
+            index += chunkCoords[i] * multiplier;
+            multiplier *= arrayShape[i];
+        }
+
+        return index;
+    }
+
+    public static long fOrderIndex(long[] chunkCoords, long[] arrayShape) {
+        int index = 0;
+        int multiplier = 1;
+
+        for (int i = 0; i < arrayShape.length; i++) {
+            index += chunkCoords[i] * multiplier;
+            multiplier *= arrayShape[i];
+        }
+
+        return index;
+    }
+
     public static void copyRegion(Array source, int[] sourceOffset, Array target, int[] targetOffset, int[] shape) {
         assert sourceOffset.length == targetOffset.length;
         assert source.getRank() == sourceOffset.length;

src/main/java/com/scalableminds/zarrjava/store/ArrayValueHandle.java

@@ -3,20 +3,23 @@
 import com.scalableminds.zarrjava.indexing.OpenSlice;
 import com.scalableminds.zarrjava.v3.Utils;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.*;
-import java.util.List;
+import java.util.Iterator;
 import java.util.Optional;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-public class FilesystemStore extends Store {
+public class FilesystemStore implements Store, Store.ListableStore {
 
+    @Nonnull
     private final FileSystem fileSystem;
+    @Nonnull
     private final String path;
 
-    public FilesystemStore(String path) {
+    public FilesystemStore(@Nonnull String path) {
         this.fileSystem = FileSystems.getDefault();
         this.path = path;
     }
@@ -90,10 +93,10 @@ public void delete(String key) {
         }
     }
 
-    public List<String> list(String key) {
+    public Iterator<String> list(String key) {
         Path keyPath = fileSystem.getPath(this.path, key);
-        try {
-            return Files.list(keyPath).map(p -> p.toFile().getName()).collect(Collectors.toList());
+        try (Stream<Path> paths = Files.list(keyPath)) {
+            return paths.map(p -> p.toFile().getName()).iterator();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }

src/main/java/com/scalableminds/zarrjava/store/BufferValueHandle.java

@@ -1,21 +1,21 @@
 package com.scalableminds.zarrjava.store;
 
 import com.scalableminds.zarrjava.indexing.OpenSlice;
-import com.squareup.okhttp.Call;
-import com.squareup.okhttp.OkHttpClient;
-import com.squareup.okhttp.Request;
-import com.squareup.okhttp.Response;
+import com.squareup.okhttp.*;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Optional;
 
-public class HttpStore extends Store {
+public class HttpStore implements Store {
 
+    @Nonnull
     private final OkHttpClient httpClient;
+    @Nonnull
     private final String path;
 
-    public HttpStore(String path) {
+    public HttpStore(@Nonnull String path) {
         this.httpClient = new OkHttpClient();
         this.path = path;
     }
@@ -47,7 +47,9 @@ public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
         Call call = httpClient.newCall(request);
         try {
             Response response = call.execute();
-            return Optional.of(ByteBuffer.wrap(response.body().bytes()));
+            try (ResponseBody body = response.body()) {
+                return Optional.of(ByteBuffer.wrap(body.bytes()));
+            }
         } catch (IOException e) {
             return Optional.empty();
         }

src/main/java/com/scalableminds/zarrjava/store/FileValueHandle.java

@@ -0,0 +1,77 @@
+package com.scalableminds.zarrjava.store;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.scalableminds.zarrjava.indexing.OpenSlice;
+import com.scalableminds.zarrjava.v3.Utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Optional;
+
+public class S3Store implements Store, Store.ListableStore {
+    @Nonnull
+    private final AmazonS3 s3client;
+    @Nonnull
+    private final String bucketName;
+    @Nullable
+    private final String prefix;
+
+    public S3Store(@Nonnull AmazonS3 s3client, @Nonnull String bucketName, @Nullable String prefix) {
+        this.s3client = s3client;
+        this.bucketName = bucketName;
+        this.prefix = prefix;
+    }
+
+    String dereferencePath(String key) {
+        if (prefix == null) {
+            return key;
+        }
+        if (key == null || key.length() == 0) {
+            return prefix;
+        }
+        return prefix + "/" + key;
+    }
+
+    @Override
+    public Optional<ByteBuffer> get(String key, OpenSlice byteRange) {
+        try (S3ObjectInputStream inputStream = s3client.getObject(bucketName,
+                dereferencePath(key)).getObjectContent()) {
+            return Optional.of(Utils.asByteBuffer(inputStream));
+        } catch (IOException e) {
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public void set(String key, ByteBuffer bytes, OpenSlice byteRange) {
+        try (InputStream byteStream = new ByteArrayInputStream(bytes.array())) {
+            s3client.putObject(bucketName, dereferencePath(key), byteStream, new ObjectMetadata());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void delete(String key) {
+
+    }
+
+    @Override
+    public Iterator<String> list(String key) {
+        final String fullKey = dereferencePath(key);
+        return s3client.listObjects(bucketName, fullKey).getObjectSummaries().stream().map(
+                p -> p.getKey().substring(fullKey.length() + 1)).iterator();
+    }
+
+    @Override
+    public String toString() {
+        return "s3://" + bucketName + "/" + prefix;
+    }
+}