Merge pull request #45 from JaneliaSciComp/s3-support

S3 support

Author: cgoina

modules/AdministrationGUI/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>org.janelia</groupId>
         <artifactId>workstation</artifactId>
-        <version>9.20.1</version>
+        <version>9.21.RC4</version>
         <relativePath>../..</relativePath>
     </parent>
 
     <name>AdministrationGUI</name>
     <groupId>org.janelia.workstation</groupId>
     <artifactId>adminstration</artifactId>
-    <version>9.20.1</version>
+    <version>9.21.RC4</version>
     <packaging>nbm</packaging>
 
     <dependencies>

modules/ColorDepthSearch/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>org.janelia</groupId>
         <artifactId>workstation</artifactId>
-        <version>9.20.1</version>
+        <version>9.21.RC4</version>
         <relativePath>../..</relativePath>
     </parent>
 
     <name>ColorDepthSearch</name>
     <groupId>org.janelia.workstation</groupId>
     <artifactId>colordepth</artifactId>
-    <version>9.20.1</version>
+    <version>9.21.RC4</version>
     <packaging>nbm</packaging>
 
     <dependencies>

modules/CommonGUI/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>org.janelia</groupId>
         <artifactId>workstation</artifactId>
-        <version>9.20.1</version>
+        <version>9.21.RC4</version>
         <relativePath>../..</relativePath>
     </parent>
 
     <name>CommonGUI</name>
     <groupId>org.janelia.workstation</groupId>
     <artifactId>common-gui</artifactId>
-    <version>9.20.1</version>
+    <version>9.21.RC4</version>
     <packaging>nbm</packaging>
 
     <dependencies>

modules/CommonLibraries/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>org.janelia</groupId>
         <artifactId>workstation</artifactId>
-        <version>9.20.1</version>
+        <version>9.21.RC4</version>
         <relativePath>../..</relativePath>
     </parent>
 
     <name>CommonLibraries</name>
     <groupId>org.janelia.workstation</groupId>
     <artifactId>libraries</artifactId>
-    <version>9.20.1</version>
+    <version>9.21.RC4</version>
     <packaging>nbm</packaging>
 
     <properties>
@@ -53,14 +53,6 @@
             <groupId>org.janelia.jacs-model</groupId>
             <artifactId>jacs-model-rendering</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.janelia.jacs-storage</groupId>
-            <artifactId>jacsstorage-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.janelia.jacs-storage</groupId>
-            <artifactId>jacsstorage-core</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.janelia.jacs-storage</groupId>
             <artifactId>jacsstorage-clients</artifactId>
@@ -776,6 +768,9 @@
                         <publicPackage>org.janelia.rendering</publicPackage>
                         <publicPackage>org.janelia.rendering.utils</publicPackage>
                         <publicPackage>org.janelia.rendering.ymlrepr</publicPackage>
+                        <publicPackage>org.janelia.jacsstorage.clients.api</publicPackage>
+                        <publicPackage>org.janelia.jacsstorage.clients.api.http</publicPackage>
+                        <publicPackage>org.janelia.jacsstorage.clients.api.rendering</publicPackage>
                         <publicPackage>org.jdesktop.swingx.*</publicPackage>
                         <publicPackage>org.jnp.*</publicPackage>
                         <publicPackage>org.perf4j</publicPackage>

modules/Core/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>org.janelia</groupId>
         <artifactId>workstation</artifactId>
-        <version>9.20.1</version>
+        <version>9.21.RC4</version>
         <relativePath>../..</relativePath>
     </parent>
 
     <name>Core</name>
     <groupId>org.janelia.workstation</groupId>
     <artifactId>core</artifactId>
-    <version>9.20.1</version>
+    <version>9.21.RC4</version>
     <packaging>nbm</packaging>
 
     <dependencies>

modules/Core/src/main/java/org/janelia/workstation/core/api/facade/impl/rest/AsyncServiceFacadeImpl.java

@@ -36,7 +36,10 @@ public AsyncServiceMonitoringWorker executeColorDepthService(ColorDepthSearch se
         // Invoke the service
         ActivityLogHelper.logUserAction("AsyncServiceFacadeImpl.executeColorDepthService", search);
         Long serviceId = asyncServiceClient.invokeService("colorDepthObjectSearch",
-                args, DEFAULT_PROCESSING_LOCATION, ImmutableMap.of());
+                args, DEFAULT_PROCESSING_LOCATION,
+                ImmutableMap.of(),
+                ImmutableMap.of() // invocation headers
+        );
 
         // Create a monitoring worker
         AsyncServiceMonitoringWorker executeWorker = new SearchMonitoringWorker(search, serviceId);

modules/Core/src/main/java/org/janelia/workstation/core/api/facade/impl/rest/DomainFacadeImpl.java

@@ -208,7 +208,7 @@ public void removeObjectStorage(List<String> storagePaths) {
         WebTarget storageService = RestJsonClientManager.getInstance().getTarget(remoteStorageUrl, true);
         for (String storagePath : storagePaths) {
             WebTarget target = storageService.path("storage_content/storage_path_redirect")
-                    .path(storagePath);
+                    .queryParam("contentPath", storagePath);
             Response response = target
                     .request("application/json")
                     .delete();

modules/Core/src/main/java/org/janelia/workstation/core/api/http/RestJsonClientManager.java

@@ -95,9 +95,11 @@ public boolean handleUnknownProperty(DeserializationContext ctxt, JsonParser jp,
                 return;
             }
 
+            log.info("Redirect to {}", clientResponseContext.getLocation());
             try (Response resp = clientRequestContext.getClient()
                     .target(clientResponseContext.getLocation())
                     .request()
+                    .headers(clientRequestContext.getHeaders())
                     .method(clientRequestContext.getMethod())) {
                 clientResponseContext.setEntityStream((InputStream) resp.getEntity());
                 clientResponseContext.setStatusInfo(resp.getStatusInfo());

modules/Core/src/main/java/org/janelia/workstation/core/api/web/AsyncServiceClient.java

@@ -7,6 +7,7 @@
 import java.util.Map;
 
 import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.Invocation;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response;
 
@@ -45,7 +46,8 @@ public AsyncServiceClient(String serverUrl) {
     public Long invokeService(String serviceName,
                               List<String> serviceArgs,
                               String processingLocation,
-                              Map<String, String> serviceResources) throws ServiceException {
+                              Map<String, String> serviceResources,
+                              Map<String, Object> invocationHeaders) throws ServiceException {
         AsyncServiceData body = new AsyncServiceData();
         if (serviceArgs != null) {
             body.args.addAll(serviceArgs);
@@ -57,9 +59,12 @@ public Long invokeService(String serviceName,
             body.processingLocation = processingLocation;
         }
         WebTarget target = service.path("async-services").path(serviceName);
-        Response response = target
-                .request("application/json")
-                .post(Entity.json(body));
+        Invocation.Builder requestInvocation = target
+                .request("application/json");
+        for (Map.Entry<String, Object> entry : invocationHeaders.entrySet()) {
+            requestInvocation = requestInvocation.header(entry.getKey(), String.valueOf(entry.getValue()));
+        }
+        Response response = requestInvocation.post(Entity.json(body));
         if (response.getStatus() != 201) {
             throw new ServiceException("Service " + serviceName + " returned status "+response.getStatus());
         }

modules/Core/src/main/java/org/janelia/workstation/core/api/web/JadeServiceClient.java

@@ -1,28 +1,24 @@
 package org.janelia.workstation.core.api.web;
 
 import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.nio.file.Paths;
-import java.util.List;
 import java.util.Optional;
 
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.Invocation;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.GenericType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
 
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
-
 import org.apache.commons.lang3.RegExUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.janelia.rendering.JADEBasedDataLocation;
+import org.janelia.jacsstorage.clients.api.JadeResults;
+import org.janelia.jacsstorage.clients.api.JadeStorageAttributes;
+import org.janelia.jacsstorage.clients.api.JadeStorageVolume;
+import org.janelia.jacsstorage.clients.api.http.HttpClientProvider;
+import org.janelia.jacsstorage.clients.api.rendering.JadeBasedDataLocation;
 import org.janelia.rendering.Streamable;
-import org.janelia.rendering.utils.ClientProxy;
-import org.janelia.rendering.utils.HttpClientProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,54 +29,18 @@ public class JadeServiceClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(JadeServiceClient.class);
 
-    @JsonIgnoreProperties(ignoreUnknown = true)
-    @JsonAutoDetect(
-            fieldVisibility = JsonAutoDetect.Visibility.ANY,
-            setterVisibility = JsonAutoDetect.Visibility.NONE
-    )
-    private static class JadeResults<T> {
-        @JsonProperty
-        @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
-        private List<T> resultList;
-    }
-
-    @JsonIgnoreProperties(ignoreUnknown = true)
-    @JsonAutoDetect(
-            fieldVisibility = JsonAutoDetect.Visibility.ANY,
-            setterVisibility = JsonAutoDetect.Visibility.NONE
-    )
-    private static class JadeStorageVolume {
-        @JsonProperty
-        private String id;
-        @JsonProperty
-        private String storageServiceURL;
-        @JsonProperty
-        private String baseStorageRootDir;
-        @JsonProperty
-        private String storageVirtualPath;
-
-        String getVolumeStorageURI() {
-            try {
-                return UriBuilder.fromUri(new URI(storageServiceURL)).path("agent_storage/storage_volume").path(id).build().toString();
-            } catch (URISyntaxException e) {
-                throw new IllegalArgumentException(e);
-            }
-        }
-
-    }
-
     private final String jadeURL; // jade master node URL
-    private final HttpClientProvider httpClientProvider;
+    private final HttpClientProvider clientProvider;
 
-    public JadeServiceClient(String jadeURL, HttpClientProvider httpClientProvider) {
+    public JadeServiceClient(String jadeURL, HttpClientProvider clientProvider) {
         Preconditions.checkArgument(StringUtils.isNotBlank(jadeURL));
         this.jadeURL = jadeURL;
-        this.httpClientProvider = httpClientProvider;
+        this.clientProvider = clientProvider;
     }
 
     public Optional<String> findStorageURL(String storagePath) {
         Preconditions.checkArgument(storagePath != null && storagePath.trim().length() > 0);
-        ClientProxy httpClient = getHttpClient();
+        Client httpClient = clientProvider.getClient();
         try {
             LOG.debug("Lookup storage for {}", storagePath);
             WebTarget target = httpClient.target(jadeURL)
@@ -95,46 +55,45 @@ public Optional<String> findStorageURL(String storagePath) {
             }
             JadeResults<JadeStorageVolume> storageContentResults = response.readEntity(new GenericType<JadeResults<JadeStorageVolume>>() {
             });
-            return storageContentResults.resultList.stream().findFirst().map(jadeVolume -> jadeVolume.storageServiceURL);
+            return storageContentResults.getResultList().stream().findFirst().map(JadeStorageVolume::getStorageServiceURL);
         } finally {
             httpClient.close();
         }
     }
 
-    public Optional<JADEBasedDataLocation> findDataLocation(String storagePathParam) {
+    public Optional<JadeBasedDataLocation> findDataLocation(String storagePathParam, JadeStorageAttributes storageAttributes) {
         Preconditions.checkArgument(storagePathParam != null && storagePathParam.trim().length() > 0);
         String storagePath = RegExUtils.replaceFirst(StringUtils.replaceChars(storagePathParam, '\\', '/'), "^((.+:)?/+)+", "/");
-        ClientProxy httpClient = getHttpClient();
+        Client httpClient = clientProvider.getClient();
         try {
             LOG.debug("Lookup storage for {}", storagePath);
             WebTarget target = httpClient.target(jadeURL)
                     .path("storage_volumes")
                     .queryParam("dataStoragePath", storagePath);
-            Response response = target.request()
-                    .get();
+            Response response = createRequest(target, storageAttributes).get();
             int responseStatus = response.getStatus();
             if (responseStatus != Response.Status.OK.getStatusCode()) {
                 LOG.error("Request to {} returned with status {}", target, responseStatus);
                 return Optional.empty();
             }
             JadeResults<JadeStorageVolume> storageContentResults = response.readEntity(new GenericType<JadeResults<JadeStorageVolume>>() {
             });
-            return storageContentResults.resultList.stream().findFirst()
+            return storageContentResults.getResultList().stream().findFirst()
                     .map(jadeVolume -> {
                         String renderedVolumePath;
-                        if (storagePath.startsWith(jadeVolume.storageVirtualPath)) {
-                            renderedVolumePath = Paths.get(jadeVolume.storageVirtualPath).relativize(Paths.get(storagePath)).toString();
+                        if (StringUtils.startsWith(storagePath, jadeVolume.getStorageVirtualPath())) {
+                            renderedVolumePath = Paths.get(jadeVolume.getStorageVirtualPath()).relativize(Paths.get(storagePath)).toString();
                         } else {
-                            renderedVolumePath = Paths.get(jadeVolume.baseStorageRootDir).relativize(Paths.get(storagePath)).toString();
+                            renderedVolumePath = Paths.get(jadeVolume.getBaseStorageRootDir()).relativize(Paths.get(storagePath)).toString();
                         }
-                        LOG.info("Create JADE volume location with URLs {}, {} and volume path {}", jadeVolume.storageServiceURL, jadeVolume.getVolumeStorageURI(), renderedVolumePath);
-                        return new JADEBasedDataLocation(
-                                jadeVolume.storageServiceURL,
+                        LOG.info("Create JADE volume location with URLs {}, {} and volume path {}", jadeVolume.getStorageServiceURL(), jadeVolume.getVolumeStorageURI(), renderedVolumePath);
+                        return new JadeBasedDataLocation(
+                                jadeVolume.getStorageServiceURL(),
                                 jadeVolume.getVolumeStorageURI(),
                                 renderedVolumePath,
                                 null,
                                 null,
-                                httpClientProvider);
+                                storageAttributes);
                     })
                     ;
         } finally {
@@ -143,17 +102,16 @@ public Optional<JADEBasedDataLocation> findDataLocation(String storagePathParam)
 
     }
 
-    public Streamable<InputStream> streamContent(String serverURL, String dataPath) {
+    public Streamable<InputStream> streamContent(String serverURL, String dataPath, JadeStorageAttributes storageAttributes) {
         Preconditions.checkArgument(serverURL != null && serverURL.trim().length() > 0);
         Preconditions.checkArgument(dataPath != null && dataPath.trim().length() > 0);
-        ClientProxy httpClient = getHttpClient();
+        Client httpClient = clientProvider.getClient();
         try {
             WebTarget target = httpClient.target(serverURL)
                     .path("agent_storage/storage_path/data_content")
                     .path(dataPath);
             LOG.info("Streaming tile from {}", target);
-            Response response = target.request()
-                    .get();
+            Response response = createRequest(target, storageAttributes).get();
             int responseStatus = response.getStatus();
             if (responseStatus == Response.Status.OK.getStatusCode()) {
                 InputStream is = (InputStream) response.getEntity();
@@ -167,16 +125,15 @@ public Streamable<InputStream> streamContent(String serverURL, String dataPath)
         }
     }
 
-    public boolean checkStoragePath(String storagePath) {
+    public boolean checkStoragePath(String storagePath, JadeStorageAttributes storageAttributes) {
         Preconditions.checkArgument(storagePath != null && storagePath.trim().length() > 0);
-        ClientProxy httpClient = getHttpClient();
+        Client httpClient = clientProvider.getClient();
         try {
             LOG.debug("Check if storage path exists {}", storagePath);
             WebTarget target = httpClient.target(jadeURL)
                     .path("storage_content/storage_path_redirect")
-                    .path(storagePath);
-            Response response = target.request()
-                    .head();
+                    .queryParam("contentPath", storagePath);
+            Response response = createRequest(target, storageAttributes).head();
             int responseStatus = response.getStatus();
             if (responseStatus != Response.Status.OK.getStatusCode()) {
                 LOG.error("Request to {} returned with status {}", target, responseStatus);
@@ -189,8 +146,14 @@ public boolean checkStoragePath(String storagePath) {
         }
     }
 
-    private ClientProxy getHttpClient() {
-        return httpClientProvider.getClient();
+    private Invocation.Builder createRequest(WebTarget target, JadeStorageAttributes storageAttributes) {
+        Invocation.Builder requestBuilder = target.request();
+        for (String storageAttribute : storageAttributes.getAttributeNames()) {
+            requestBuilder = requestBuilder.header(
+                    storageAttribute,
+                    storageAttributes.getAttributeValue(storageAttribute)
+            );
+        }
+        return requestBuilder;
     }
-
 }