Merge pull request #473 from scribe/custom_delim

Custom Delimiter in S3

Author: rpmoore

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java

@@ -15,15 +15,18 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
-import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -33,10 +36,10 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.Iterator;
+import java.util.List;
 import java.util.UUID;
 
-import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
-import static com.spectralogic.ds3client.integration.Util.loadBookTestData;
+import static com.spectralogic.ds3client.integration.Util.*;
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.Assert.*;
 
@@ -47,6 +50,7 @@ public class Iterators_Test {
     private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
     private static final String TEST_ENV_NAME = "lazy_iterator_test";
     private static final int RETRIES = 5;
+    private static final String TEST_DELIMITER = null;
 
     private static TempStorageIds envStorageIds;
     private static UUID envDataPolicyId;
@@ -67,21 +71,39 @@ public static void teardown() throws IOException {
         LOG.info("Finished test teardown...");
     }
 
+    @Test
+    public void iterateWithDelimiter() throws IOException, URISyntaxException {
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
+            final Iterable<FileSystemKey>  fileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            final List<FileSystemKey> fileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            assertThat(fileSystemKeys.size(), is(4));
+
+            loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
+            final Iterable<FileSystemKey>  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            final List<FileSystemKey> anotherFileSystemKeys = Lists.newArrayList(anotherFileSystemKeysIterator);
+            assertThat(anotherFileSystemKeys.size(), is(5));
+        } finally {
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+        }
+    }
+
     @Test
     public void emptyGetBucket() throws IOException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 100;
 
-        emptyTest(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        emptyTest(new GetBucketKeyLoaderFactory<Contents>(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
     @Test
     public void singlePageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 100;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        paginate(new GetBucketKeyLoaderFactory<Contents>(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
 
@@ -90,13 +112,13 @@ public void multiPageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 2;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        paginate(new GetBucketKeyLoaderFactory<Contents>(CLIENT, TEST_ENV_NAME, prefix, nextMarker, TEST_DELIMITER, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
 
     }
 
     @Test
     public void failedRequestGetBucket() {
-        testFailedRequest(new GetBucketLoaderFactory(CLIENT, "Unknown_Bucket",null, null, 1000, 5));
+        testFailedRequest(new GetBucketKeyLoaderFactory<Contents>(CLIENT, "Unknown_Bucket",null, TEST_DELIMITER,null, 1000, 5, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
    @Test

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageIds.java

@@ -23,8 +23,8 @@
  * These IDs are stored for teardown of testing environment.
  */
 public class TempStorageIds {
-    private UUID storageDomainMemberId;
-    private UUID dataPersistenceRuleId;
+    private final UUID storageDomainMemberId;
+    private final UUID dataPersistenceRuleId;
 
     public TempStorageIds(final UUID storageDomainMemberId, final UUID dataPersistenceRuleId) {
         this.storageDomainMemberId = storageDomainMemberId;

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractPaginationResponse.java

@@ -19,8 +19,8 @@
 
 public abstract class AbstractPaginationResponse extends AbstractResponse implements PaginationResponse {
 
-    private Integer pagingTruncated;
-    private Integer pagingTotalResultCount;
+    private final Integer pagingTruncated;
+    private final Integer pagingTotalResultCount;
 
     public AbstractPaginationResponse(final Integer pagingTotalResultCount, final Integer pagingTruncated, final String checksum, final ChecksumType.Type checksumType) {
         super(checksum, checksumType);

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java

@@ -22,6 +22,7 @@
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.Predicate;
 
@@ -467,6 +468,46 @@ public abstract Iterable<Contents> listObjects(final String bucket, final String
      */
     public abstract Iterable<Ds3Object> listObjectsForDirectory(final Path directory) throws IOException;
 
+
+    /**
+     * The '/' character is traditionally used as a path delimiter, these methods provide '/' as a default delimiter.
+     * An S3 object like foo/bar/baz with the prefix of foo/ and the delimiter of '/' will act similar to a traditional file system
+     *  proving a common prefix for each 'directory' descending from the current folder of the prefix.
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     */
+    public abstract Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix) throws IOException;
+
+    /**
+     * Defaults to the path delimiter of '/'
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param nextMarker Specifies the key to start with
+     */
+    public abstract Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker) throws IOException;
+
+    /**
+     * Defaults to the path delimiter of '/'
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param nextMarker Specifies the key to start with when listing objects
+     * @param maxKeys Sets the maximum number of keys returned in the response body.
+     */
+    public abstract Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException;
+
+    /**
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param delimiter Specifies a path delimiter for the S3 query
+     * @param nextMarker Specifies the key to start with when listing objects
+     * @param maxKeys Sets the maximum number of keys returned in the response body.
+     */
+    public abstract Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
+
     /**
      * Returns an Iterable of {@link Ds3Object} that have a prefix added.
      */

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java

@@ -22,21 +22,23 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.HeadBucketRequest;
+import com.spectralogic.ds3client.commands.HeadBucketResponse;
+import com.spectralogic.ds3client.commands.PutBucketRequest;
+import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
-import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
-import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
 import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
 import com.spectralogic.ds3client.models.*;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.models.common.Range;
-import com.spectralogic.ds3client.models.bulk.*;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.slf4j.Logger;
@@ -56,6 +58,7 @@
 class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
     private final static Logger LOG = LoggerFactory.getLogger(Ds3ClientHelpersImpl.class);
     private final static int DEFAULT_LIST_OBJECTS_RETRIES = 5;
+    public final static String DEFAULT_DELIMITER = null;
 
     private final Ds3Client client;
     private final int maxChunkAttempts;
@@ -505,13 +508,13 @@ public Iterable<Contents> listObjects(final String bucket, final String keyPrefi
     @Override
     public Iterable<Contents> listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        return this.listObjects(bucket, keyPrefix, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES);
     }
 
     @Override
     public Iterable<Contents> listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, nextMarker, maxKeys, retries));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory<Contents>(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
     @Override
@@ -527,6 +530,24 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         return objects.build();
     }
 
+    @Override
+    public Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix) throws IOException {
+        return this.remoteListDirectory(bucket, keyPrefix, null);
+    }
+
+    @Override
+    public Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker) throws IOException {
+        return this.remoteListDirectory(bucket, keyPrefix, nextMarker,1000);
+    }
+    @Override
+    public Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
+        return remoteListDirectory(bucket,keyPrefix,"/", keyPrefix, maxKeys);
+    }
+
+    public Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory<FileSystemKey>(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction));
+    }
+
     public Iterable<Ds3Object> addPrefixToDs3ObjectsList(final Iterable<Ds3Object> objectsList, final String prefix) {
         final FluentIterable<Ds3Object> objectIterable = FluentIterable.from(objectsList);
 

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java renamed to ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java

@@ -12,15 +12,15 @@
  *   specific language governing permissions and limitations under the License.
  * ****************************************************************************
  */
-
 package com.spectralogic.ds3client.helpers.pagination;
 
+import com.google.common.base.Function;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.GetBucketRequest;
 import com.spectralogic.ds3client.commands.GetBucketResponse;
-import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.networking.FailedToGetBucketException;
 import com.spectralogic.ds3client.networking.TooManyRetriesException;
 import com.spectralogic.ds3client.utils.Guard;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
@@ -29,69 +29,66 @@
 import java.util.Collections;
 import java.util.List;
 
-public class GetBucketLoader implements LazyIterable.LazyLoader<Contents> {
+public class GetBucketKeyLoader<T> implements LazyIterable.LazyLoader<T> {
     private static final int DEFAULT_MAX_KEYS = 1000;
-
+    private final List<T> emptyList = Collections.emptyList();
     private final Ds3Client client;
     private final String bucket;
     private final String prefix;
+    private final String delimiter;
     private final int maxKeys;
     private final int retryCount;
-
+    private final Function<ListBucketResult, Iterable<T>> function;
     private String nextMarker;
     private boolean truncated;
     private boolean endOfInput = false;
 
-    public GetBucketLoader(final Ds3Client client, final String bucket, final String prefix, final String nextMarker, final int maxKeys, final int retryCount) {
+    GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function<ListBucketResult, Iterable<T>> function) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
-        this.maxKeys = maxKeys;
+        this.delimiter = delimiter;
+        this.nextMarker = nextMarker;
+        this.maxKeys = Math.min(maxKeys, DEFAULT_MAX_KEYS);
         this.retryCount = retryCount;
-
         this.nextMarker = nextMarker;
         this.truncated = nextMarker != null;
+        this.function = function;
+    }
+
+    private GetBucketRequest prepRequest() {
+        final GetBucketRequest request = new GetBucketRequest(bucket);
+        if (prefix != null) { request.withPrefix(prefix); }
+        if (truncated) { request.withMarker(nextMarker); }
+        if (delimiter != null) { request.withDelimiter(delimiter); }
+        request.withMaxKeys(maxKeys);
+        return request;
     }
 
     @Override
-    public List<Contents> getNextValues() {
-        if (endOfInput) {
-            return Collections.emptyList();
-        }
+    public Iterable<T> getNextValues() {
+        if (endOfInput) { return emptyList; }
         int retryAttempt = 0;
-        while(true) {
-            final GetBucketRequest request = new GetBucketRequest(bucket);
-            request.withMaxKeys(Math.min(maxKeys, DEFAULT_MAX_KEYS));
-            if (prefix != null) {
-                request.withPrefix(prefix);
-            }
-            if (truncated) {
-                request.withMarker(nextMarker);
-            }
-
+        while (retryCount > retryAttempt) {
+            final GetBucketRequest request = prepRequest();
+            final ListBucketResult result;
             final GetBucketResponse response;
             try {
                 response = this.client.getBucket(request);
-                final ListBucketResult result = response.getListBucketResult();
-
-                truncated = result.getTruncated();
-                this.nextMarker = result.getNextMarker();
-
-                if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
-                    endOfInput = true;
-                }
-
-                return result.getObjects();
             } catch (final FailedRequestException e) {
-              throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
+                throw new FailedToGetBucketException("Failed to get the list of objects due to a failed request", e);
             } catch (final IOException e) {
-                if (retryAttempt >= retryCount) {
-                    //TODO need a proxied test to validate this retry logic
-                    throw new TooManyRetriesException("Failed to get the next set of objects from the getBucket request after " + retryCount + " retries", e);
-                }
                 retryAttempt++;
+                continue;
             }
+            result = response.getListBucketResult();
+            this.truncated = result.getTruncated();
+            this.nextMarker = result.getNextMarker();
+            if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
+                endOfInput = true;
+            }
+            return function.apply(result);
         }
-
+        throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries");
     }
-}
+}