Merge pull request #291 from rpmoore/folder_filter

Creating a map and filter that will take a list of Contents, Filter t…

Author: Denver

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java

@@ -21,11 +21,13 @@
 import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.serializer.XmlProcessingException;
+import com.spectralogic.ds3client.utils.Predicate;
 
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Path;
 import java.security.SignatureException;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.UUID;
 
@@ -277,6 +279,14 @@ public abstract Iterable<Contents> listObjects(final String bucket, final String
      */
     public abstract Iterable<Ds3Object> removePrefixFromDs3ObjectsList(final Iterable<Ds3Object> objectsList, final String prefix);
 
+    /**
+     *  Filter out folders from the object list.  Use this method when piping the list of objects from listObjects to a bulk job.
+      * @param objects
+     * @return
+     */
+    public abstract Iterable<Ds3Object> toDs3Iterable(final Iterable<Contents> objects);
+
+    public abstract Iterable<Ds3Object> toDs3Iterable(final Iterable<Contents> objects, final Predicate<Contents> filter);
     /**
      * Strip prefix from the beginning of objectName.  If objectName does not start with prefix, return objectName unmodified.
      * @param objectName

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java

@@ -1,6 +1,6 @@
 /*
  * ******************************************************************************
- *   Copyright 2014-2015 Spectra Logic Corporation. All Rights Reserved.
+ *   Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
  *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
  *   this file except in compliance with the License. A copy of the License is located at
  *
@@ -15,6 +15,8 @@
 
 package com.spectralogic.ds3client.helpers;
 
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
@@ -29,9 +31,11 @@
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.serializer.XmlProcessingException;
+import com.spectralogic.ds3client.utils.Predicate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -135,10 +139,7 @@ private Ds3ClientHelpers.Job innerStartReadAllJob(final String bucket, final Rea
             throws SignatureException, IOException, XmlProcessingException {
         final Iterable<Contents> contentsList = this.listObjects(bucket);
 
-        final List<Ds3Object> ds3Objects = new ArrayList<>();
-        for (final Contents objectApiBean : contentsList) {
-            ds3Objects.add(new Ds3Object(objectApiBean.getKey()));
-        }
+        final Iterable<Ds3Object> ds3Objects = this.toDs3Iterable(contentsList, FolderNameFilter.filter());
 
         return this.startReadJob(bucket, ds3Objects, options);
     }
@@ -268,4 +269,34 @@ public Iterable<Ds3Object> removePrefixFromDs3ObjectsList(final Iterable<Ds3Obje
         }
         return newObjectsList;
     }
+
+    @Override
+    public Iterable<Ds3Object> toDs3Iterable(final Iterable<Contents> objects) {
+        return toDs3Iterable(objects, null);
+    }
+
+    @Override
+    public Iterable<Ds3Object> toDs3Iterable(final Iterable<Contents> objects, final Predicate<Contents> filter) {
+        return FluentIterable.from(objects).filter(new com.google.common.base.Predicate<Contents>() {
+            @Override
+            public boolean apply(@Nullable final Contents input) {
+                return input == null;
+            }
+        }).filter(new com.google.common.base.Predicate<Contents>() {
+            @Override
+            public boolean apply(@Nullable final Contents input) {
+                if (filter != null) {
+                    return filter.test(input);
+                } else {
+                    return false; // do not filter anything if filter is null
+                }
+            }
+        }).transform(new Function<Contents, Ds3Object>() {
+            @Nullable
+            @Override
+            public Ds3Object apply(@Nullable final Contents input) {
+                return new Ds3Object(input.getKey(), input.getSize());
+            }
+        });
+    }
 }

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FolderNameFilter.java

@@ -0,0 +1,31 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers;
+
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.utils.Predicate;
+
+public class FolderNameFilter implements Predicate<Contents> {
+
+    @Override
+    public boolean test(final Contents contents) {
+        return contents.getKey().endsWith("/");
+    }
+
+    public static Predicate<Contents> filter() {
+        return new FolderNameFilter();
+    }
+}

ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/EmptySeekableByteChannel.java

@@ -0,0 +1,63 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.utils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+
+public class EmptySeekableByteChannel implements SeekableByteChannel {
+    private boolean open = true;
+    @Override
+    public int read(final ByteBuffer dst) throws IOException {
+        return 0;
+    }
+
+    @Override
+    public int write(final ByteBuffer src) throws IOException {
+        return 0;
+    }
+
+    @Override
+    public long position() throws IOException {
+        return 0;
+    }
+
+    @Override
+    public SeekableByteChannel position(final long newPosition) throws IOException {
+        return this;
+    }
+
+    @Override
+    public long size() throws IOException {
+        return 0;
+    }
+
+    @Override
+    public SeekableByteChannel truncate(final long size) throws IOException {
+        return this;
+    }
+
+    @Override
+    public boolean isOpen() {
+        return open;
+    }
+
+    @Override
+    public void close() throws IOException {
+        open = false;
+    }
+}

ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Predicate.java

@@ -0,0 +1,20 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.utils;
+
+public interface Predicate<T> {
+    boolean test(T t);
+}

ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FolderNameFilter_Test.java

@@ -0,0 +1,44 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers;
+
+import com.spectralogic.ds3client.models.Contents;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class FolderNameFilter_Test {
+    @Test
+    public void filterTest() {
+        final FolderNameFilter filter = new FolderNameFilter();
+
+        final Contents contents = new Contents();
+        contents.setKey("name/");
+
+        assertTrue(filter.test(contents));
+    }
+
+    @Test
+    public void noFilterTest() {
+        final FolderNameFilter filter = new FolderNameFilter();
+
+        final Contents contents = new Contents();
+        contents.setKey("name");
+
+        assertFalse(filter.test(contents));
+    }
+}