Merging 5.0 branch into master

Author: RachelTucker

build.gradle

@@ -23,26 +23,28 @@ buildscript {
 
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
-        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3'
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.0'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2'
+        classpath "gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.2.0-beta"
     }
 }
 
 
 plugins {
-    id "com.palantir.git-version" version "0.10.0"
+    id "com.palantir.git-version" version "0.10.1"
 }
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '4.1.0'
+    version = '5.0.3'
 }
 
 subprojects {
     apply plugin: 'maven'
     apply plugin: 'java'
     apply plugin: 'kotlin'
     apply plugin: 'findbugs'
+    apply plugin: 'net.ossindex.audit'
 
     sourceCompatibility = JavaVersion.VERSION_1_8
     repositories {
@@ -62,7 +64,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.5.1'
+    gradleVersion = '4.7'
 }
 
 project(':ds3-sdk') {

contract/4_0_0_contract.xml

@@ -0,0 +1,67 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.integration;
+
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Creates a channel builder that generates random data. All object keys and sizes need to be specified.
+ */
+public class RandomDataChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
+
+    private static final int seed = 12345;
+    private final Map<String, Long> objectMap = new HashMap();
+
+    public RandomDataChannelBuilder() {
+    }
+
+    public RandomDataChannelBuilder(final List<Ds3Object> objects) {
+        for (final Ds3Object obj : objects) {
+            this.objectMap.put(obj.getName(), obj.getSize());
+        }
+    }
+
+    RandomDataChannelBuilder withObject(final String key, final Long size) {
+        this.objectMap.put(key, size);
+        return this;
+    }
+
+    @Override
+    public SeekableByteChannel buildChannel(final String key) throws IOException {
+        if (!this.objectMap.containsKey(key)) {
+            throw new IllegalArgumentException(String.format("Object with name '%s' was not defined for this channel builder.", key));
+        }
+
+        final Long size = this.objectMap.get(key);
+        final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(seed, size));
+        final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
+
+        final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(size.intValue());
+        channel.write(randomBuffer);
+
+        return channel;
+    }
+}

contract/5_0_x_contract.xml

@@ -15,24 +15,27 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.Ds3ClientBuilder;
-import com.spectralogic.ds3client.commands.DeleteBucketRequest;
-import com.spectralogic.ds3client.commands.DeleteObjectRequest;
-import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
-import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.Bucket;
+import com.spectralogic.ds3client.models.Job;
+import com.spectralogic.ds3client.models.JobStatus;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.ResourceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -60,6 +63,22 @@ public static Ds3Client insecureFromEnv() {
         return builder.build();
     }
 
+    /**
+     * Goes through all buckets on a BP and attempts to delete them and all their contents. It cancels all active jobs
+     * associated with this bucket. Created for cleaning up after cascade failures in functional tests.
+     *
+     * USE ONLY WHEN DELETING ALL BUCKETS AND ALL CONTENT IS YOUR GOAL.
+     */
+    public static void forceCleanupBucketsOnBP(final Ds3Client client) throws IOException {
+        final GetBucketsSpectraS3Request request = new GetBucketsSpectraS3Request();
+        final GetBucketsSpectraS3Response response = client.getBucketsSpectraS3(request);
+
+        for (final Bucket bucket : response.getBucketListResult().getBuckets()) {
+            cancelAllJobsForBucket(client, bucket.getName());
+            client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucket.getName()).withForce(true));
+        }
+    }
+
     public static void assumeVersion1_2(final Ds3Client client) throws IOException {
         final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
                 new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
@@ -79,6 +98,19 @@ public static void loadBookTestData(final Ds3Client client, final String bucketN
         LOG.info("Finished loading test data...");
     }
 
+    /**
+     * Loads a single test book to a BP with the specified object name.
+     */
+    public static void loadTestBook(final Ds3Client client, final String fileName, final String objectName, final String bucketName) throws IOException, URISyntaxException {
+        final Path filePath = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + fileName);
+        final Ds3Object obj = new Ds3Object(objectName, Files.size(filePath));
+
+        final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
+
+        final Ds3ClientHelpers.Job job = helpers.startWriteJob(bucketName, ImmutableList.of(obj));
+        job.transfer(key -> FileChannel.open(filePath, StandardOpenOption.READ));
+    }
+
     public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, URISyntaxException {
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
 
@@ -118,4 +150,19 @@ public static void deleteBucketContents(final Ds3Client client, final String buc
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
         DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName);
     }
+
+    // Cancels all in-progress jobs associated with a bucket
+    public static void cancelAllJobsForBucket(final Ds3Client client, final String bucketName) {
+        try {
+            final GetJobsSpectraS3Response getJobs = client.getJobsSpectraS3(new GetJobsSpectraS3Request().withBucketId(bucketName));
+
+            for (final Job job : getJobs.getJobListResult().getJobs()) {
+                if (job.getStatus() == JobStatus.IN_PROGRESS) {
+                    client.cancelJobSpectraS3(new CancelJobSpectraS3Request(job.getJobId()));
+                }
+            }
+        } catch (final IOException e) {
+            LOG.debug("Could not cancel jobs for bucket '%s': %s", bucketName, e.getMessage());
+        }
+    }
 }