Java 8 and kotlin (#504)

* Adding java 8 and kotlin to the build.  Updated to the latest version of gradle.  Updated the listObjectsForDirectory helper method to be lazy.  Updated the start_job commands to fully pass the Iterable along instead of copying it to a List.  Added a paginated bucket delete call that minimizes the number of API calls that are used to delete all the objects in a bucket.  Fixed an issue in the payload generation of the bulk request body to work around an issue when trying to serialize the payload from an Iterable.  Updated the version of Jackson.

* Updating dependencies to their latest version.  Pulling most of the version information into a single file.  Fixed a bug in the network code introduced by reving the http client version.

* Updating the shadowjar build to reflect the most recent changes, and to make the service loader work correctly as well.

* Updating the shadowjar build more so that all the versions reference the same variable

* Working on the last few finishing touches on the shadowJar setup.  We're excluding the ds3-interfaces and ds3-utils projects from the metadata build since those classes must be included as apart of the ds3-sdk project, and will already exist.

* Updating the shadow build for the metadata module

* Removing some unused import.  Removing some un-needed type declarations.  Performing some minor code cleanup

* Adding copyright information

* Adding another missing copyright

* Updating to use a kotlin built in function instead of our own Guard class

* Removing a debug print statement

Author: rpmoore

.travis.yml

@@ -1,6 +1,4 @@
 language: java
 jdk:
   - oraclejdk8
-  - oraclejdk7
-  - openjdk7
 script: ./gradlew -S clean jar ds3-sdk:test ds3-utils:test ds3-metadata:test

build.gradle

@@ -13,45 +13,61 @@
  *  ****************************************************************************
  */
 
+buildscript {
+    ext.kotlin_version = '1.1.4-2'
+
+    repositories {
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '4.0.0'
+    version = '4.0.1'
 }
 
 subprojects {
     apply plugin: 'maven'
     apply plugin: 'java'
+    apply plugin: 'kotlin'
     apply plugin: 'findbugs'
 
-    sourceCompatibility = JavaVersion.VERSION_1_7
+    sourceCompatibility = JavaVersion.VERSION_1_8
     repositories {
         mavenCentral()
         mavenLocal()
     }
 
     dependencies {
-        compile     'org.slf4j:slf4j-api:1.7.22'
-        testCompile ('org.mockito:mockito-core:1.10.19') {
+        compile "org.slf4j:slf4j-api:$slf4jVersion"
+        testCompile ("org.mockito:mockito-core:$mockitoVersion") {
             exclude group: 'org.hamcrest'
         }
 
-        testCompile 'junit:junit:4.12'
-        testCompile 'org.slf4j:slf4j-simple:1.7.22'
+        testCompile "junit:junit:$junitVersion"
+        testCompile "org.slf4j:slf4j-simple:$slf4jVersion"
     }
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '3.5'
+    gradleVersion = '4.1'
 }
 
 project(':ds3-sdk') {
     dependencies {
+        compile project(':ds3-interfaces')
         compile project(':ds3-utils')
     }
 }
 
 project(':ds3-metadata') {
     dependencies {
+        compile project(':ds3-interfaces')
         compile project(':ds3-utils')
     }
 }

ds3-metadata/build.gradle

@@ -16,7 +16,7 @@
 buildscript {
     repositories { jcenter() }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
     }
 }
 
@@ -26,14 +26,19 @@ shadowJar {
     relocate 'org.apache', 'ds3metafatjar.org.apache'
     relocate 'com.google', 'ds3metafatjar.com.google'
     relocate 'com.sun.jna', 'ds3metafatjar.net.java.dev.jna'
+    relocate 'org.jetbrains', 'ds3metafatjar.org.jetbrains'
+    relocate 'org.intellij', 'ds3metafatjar.org.intellij'
+    relocate 'org.codehaus', 'ds3metafatjar.org.codehaus'
 
     dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
-        exclude(dependency('org.apache.commons:commons-lang3:3.0'))
+        exclude(project(":ds3-interfaces")) // this is being excluded since it must be used with the sdk, which already has this dependency included
+        exclude(project(":ds3-utils")) // this is being excluded since it must be used with the sdk, which already has this dependency included
+        exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion"))
+        exclude(dependency("org.mockito:mockito-core:$mockitoVersion"))
+        exclude(dependency("junit:junit:$junitVersion"))
+        exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion"))
+        exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion"))
+        exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion"))
     }
 }
 
@@ -42,10 +47,10 @@ artifacts {
 }
 
 dependencies {
-    compile     group: 'net.java.dev.jna', name: 'jna-platform', version: '4.2.2'
-    compile     group: 'net.java.dev.jna', name: 'jna', version: '4.2.2'
-    compile     'commons-io:commons-io:2.4'
-    compile     'org.apache.httpcomponents:httpclient:4.5.1'
+    compile     group: 'net.java.dev.jna', name: 'jna-platform', version: "$jnaVersion"
+    compile     group: 'net.java.dev.jna', name: 'jna', version: "$jnaVersion"
+    compile     "commons-io:commons-io:$commonsioVersion"
+    compile     "org.apache.httpcomponents:httpclient:$httpclientVersion"
 
-    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
+    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion"
 }

ds3-sdk-integration/build.gradle

@@ -14,9 +14,9 @@
  */
 
 dependencies {
-    compile 'commons-codec:commons-codec:1.10'
-    compile 'junit:junit:4.12'
-    testCompile 'org.hamcrest:hamcrest-library:1.3'
-    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
+    compile "commons-codec:commons-codec:$commonscodecVersion"
+    compile "junit:junit:$junitVersion"
+    testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion"
+    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion"
 }
 

ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java

@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.commands.DeleteBucketRequest;
 import com.spectralogic.ds3client.commands.DeleteObjectRequest;
 import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
+import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
 import com.spectralogic.ds3client.models.Contents;
@@ -40,7 +41,7 @@
 import static org.junit.Assume.assumeTrue;
 
 
-public class Util {
+public final class Util {
     private static final Logger LOG = LoggerFactory.getLogger(Util.class);
     public static final String RESOURCE_BASE_NAME = "books/";
     public static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"};
@@ -110,22 +111,11 @@ public static void loadBookTestDataWithPrefix(final Ds3Client client, final Stri
     }
 
     public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException {
-        final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
-
-        final Iterable<Contents> objects = helpers.listObjects(bucketName);
-        for(final Contents contents : objects) {
-            client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey()));
-        }
-
-        client.deleteBucket(new DeleteBucketRequest(bucketName));
+        Ds3ClientHelpers.wrap(client).deleteBucket(bucketName);
     }
 
     public static void deleteBucketContents(final Ds3Client client, final String bucketName) throws IOException {
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
-
-        final Iterable<Contents> objects = helpers.listObjects(bucketName);
-        for(final Contents contents : objects) {
-            client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey()));
-        }
+        DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName);
     }
 }

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java

@@ -30,7 +30,7 @@
  * and partition for use in the integration tests to avoid error if the BP does not currently
  * have a partition available for running the unit tests.
  */
-public class TempStorageUtil {
+public final class TempStorageUtil {
 
     private static final String DATA_POLICY_NAME = "_dp";
     private static final String STORAGE_DOMAIN_NAME = "_sd";

ds3-sdk-samples/build.gradle

@@ -18,5 +18,5 @@ apply plugin: 'application'
 mainClassName = 'com.spectralogic.ds3client.samples.PartialObjectGetExample'
 
 dependencies {
-    compile 'org.slf4j:slf4j-simple:1.7.22'
+    compile "org.slf4j:slf4j-simple:$slf4jVersion"
 }

ds3-sdk/build.gradle

@@ -19,25 +19,33 @@ import java.nio.file.Path
 buildscript {
     repositories { jcenter() }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
     }
 }
 
 apply plugin: 'com.github.johnrengelman.shadow'
 
 shadowJar {
     relocate 'com.google', 'ds3fatjar.com.google'
-    relocate 'org.apache', 'ds3fatjar.org.apache'
+    relocate 'org.jetbrains', 'ds3fatjar.org.jetbrains'
+    relocate 'org.intellij', 'ds3fatjar.org.intellij'
     relocate 'org.codehaus', 'ds3fatjar.org.codehaus'
+    relocate 'kotlin', 'ds3fatjar.kotlin'
+    relocate 'edu.umd', 'ds3fatjar.edu.emd'
+    relocate 'net.jcip', 'ds3fatjar.net.jcip'
+    relocate 'com.ctc', 'ds3fatjar.com.ctc'
+    relocate 'org.apache', 'ds3fatjar.org.apache'
     relocate 'com.fasterxml', 'ds3fatjar.com.fasterxml'
     dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
-        exclude(dependency('org.apache.commons:commons-lang3:3.0'))
+        exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion"))
+        exclude(dependency("org.mockito:mockito-core:$mockitoVersion"))
+        exclude(dependency("junit:junit:$junitVersion"))
+        exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion"))
+        exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion"))
+        exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion"))
     }
+
+    mergeServiceFiles()
 }
 
 artifacts {
@@ -78,10 +86,12 @@ jar {
 jar.dependsOn genConfigProperties
 
 dependencies {
-    compile     'org.apache.httpcomponents:httpclient:4.5.1'
+    compile     "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    compile     "org.apache.httpcomponents:httpclient:$httpclientVersion"
+    compile     "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion"
+    compile     "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion"
+    compile     "com.google.guava:guava:$guavaVersion"
     compile     'org.codehaus.woodstox:woodstox-core-asl:4.4.1'
-    compile     'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.7.1'
-    compile     'com.google.guava:guava:20.0'
     compile     'com.google.code.findbugs:annotations:3.0.1'
-    testCompile 'org.hamcrest:hamcrest-library:1.3'
+    testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion"
 }

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java

@@ -36,7 +36,7 @@ private static RequestHeaders buildDefaultHeaders() {
 
     @Override
     public String getContentType() {
-        return ContentType.APPLICATION_XML.toString();
+        return ContentType.APPLICATION_XML.getMimeType();
     }
 
     @Override

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java

@@ -508,6 +508,12 @@ public abstract Iterable<Contents> listObjects(final String bucket, final String
      */
     public abstract Iterable<FileSystemKey> remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
 
+    /**
+     *
+     * @param bucket
+     */
+    public abstract void deleteBucket(final String bucket) throws IOException;
+
     /**
      * Returns an Iterable of {@link Ds3Object} that have a prefix added.
      */
@@ -525,20 +531,12 @@ public abstract Iterable<Contents> listObjects(final String bucket, final String
     @SafeVarargs
     public final Iterable<Ds3Object> toDs3Iterable(final Iterable<Contents> objects, final Predicate<Contents>... filters) {
 
-        FluentIterable<Contents> fluentIterable = FluentIterable.from(objects).filter(new com.google.common.base.Predicate<Contents>() {
-            @Override
-            public boolean apply(@Nullable final Contents input) {
-                return input != null;
-            }
-        });
+        FluentIterable<Contents> fluentIterable = FluentIterable.from(objects).filter(input -> input != null);
 
         if (filters != null) {
             for (final Predicate<Contents> filter : filters) {
-                fluentIterable = fluentIterable.filter(new com.google.common.base.Predicate<Contents>() {
-                    @Override
-                    public boolean apply(@Nullable final Contents input) {
-                        return filter == null || filter.test(input); // do not filter anything if filter is null
-                    }
+                fluentIterable = fluentIterable.filter(input -> {
+                    return filter == null || filter.test(input); // do not filter anything if filter is null
                 });
             }
         }
@@ -585,4 +583,9 @@ public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInD
      * Creates a folder in the specified bucket
      */
     public abstract void createFolder(final String bucketName, final String folderName) throws IOException;
+
+    /**
+     * Returns the client being wrapped
+     */
+    public abstract Ds3Client getClient();
 }