Merge pull request #559 from RachelTucker/jsdk-285

JSDK-285: Updating get job transfer methods to add version id when getting object

Author: rpmoore

ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java

@@ -18,13 +18,11 @@
 import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.Ds3ClientBuilder;
-import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Response;
-import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
+import com.spectralogic.ds3client.models.Bucket;
 import com.spectralogic.ds3client.models.Job;
 import com.spectralogic.ds3client.models.JobStatus;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -65,6 +63,22 @@ public static Ds3Client insecureFromEnv() {
         return builder.build();
     }
 
+    /**
+     * Goes through all buckets on a BP and attempts to delete them and all their contents. It cancels all active jobs
+     * associated with this bucket. Created for cleaning up after cascade failures in functional tests.
+     *
+     * USE ONLY WHEN DELETING ALL BUCKETS AND ALL CONTENT IS YOUR GOAL.
+     */
+    public static void forceCleanupBucketsOnBP(final Ds3Client client) throws IOException {
+        final GetBucketsSpectraS3Request request = new GetBucketsSpectraS3Request();
+        final GetBucketsSpectraS3Response response = client.getBucketsSpectraS3(request);
+
+        for (final Bucket bucket : response.getBucketListResult().getBuckets()) {
+            cancelAllJobsForBucket(client, bucket.getName());
+            client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucket.getName()).withForce(true));
+        }
+    }
+
     public static void assumeVersion1_2(final Ds3Client client) throws IOException {
         final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
                 new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java

@@ -22,26 +22,34 @@
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Request;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.helpers.FileObjectGetter;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.VersioningLevel;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.utils.ResourceUtils;
+import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static com.spectralogic.ds3client.integration.Util.*;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class VersionedObject_Test {
 
@@ -138,4 +146,46 @@ public void stageObjectsWithVersioning() throws IOException, URISyntaxException
             CLIENT.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(TEST_ENV_NAME).withForce(true));
         }
     }
+
+    @Test
+    public void getObjectsWithVersioningUsingHelpers() throws IOException, URISyntaxException, InterruptedException {
+        // Create temp directory for files we will retrieve from bp
+        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), null);
+
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            final String objectName = "object_with_versions";
+
+            // Put different content for object twice
+            loadTestBook(CLIENT, BOOKS[0], objectName, TEST_ENV_NAME); // putting beowulf as content
+            loadTestBook(CLIENT, BOOKS[1], objectName, TEST_ENV_NAME); // putting sherlock holmes as content
+
+            // Get the version of the objects
+            final GetBucketRequest getBucketRequest = new GetBucketRequest(TEST_ENV_NAME).withVersions(true);
+            final GetBucketResponse getBucketResponse = CLIENT.getBucket(getBucketRequest);
+
+            assertThat(getBucketResponse.getListBucketResult().getObjects().size(), is(0));
+            assertThat(getBucketResponse.getListBucketResult().getVersionedObjects().size(), is(2));
+
+            // Create bulk get job with the oldest version of the object
+            final List<Ds3Object> objects = getBucketResponse.getListBucketResult().getVersionedObjects().stream()
+                    .filter(obj -> !obj.getIsLatest()) // filters out the latest version of the object
+                    .map(obj -> new Ds3Object(obj.getKey(), obj.getVersionId()))
+                    .collect(Collectors.toList());
+
+            final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(TEST_ENV_NAME, objects);
+
+            readJob.transfer(new FileObjectGetter(tempDirectory));
+
+            // Check content is beowulf, which was the first version of content
+            final File originalFile = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + BOOKS[0]).toFile();
+            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), objectName).toFile();
+            assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));
+
+        } finally {
+            cancelAllJobsForBucket(CLIENT, TEST_ENV_NAME);
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+            FileUtils.deleteDirectory(tempDirectory.toFile());
+        }
+    }
 }

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java

@@ -92,6 +92,9 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
+        if (blob.getVersionId() != null) {
+            getObjectRequest.withVersionId(blob.getVersionId());
+        }
         getObjectRequest.withByteRanges(blobRanges);
 
         return getObjectRequest;

ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java

@@ -86,6 +86,10 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
+        if (jobPart.getBlob().getVersionId() != null) {
+            getObjectRequest.withVersionId(jobPart.getBlob().getVersionId());
+        }
+
         final ImmutableCollection<Range> rangesForBlob = StrategyUtils.getRangesForBlob(rangesForBlobs, blob);
 
         if (rangesForBlob != null) {