Adding a read retry test for partial reads

Author: GraciesPadre

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java

@@ -34,6 +34,8 @@
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.Priority;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.models.bulk.PartialDs3Object;
+import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.utils.ResourceUtils;
 import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
@@ -44,17 +46,15 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.net.URISyntaxException;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 
 import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
@@ -140,33 +140,6 @@ public void createReadJob() throws IOException, InterruptedException, URISyntaxE
         assertThat(jobSpectraS3Response.getStatusCode(), is(200));
     }
 
-    private void putBigFile() throws IOException, URISyntaxException {
-        final String DIR_NAME = "largeFiles/";
-        final String[] FILE_NAMES = new String[] { "lesmis-copies.txt" };
-
-        final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);
-
-        final List<String> bookTitles = new ArrayList<>();
-        final List<Ds3Object> objects = new ArrayList<>();
-        for (final String book : FILE_NAMES) {
-            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
-            final long bookSize = Files.size(objPath);
-            final Ds3Object obj = new Ds3Object(book, bookSize);
-
-            bookTitles.add(book);
-            objects.add(obj);
-        }
-
-        final int maxNumBlockAllocationRetries = 1;
-        final int maxNumObjectTransferAttempts = 3;
-        final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
-                maxNumBlockAllocationRetries,
-                maxNumObjectTransferAttempts);
-
-        final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
-        writeJob.transfer(new FileObjectPutter(dirPath));
-    }
-
     @Test
     public void createReadJobWithBigFile() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
         putBigFile();
@@ -208,6 +181,34 @@ public void createReadJobWithBigFile() throws IOException, URISyntaxException, N
         }
     }
 
+    private void putBigFile() throws IOException, URISyntaxException {
+        final String DIR_NAME = "largeFiles/";
+        final String[] FILE_NAMES = new String[] { "lesmis-copies.txt" };
+
+        final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);
+
+        final List<String> bookTitles = new ArrayList<>();
+        final List<Ds3Object> objects = new ArrayList<>();
+        for (final String book : FILE_NAMES) {
+            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
+            final long bookSize = Files.size(objPath);
+            final Ds3Object obj = new Ds3Object(book, bookSize);
+
+            bookTitles.add(book);
+            objects.add(obj);
+        }
+
+        final int maxNumBlockAllocationRetries = 1;
+        final int maxNumObjectTransferAttempts = 3;
+        final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client,
+                maxNumBlockAllocationRetries,
+                maxNumObjectTransferAttempts);
+
+        final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
+        writeJob.transfer(new FileObjectPutter(dirPath));
+    }
+
+
     @Test
     public void createReadJobWithPriorityOption() throws IOException,
             InterruptedException, URISyntaxException {
@@ -245,4 +246,53 @@ public void createReadJobWithNameAndPriorityOptions() throws IOException,
         assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job"));
         assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW));
     }
+
+    @Test
+    public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException, URISyntaxException {
+        putBigFile();
+
+        final String tempPathPrefix = null;
+        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
+
+        try {
+            final List<Ds3Object> filesToGet = new ArrayList<>();
+
+            filesToGet.add(new PartialDs3Object("lesmis-copies.txt", Range.byLength(0, 100)));
+
+            filesToGet.add(new PartialDs3Object("lesmis-copies.txt", Range.byLength(100, 100)));
+
+            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);
+
+            final int maxNumBlockAllocationRetries = 1;
+            final int maxNumObjectTransferAttempts = 3;
+            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
+                    maxNumBlockAllocationRetries,
+                    maxNumObjectTransferAttempts);
+
+            final Ds3ClientHelpers.Job job = ds3ClientHelpers.startReadJob(BUCKET_NAME, filesToGet);
+
+            job.transfer(new FileObjectGetter(tempDirectory));
+
+            final String DIR_NAME = "largeFiles/";
+            final String FILE_NAME = "lesmis-copies.txt";
+
+            try (final InputStream originalFileStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(DIR_NAME + FILE_NAME)) {
+                final byte[] first200Bytes = new byte[200];
+                int numBytesRead = originalFileStream.read(first200Bytes, 0, 200);
+
+                assertThat(numBytesRead, is(200));
+
+                try (final InputStream fileReadFromBP = Files.newInputStream(Paths.get(tempDirectory.toString(), "lesmis-copies.txt"))) {
+                    final byte[] first200BytesFromBP = new byte[200];
+
+                    numBytesRead = fileReadFromBP.read(first200BytesFromBP, 0, 200);
+                    assertThat(numBytesRead, is(200));
+
+                    assertTrue(Arrays.equals(first200Bytes, first200BytesFromBP));
+                }
+            }
+        } finally {
+            FileUtils.deleteDirectory(tempDirectory.toFile());
+        }
+    }
 }