Merge pull request #1441 from digital-preservation/increase-window-size

Use larger window size.

Author: MancunianSam

droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java

@@ -91,7 +91,7 @@ public final class DroidAPI implements AutoCloseable {
     private static final String OLE2_PUID = "fmt/111";
     private static final String S3_SCHEME = "s3";
     private static final String GZIP_PUID = "x-fmt/266";
-    private static final long DEFAULT_MAX_BYTES_TO_SCAN = 65536;
+    private static final long DEFAULT_MAX_BYTES_TO_SCAN = -1;
 
     private static final AtomicLong ID_GENERATOR = new AtomicLong();
 

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReader.java

@@ -49,14 +49,16 @@ public class S3WindowReader extends AbstractReader implements SoftWindowRecovery
 
     private static final int BUFFER_LENGTH = 8192;
 
+    private static final int WINDOW_SIZE = 4 * 1024 * 1024;
+
     private final S3Utils.S3ObjectMetadata s3ObjectMetadata;
 
     private final S3Client s3Client;
 
     private final Long length;
 
     public S3WindowReader(WindowCache cache, S3Utils.S3ObjectMetadata s3ObjectMetadata, S3Client s3Client) {
-        super(cache);
+        super(WINDOW_SIZE, cache);
         this.s3Client = s3Client;
         this.length = s3ObjectMetadata.contentLength();
         this.s3ObjectMetadata = s3ObjectMetadata;
@@ -81,7 +83,6 @@ private byte[] bytesForRange(long windowStart) throws IOException {
                 .key(key)
                 .range("bytes=" + windowStart + "-" + (windowStart + this.windowSize -1))
                 .build();
-
         try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(getS3ObjectRequest)) {
             return toByteArray(response);
         }

droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequestTest.java

@@ -67,7 +67,7 @@ public void testS3IdentificationRequestGetsFirstWindowOnCreation() throws IOExce
         verify(mockS3Client, times(1)).getObject(getObjectRequestCaptor.capture());
 
         GetObjectRequest getRequestValue = getObjectRequestCaptor.getValue();
-        assertEquals(getRequestValue.range(), "bytes=0-4095");
+        assertEquals(getRequestValue.range(), "bytes=0-4194303");
     }
 
     @Test

droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReaderTest.java

@@ -35,13 +35,15 @@
 import net.byteseek.io.reader.windows.SoftWindow;
 import net.byteseek.io.reader.windows.Window;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.Invocation;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Uri;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
 
 import java.net.URI;
+import java.util.Collection;
 import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -50,6 +52,7 @@
 import static org.mockito.Mockito.when;
 import static uk.gov.nationalarchives.droid.core.interfaces.resource.S3TestUtils.mockS3Client;
 
+
 public class S3WindowReaderTest {
 
     @Test
@@ -70,6 +73,20 @@ public void testWindowReaderReturnsExpectedWindow() throws Exception {
         }
     }
 
+    @Test
+    public void testWindowReaderReturnsExpectedWindowForLargeFile() throws Exception {
+        WindowCache windowCache = mock(WindowCache.class);
+        S3Client s3Client = mockS3Client();
+        S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
+        S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 4L, 1L);
+        S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
+
+        s3WindowReader.createWindow(0);
+        Collection<Invocation> invocations = Mockito.mockingDetails(s3Client).getInvocations();
+        GetObjectRequest getObjectRequest = (GetObjectRequest)invocations.stream().toList().getFirst().getArguments()[0];
+        assertEquals(getObjectRequest.range(), "bytes=0-" + ((4 * 1024 * 1024) - 1));
+    }
+
     @Test
     public void testWindowReaderReturnsNullIfPositionLessThanZero() throws Exception {
         WindowCache windowCache = mock(WindowCache.class);