add range parameter in DownloadFileRequest

Author: huiguangjun

src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java

@@ -221,6 +221,7 @@ public int hashCode() {
             result = prime * result + (int) (end ^ (end >>> 32));
             result = prime * result + (int) (start ^ (start >>> 32));
             result = prime * result + (int) (crc ^ (crc >>> 32));
+            result = prime * result + (int) (fileStart ^ (fileStart >>> 32));
             return result;
         }
 
@@ -230,6 +231,7 @@ public int hashCode() {
         public boolean isCompleted; // flag of part download finished or not;
         public long length; // length of part
         public long crc; // part crc.
+        public long fileStart;  // start index in file, for range get
     }
 
     static class PartResult {
@@ -412,7 +414,8 @@ private DownloadFileResult downloadFileWithCheckpoint(DownloadFileRequest downlo
         }
 
         // check crc64
-        if(objectOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled()) {
+        if(objectOperation.getInnerClient().getClientConfiguration().isCrcCheckEnabled() &&
+           !hasRangeInRequest(downloadFileRequest)) {
             Long clientCRC = calcObjectCRCFromParts(downloadResult.getPartResults());
             try {
                 OSSUtils.checkChecksum(clientCRC, serverCRC, downloadResult.getObjectMetadata().getRequestId());
@@ -446,10 +449,17 @@ private void prepare(DownloadCheckPoint downloadCheckPoint, DownloadFileRequest
         downloadCheckPoint.bucketName = downloadFileRequest.getBucketName();
         downloadCheckPoint.objectKey = downloadFileRequest.getKey();
         downloadCheckPoint.objectStat = ObjectStat.getFileStat(objectOperation, downloadFileRequest);
-        downloadCheckPoint.downloadParts = splitFile(downloadCheckPoint.objectStat.size,
-                downloadFileRequest.getPartSize());
-
-        createFixedFile(downloadFileRequest.getTempDownloadFile(), downloadCheckPoint.objectStat.size);
+        long downloadSize;
+        if (downloadCheckPoint.objectStat.size > 0) {
+            long[] slice = getSlice(downloadFileRequest.getRange(), downloadCheckPoint.objectStat.size);
+            downloadCheckPoint.downloadParts = splitFile(slice[0], slice[1], downloadFileRequest.getPartSize());
+            downloadSize = slice[1];
+        } else {
+            //download whole file
+            downloadSize = 0;
+            downloadCheckPoint.downloadParts = splitOneFile();
+        }
+        createFixedFile(downloadFileRequest.getTempDownloadFile(), downloadSize);
     }
 
     public static void createFixedFile(String filePath, long length) throws IOException {
@@ -544,6 +554,10 @@ public int compare(PartResult p1, PartResult p2) {
         return downloadResult;
     }
 
+    private boolean hasRangeInRequest(DownloadFileRequest downloadFileRequest) {
+        return downloadFileRequest.getRange() != null;
+    }
+
     static class Task implements Callable<PartResult> {
 
         public Task(int id, String name, DownloadCheckPoint downloadCheckPoint, int partIndex,
@@ -569,7 +583,7 @@ public PartResult call() throws Exception {
                 tr = new PartResult(partIndex + 1, downloadPart.start, downloadPart.end);
 
                 output = new RandomAccessFile(downloadFileRequest.getTempDownloadFile(), "rw");
-                output.seek(downloadPart.start);
+                output.seek(downloadPart.fileStart);
 
                 GetObjectRequest getObjectRequest = new GetObjectRequest(downloadFileRequest.getBucketName(),
                         downloadFileRequest.getKey());
@@ -650,7 +664,7 @@ public ObjectMetadata GetobjectMetadata() {
         private ProgressListener progressListener;
     }
 
-    private ArrayList<DownloadPart> splitFile(long objectSize, long partSize) {
+    private ArrayList<DownloadPart> splitFile(long start, long objectSize, long partSize) {
         ArrayList<DownloadPart> parts = new ArrayList<DownloadPart>();
 
         long partNum = objectSize / partSize;
@@ -662,8 +676,9 @@ private ArrayList<DownloadPart> splitFile(long objectSize, long partSize) {
         for (int i = 0; offset < objectSize; offset += partSize, i++) {
             DownloadPart part = new DownloadPart();
             part.index = i;
-            part.start = offset;
-            part.end = getPartEnd(offset, objectSize, partSize);
+            part.start = offset + start;
+            part.end = getPartEnd(offset, objectSize, partSize) + start;
+            part.fileStart = offset;
             parts.add(part);
         }
 
@@ -677,6 +692,45 @@ private long getPartEnd(long begin, long total, long per) {
         return begin + per - 1;
     }
 
+    private ArrayList<DownloadPart> splitOneFile() {
+        ArrayList<DownloadPart> parts = new ArrayList<DownloadPart>();
+        DownloadPart part = new DownloadPart();
+        part.index = 0;
+        part.start = 0;
+        part.end = -1;
+        part.fileStart = 0;
+        parts.add(part);
+        return parts;
+    }
+
+    private long[] getSlice(long[] range, long totalSize) {
+        long start = 0;
+        long size = totalSize;
+
+        if ((range == null) ||
+            (range.length != 2) ||
+            (totalSize < 1) ||
+            (range[0] < 0 && range[1] < 0) ||
+            (range[0] > 0 && range[1] > 0 && range[0] > range[1])||
+            (range[0] >= totalSize)) {
+            //download all
+        } else {
+            //dwonload part by range & total size
+            long begin = range[0];
+            long end = range[1];
+            if (range[0] < 0) {
+                begin = 0;
+            }
+            if (range[1] < 0 || range[1] >= totalSize) {
+                end = totalSize -1;
+            }
+            start = begin;
+            size = end - begin + 1;
+        }
+
+        return new long[]{start, size};
+    }
+
     private boolean remove(String filePath) {
         boolean flag = false;
         File file = new File(filePath);

src/main/java/com/aliyun/oss/model/DownloadFileRequest.java

@@ -235,6 +235,46 @@ public int getTrafficLimit() {
         return trafficLimit;
     }
 
+    /**
+     * Gets the range of the object to download. The range is in the form of
+     * {start, end}---start and end is position of the object's content.
+     *
+     * @return The range of the object to download.
+     */
+    public long[] getRange() {
+        return range;
+    }
+
+    /**
+     * Sets the range of the object to download (optional).
+     *
+     * @param start
+     *            <p>
+     *            Start position
+     *            </p>
+     *            <p>
+     *            When the start is non-negative, it means the starting position
+     *            to download. When the start is -1, it means the range is
+     *            determined by the end only and the end could not be -1. For
+     *            example, when start is -1 and end is 100. It means the
+     *            download range will be the last 100 bytes.
+     *            </p>
+     * @param end
+     *            <p>
+     *            End position
+     *            </p>
+     *            <p>
+     *            When the end is non-negative, it means the ending position to
+     *            download. When the end is -1, it means the range is determined
+     *            by the start only and the start could not be -1. For example,
+     *            when end is -1 and start is 100. It means the download range
+     *            will be all exception first 100 bytes.
+     *            </p>
+     */
+    public void setRange(long start, long end) {
+        range = new long[] { start, end };
+    }
+
     // Part size in byte, by default it's 100KB.
     private long partSize = 1024 * 100;
     // Thread count for downloading parts, by default it's 1.
@@ -259,4 +299,6 @@ public int getTrafficLimit() {
 
     // Traffic limit speed, its uint is bit/s
     private int trafficLimit;
+
+    private long[] range;
 }

src/test/java/com/aliyun/oss/integrationtests/DownloadFileTest.java

@@ -20,6 +20,8 @@
 package com.aliyun.oss.integrationtests;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 
 import junit.framework.Assert;
 
@@ -171,4 +173,170 @@ public void testUploadFileWithCheckpointFile() {
         }
     }
 
+    static class RangeInfo {
+        RangeInfo(long start, long end) {
+            this.start = start;
+            this.end = end;
+        }
+        public long start; // start index;
+        public long end; // end index;
+    }
+
+    @Test
+    public void testDownloadFileWithRange() {
+        final String key = "obj-download-file-range-cp";
+
+        try {
+            File file = createSampleFile(key, 1024 * 500);
+
+            // upload file
+            UploadFileRequest uploadFileRequest = new UploadFileRequest(bucketName, key);
+            uploadFileRequest.setUploadFile(file.getAbsolutePath());
+            uploadFileRequest.setTaskNum(10);
+            uploadFileRequest.setEnableCheckpoint(true);
+            ObjectMetadata objMetadata = new ObjectMetadata();
+            objMetadata.addUserMetadata("prop", "propval");
+            uploadFileRequest.setObjectMetadata(objMetadata);
+
+            UploadFileResult uploadRes = ossClient.uploadFile(uploadFileRequest);
+            Assert.assertEquals(uploadRes.getMultipartUploadResult().getBucketName(), bucketName);
+            Assert.assertEquals(uploadRes.getMultipartUploadResult().getKey(), key);
+
+            //range
+            List<RangeInfo> rangeInfo = new ArrayList<RangeInfo>();
+            List<RangeInfo> expectRangeInfo = new ArrayList<RangeInfo>();
+            // case normal range
+            rangeInfo.add(new RangeInfo(20, 200*1024 + 100));
+            expectRangeInfo.add(new RangeInfo(20, 200*1024 + 100));
+
+            rangeInfo.add(new RangeInfo(100, 101));
+            expectRangeInfo.add(new RangeInfo(100, 101));
+
+            rangeInfo.add(new RangeInfo(200, 200));
+            expectRangeInfo.add(new RangeInfo(200, 200));
+
+            //to end
+            rangeInfo.add(new RangeInfo(102402, -1));
+            expectRangeInfo.add(new RangeInfo(102402, file.length() -1));
+
+            //end > size
+            rangeInfo.add(new RangeInfo(102402, file.length() + 20));
+            expectRangeInfo.add(new RangeInfo(102402, file.length() -1));
+
+            //start < 0, end > 0 &&  end < size
+            rangeInfo.add(new RangeInfo(-1, 300*1024 + 100));
+            expectRangeInfo.add(new RangeInfo(0, 300*1024 + 100));
+
+            //start < 0, end > size
+            rangeInfo.add(new RangeInfo(-1, file.length()));
+            expectRangeInfo.add(new RangeInfo(0, file.length() -1));
+
+            //start < 0, end < 0
+            rangeInfo.add(new RangeInfo(-1, -1));
+            expectRangeInfo.add(new RangeInfo(0, file.length() -1));
+
+            //start >= size
+            rangeInfo.add(new RangeInfo(file.length(), file.length() + 20));
+            expectRangeInfo.add(new RangeInfo(0, file.length() -1));
+
+            //start > end
+            rangeInfo.add(new RangeInfo(1024, 100));
+            expectRangeInfo.add(new RangeInfo(0, file.length() -1));
+
+            Assert.assertEquals(rangeInfo.size(), expectRangeInfo.size());
+            Assert.assertEquals(true, rangeInfo.size() > 0);
+
+            for (int i = 0; i < rangeInfo.size(); i++) {
+                // download file
+                String filePathNew = key + "-" + i +"-new.txt";
+                long start = rangeInfo.get(i).start;
+                long end = rangeInfo.get(i).end;
+                DownloadFileRequest downloadFileRequest = new DownloadFileRequest(bucketName, key);
+                downloadFileRequest.setDownloadFile(filePathNew);
+                downloadFileRequest.setTaskNum(10);
+                downloadFileRequest.setEnableCheckpoint(true);
+                downloadFileRequest.setRange(start, end);
+
+                DownloadFileResult downloadRes = ossClient.downloadFile(downloadFileRequest);
+
+                ObjectMetadata objMeta = downloadRes.getObjectMetadata();
+                //Assert.assertEquals(objMeta.getContentLength(), downloadFileRequest.getPartSize());
+                Assert.assertEquals(objMeta.getObjectType(), "Multipart");
+                Assert.assertEquals(objMeta.getUserMetadata().get("prop"), "propval");
+
+                File fileNew = new File(filePathNew);
+                long expectStart = expectRangeInfo.get(i).start;
+                long expectEnd = expectRangeInfo.get(i).end;
+                Assert.assertTrue("comparte file", compareFileWithRange(file.getAbsolutePath(), expectStart, expectEnd, fileNew.getAbsolutePath()));
+                fileNew.delete();
+            }
+            ossClient.deleteObject(bucketName, key);
+        } catch (Throwable e) {
+            e.printStackTrace();
+            Assert.fail(e.getMessage());
+            ossClient.deleteBucket(bucketName);
+        }
+    }
+
+    @Test
+    public void testDownloadEmptyFileWithoutCheckpoint() {
+        final String key = "obj-download-empty-file-wcp";
+
+        try {
+            File file = createSampleFile(key, 0);
+            Assert.assertEquals(0, file.length());
+
+            // upload file
+            UploadFileRequest uploadFileRequest = new UploadFileRequest(bucketName, key);
+            uploadFileRequest.setUploadFile(file.getAbsolutePath());
+            uploadFileRequest.setTaskNum(10);
+            ObjectMetadata objMetadata = new ObjectMetadata();
+            objMetadata.addUserMetadata("prop", "propval");
+            uploadFileRequest.setObjectMetadata(objMetadata);
+
+            UploadFileResult uploadRes = ossClient.uploadFile(uploadFileRequest);
+            Assert.assertEquals(uploadRes.getMultipartUploadResult().getBucketName(), bucketName);
+            Assert.assertEquals(uploadRes.getMultipartUploadResult().getKey(), key);
+
+            // download file
+            String filePathNew = key + "-new.txt";
+            DownloadFileRequest downloadFileRequest = new DownloadFileRequest(bucketName, key);
+            downloadFileRequest.setDownloadFile(filePathNew);
+            downloadFileRequest.setTaskNum(10);
+
+            DownloadFileResult downloadRes = ossClient.downloadFile(downloadFileRequest);
+
+            ObjectMetadata objMeta = downloadRes.getObjectMetadata();
+            Assert.assertEquals(objMeta.getContentLength(), 0);
+            Assert.assertEquals(objMeta.getObjectType(), "Multipart");
+            Assert.assertEquals(objMeta.getUserMetadata().get("prop"), "propval");
+
+            File fileNew = new File(filePathNew);
+            Assert.assertEquals(0, fileNew.length());
+            fileNew.delete();
+
+            //download file with range
+            filePathNew = key + "range-new.txt";
+            downloadFileRequest = new DownloadFileRequest(bucketName, key);
+            downloadFileRequest.setDownloadFile(filePathNew);
+            downloadFileRequest.setTaskNum(10);
+            downloadFileRequest.setRange(0, 0);
+            downloadRes = ossClient.downloadFile(downloadFileRequest);
+
+            objMeta = downloadRes.getObjectMetadata();
+            Assert.assertEquals(objMeta.getContentLength(), 0);
+            Assert.assertEquals(objMeta.getObjectType(), "Multipart");
+            Assert.assertEquals(objMeta.getUserMetadata().get("prop"), "propval");
+
+            fileNew = new File(filePathNew);
+            Assert.assertEquals(0, fileNew.length());
+            fileNew.delete();
+
+            ossClient.deleteObject(bucketName, key);
+        } catch (Throwable e) {
+            e.printStackTrace();
+            Assert.fail(e.getMessage());
+            ossClient.deleteBucket(bucketName);
+        }
+    }
 }