Merge SSE and file lock updates to api (#149)

* merge SSE and File Lock updates to public repository

* update ChangeLog and version number to 5.0.0

* add pip3 install of setuptools_scm to travis.yml

* unset B2_APPLICATION_KEY before calling b2 upload_file in maybe_upload_build_results

Co-authored-by: Keith Felt <[email protected]>
Co-authored-by: keithele <[email protected]>
Co-authored-by: John Leacox <[email protected]>
Co-authored-by: mxue-BB <[email protected]>
Co-authored-by: Fabian <[email protected]>
Co-authored-by: fmbz <[email protected]>
Co-authored-by: malaysf <[email protected]>

Author: 6 people

.travis.yml

@@ -18,6 +18,7 @@ env:
 
 before_install:
   - mkdir -p ~/.gradle && echo "org.gradle.daemon=false" >> ~/.gradle/gradle.properties
+  - pip3 install "setuptools_scm[toml]<6.0"
   - pip3 install b2
 
 script:

CHANGELOG.md

@@ -1,20 +1,20 @@
 # Changelog
 
-## [Unreleased]
-
-## [5.0.0] - 2021-01-20
-### Added
-* added an asynchronous large file upload method `B2StorageClient.storeLargeFileFromLocalContentAsync`
+## [5.0.0] - 2021-05-10
 ### Changed `[Incompatible]`
-* Disable automatic decompressing compressed content in HTTP client download library
+* Disabled automatic decompression of compressed content in HTTP client download library
+* Added `updateFileRetention` and `updateLegalHold` to `B2StorageClient`
+* Added `storeLargeFileFromLocalContentAsync` to `B2StorageClient` to support asynchronous large file uploads
 
 ### Added
+* Added Server-Side Encryption (SSE) support to relevant API requests/responses
+* Added File Lock support to relevant API requests/responses
 * Set gradle targetCompatibility to 1.8
 * Added support for java.util.SortedMap interface
 * Support more than 64 fields per object
 
 ### Fixed
-* Fix regular expression that has an unescaped .
+* Fixed regular expression that had an unescaped '.'
 
 ## [4.0.0] - 2020-05-11
 ### Added

core/src/main/java/com/backblaze/b2/client/B2LargeFileStorer.java

@@ -9,16 +9,19 @@
 import com.backblaze.b2.client.exceptions.B2Exception;
 import com.backblaze.b2.client.exceptions.B2LocalException;
 import com.backblaze.b2.client.structures.B2CopyPartRequest;
+import com.backblaze.b2.client.structures.B2FileSseForRequest;
 import com.backblaze.b2.client.structures.B2FileVersion;
 import com.backblaze.b2.client.structures.B2FinishLargeFileRequest;
 import com.backblaze.b2.client.structures.B2Part;
+import com.backblaze.b2.client.structures.B2StoreLargeFileRequest;
 import com.backblaze.b2.client.structures.B2UploadListener;
 import com.backblaze.b2.client.structures.B2UploadPartRequest;
 import com.backblaze.b2.client.structures.B2UploadPartUrlResponse;
 import com.backblaze.b2.client.structures.B2UploadProgress;
 import com.backblaze.b2.client.structures.B2UploadState;
 import com.backblaze.b2.util.B2ByteProgressListener;
 import com.backblaze.b2.util.B2ByteRange;
+import com.backblaze.b2.util.B2Preconditions;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -31,6 +34,8 @@
 import java.util.concurrent.Future;
 import java.util.function.Supplier;
 
+import static com.backblaze.b2.client.structures.B2ServerSideEncryptionMode.SSE_C;
+
 /**
  * A class for handling the creation of large files.
  *
@@ -45,6 +50,13 @@ public class B2LargeFileStorer {
      */
     private final B2FileVersion fileVersion;
 
+    /**
+     * The B2FileSseForRequest for the large file that is being created.
+     * This contains the SSE-C parameters for SSE-C uploads and is null
+     * otherwise.
+     */
+    private final B2FileSseForRequest serverSideEncryptionOrNull;
+
     /**
      * The parts that need to be stored before finishing the large file.
      */
@@ -70,15 +82,17 @@ public class B2LargeFileStorer {
     private final ExecutorService executor;
 
     B2LargeFileStorer(
-            B2FileVersion fileVersion,
+            B2StoreLargeFileRequest storeLargeFileRequest,
             List<B2PartStorer> partStorers,
             B2AccountAuthorizationCache accountAuthCache,
             B2StorageClientWebifier webifier,
             B2Retryer retryer,
             Supplier<B2RetryPolicy> retryPolicySupplier,
             ExecutorService executor) {
+        B2Preconditions.checkArgumentIsNotNull(storeLargeFileRequest, "storeLargeFileRequest");
 
-        this.fileVersion = fileVersion;
+        this.fileVersion = storeLargeFileRequest.getFileVersion();
+        this.serverSideEncryptionOrNull = storeLargeFileRequest.getServerSideEncryption();
         this.partStorers = validateAndSortPartStorers(new ArrayList<>(partStorers));
         this.startingBytePositions = computeStartingBytePositions(partStorers);
 
@@ -141,7 +155,6 @@ List<B2PartStorer> getPartStorers() {
     long getStartByteOrUnknown(int partNumber) {
         return startingBytePositions.get(partNumber - 1);
     }
-
     public static B2LargeFileStorer forLocalContent(
             B2FileVersion largeFileVersion,
             B2ContentSource contentSource,
@@ -151,6 +164,27 @@ public static B2LargeFileStorer forLocalContent(
             B2Retryer retryer,
             Supplier<B2RetryPolicy> retryPolicySupplier,
             ExecutorService executor) throws B2Exception {
+        return forLocalContent(
+                B2StoreLargeFileRequest.builder(largeFileVersion).build(),
+                contentSource,
+                partSizes,
+                accountAuthCache,
+                webifier,
+                retryer,
+                retryPolicySupplier,
+                executor);
+    }
+
+    public static B2LargeFileStorer forLocalContent(
+            B2StoreLargeFileRequest storeLargeFileRequest,
+            B2ContentSource contentSource,
+            B2PartSizes partSizes,
+            B2AccountAuthorizationCache accountAuthCache,
+            B2StorageClientWebifier webifier,
+            B2Retryer retryer,
+            Supplier<B2RetryPolicy> retryPolicySupplier,
+            ExecutorService executor) throws B2Exception {
+        B2Preconditions.checkArgumentIsNotNull(storeLargeFileRequest, "storeLargeFileRequest");
 
         // Convert the contentSource into a list of B2PartStorer objects.
         final List<B2PartStorer> partContentSources = new ArrayList<>();
@@ -167,7 +201,7 @@ public static B2LargeFileStorer forLocalContent(
 
         // Instantiate and return the manager.
         return new B2LargeFileStorer(
-                largeFileVersion,
+                storeLargeFileRequest,
                 partContentSources,
                 accountAuthCache,
                 webifier,
@@ -395,6 +429,7 @@ B2Part uploadPart(
                                 new B2ContentSourceWithByteProgressListener(contentSource, progressListener);
                         final B2UploadPartRequest uploadPartRequest = B2UploadPartRequest
                                 .builder(partNumber, contentSourceThatReportsProgress)
+                                .setServerSideEncryption(serverSideEncryptionOrNull)
                                 .build();
 
                         updateProgress(

core/src/main/java/com/backblaze/b2/client/B2LargeFileUploader.java

@@ -9,9 +9,11 @@
 import com.backblaze.b2.client.contentSources.B2Headers;
 import com.backblaze.b2.client.exceptions.B2Exception;
 import com.backblaze.b2.client.exceptions.B2LocalException;
+import com.backblaze.b2.client.structures.B2FileSseForRequest;
 import com.backblaze.b2.client.structures.B2FileVersion;
 import com.backblaze.b2.client.structures.B2FinishLargeFileRequest;
 import com.backblaze.b2.client.structures.B2Part;
+import com.backblaze.b2.client.structures.B2ServerSideEncryptionMode;
 import com.backblaze.b2.client.structures.B2StartLargeFileRequest;
 import com.backblaze.b2.client.structures.B2UploadFileRequest;
 import com.backblaze.b2.client.structures.B2UploadListener;
@@ -134,6 +136,7 @@ B2FileVersion finishUploadingLargeFile(B2FileVersion largeFileVersion,
         // LARGE_FILE_SHA1 is a bit "special" since the SDK quietly adds it for the user.
         // we've checked LARGE_FILE_SHA1 above, so here, remove it and check any other entries against the request.
         final Map<String,String> infos = new TreeMap<>();
+        //noinspection CollectionAddAllCanBeReplacedWithConstructor
         infos.putAll(largeFileVersion.getFileInfo());
         infos.remove(B2Headers.LARGE_FILE_SHA1);
         throwIfMismatch("fileInfo", toString(request.getFileInfo()), toString(infos));
@@ -287,8 +290,17 @@ private B2Part uploadOnePart(B2UploadPartUrlCache uploadPartUrlCache,
                         }
                         source = new B2ContentSourceWithByteProgressListener(source, progressListener);
 
+                        // if original upload request includes SSE-C parameters, than we need to include those in
+                        // each uploadPart request as well
+                        final B2FileSseForRequest uploadFileSse = request.getServerSideEncryption();
+                        final B2FileSseForRequest uploadPartSse =
+                                (uploadFileSse != null && uploadFileSse.getMode().equals(B2ServerSideEncryptionMode.SSE_C))
+                                        ? uploadFileSse
+                                        : null;
+
                         final B2UploadPartRequest partRequest = B2UploadPartRequest
                                 .builder(partSpec.partNumber, source)
+                                .setServerSideEncryption(uploadPartSse)
                                 .build();
 
                         final B2Part part = webifier.uploadPart(uploadPartUrlResponse, partRequest);

core/src/main/java/com/backblaze/b2/client/B2StorageClient.java

@@ -38,11 +38,17 @@
 import com.backblaze.b2.client.structures.B2ListPartsRequest;
 import com.backblaze.b2.client.structures.B2ListUnfinishedLargeFilesRequest;
 import com.backblaze.b2.client.structures.B2StartLargeFileRequest;
+import com.backblaze.b2.client.structures.B2StoreLargeFileRequest;
 import com.backblaze.b2.client.structures.B2UpdateBucketRequest;
+import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldRequest;
+import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldResponse;
+import com.backblaze.b2.client.structures.B2UpdateFileRetentionRequest;
+import com.backblaze.b2.client.structures.B2UpdateFileRetentionResponse;
 import com.backblaze.b2.client.structures.B2UploadFileRequest;
 import com.backblaze.b2.client.structures.B2UploadListener;
 import com.backblaze.b2.client.structures.B2UploadPartUrlResponse;
 import com.backblaze.b2.client.structures.B2UploadUrlResponse;
+import com.backblaze.b2.util.B2Preconditions;
 
 import java.io.Closeable;
 import java.util.List;
@@ -265,6 +271,37 @@ B2FileVersion storeLargeFileFromLocalContent(
             B2UploadListener uploadListenerOrNull,
             ExecutorService executor) throws B2Exception;
 
+    /**
+     * Uploads the specified content source as separate parts to form a B2 large file,
+     * optionally allowing caller to pass SSE-C parameters to match those given to
+     * startLargeFile().
+     *
+     * This method assumes you have already called startLargeFile(). The return value
+     * of that call needs to be passed into this method as part of a
+     * B2StoreLargeFileRequest. However, this method will currently call finish file.
+     *
+     * XXX: should we switch to letting the caller finish the large file?
+     *
+     * @param storeLargeFileRequest The B2StoreLargeFileRequest for the large file
+     *                              getting stored. This is built from the return
+     *                              value of startLargeFile() and any other relevant
+     *                              parameters.
+     * @param contentSource The contentSource to upload.
+     * @param uploadListenerOrNull The object that handles upload progress events.
+     *                             This may be null if you do not need to be notified
+     *                             of progress events.
+     * @param executor The executor for uploading parts in parallel. The caller
+     *                 retains ownership of the executor and is responsible for
+     *                 shutting it down.
+     * @return The fileVersion of the large file after it has been finished.
+     * @throws B2Exception If there's trouble.
+     */
+    B2FileVersion storeLargeFileFromLocalContent(
+            B2StoreLargeFileRequest storeLargeFileRequest,
+            B2ContentSource contentSource,
+            B2UploadListener uploadListenerOrNull,
+            ExecutorService executor) throws B2Exception;
+
     /**
      * Initiates uploading the specified content source as separate parts to form a
      * B2 large file. This allows the upload to be cancelled partway through.
@@ -298,6 +335,41 @@ CompletableFuture<B2FileVersion> storeLargeFileFromLocalContentAsync(
             B2UploadListener uploadListenerOrNull,
             ExecutorService executor) throws B2Exception;
 
+    /**
+     * Initiates uploading the specified content source as separate parts to form a
+     * B2 large file. This allows the upload to be cancelled partway through.
+     *
+     * This method assumes you have already called startLargeFile(). The return value
+     * of that call needs to be passed into this method.
+     *
+     * The returned future can be cancelled and this will also attempt to stop any part
+     * uploads in progress.
+     *
+     * Cancelling after the b2_finish_large_file API call has been started will result in the
+     * future being cancelled, but the API call can still succeed. There is no way to tell from
+     * the future whether this is the case. The caller is responsible for checking and calling
+     * B2StorageClient.cancelLargeFile.
+     *
+     * @param storeLargeFileRequest The B2StoreLargeFileRequest for the large file
+     *                              getting stored. This is built from the return
+     *                              value of startLargeFile() and any other relevant
+     *                              parameters.
+     * @param contentSource The contentSource to upload.
+     * @param uploadListenerOrNull The object that handles upload progress events.
+     *                             This may be null if you do not need to be notified
+     *                             of progress events.
+     * @param executor The executor for uploading parts in parallel. The caller
+     *                 retains ownership of the executor and is responsible for
+     *                 shutting it down.
+     * @return CompletableFuture with the resulting B2FileVersion of the completed file
+     * @throws B2Exception on error
+     */
+    CompletableFuture<B2FileVersion> storeLargeFileFromLocalContentAsync(
+            B2StoreLargeFileRequest storeLargeFileRequest,
+            B2ContentSource contentSource,
+            B2UploadListener uploadListenerOrNull,
+            ExecutorService executor) throws B2Exception;
+
     /**
      * Stores a large file, where storing each part may involve different behavior
      * or byte sources.
@@ -337,6 +409,47 @@ B2FileVersion storeLargeFile(
             B2UploadListener uploadListenerOrNull,
             ExecutorService executor) throws B2Exception;
 
+    /**
+     * Stores a large file, where storing each part may involve different behavior
+     * or byte sources, optionally allowing caller to pass SSE-C parameters to match
+     * those given to startLargeFile().
+     *
+     * For example, this method supports the use case of making a copy of a file
+     * that mostly has not changed, and the user only wishes to upload the parts
+     * that have changed. In this case partStorers would be a mix of
+     * B2CopyingPartStorers and one or more B2UploadingPartStorers.
+     *
+     * Another use case would be reattempting an upload of a large file where some
+     * parts have completed, and some haven't. In this case, partStorers would
+     * be a mix of B2AlreadyStoredPartStorer and B2UploadingPartStorers.
+     *
+     * This method assumes you have already called startLargeFile(). The return value
+     * of that call needs to be passed into this method as part of a
+     * B2StoreLargeFileRequest. However, this method will currently call finish file.
+     * Note that each part, whether copied or uploaded,
+     * is still subject to the minimum part size.
+     *
+     * @param storeLargeFileRequest The B2StoreLargeFileRequest for the large file
+     *                              getting stored. This is built from the return
+     *                              value of startLargeFile() and any other relevant
+     *                              parameters.
+     * @param partStorers The list of objects that know how to store the part
+     *                    they are responsible for.
+     * @param uploadListenerOrNull The object that handles upload progress events.
+     *                             This may be null if you do not need to be notified
+     *                             of progress events.
+     * @param executor The executor for uploading parts in parallel. The caller
+     *                 retains ownership of the executor and is responsible for
+     *                 shutting it down.
+     * @return The fileVersion of the large file after it has been finished.
+     * @throws B2Exception If there's trouble.
+     */
+    B2FileVersion storeLargeFile(
+            B2StoreLargeFileRequest storeLargeFileRequest,
+            List<B2PartStorer> partStorers,
+            B2UploadListener uploadListenerOrNull,
+            ExecutorService executor) throws B2Exception;
+
     /**
      * Verifies that the given fileVersion represents an unfinished large file
      * and that the specified content is compatible-enough with the information
@@ -877,6 +990,25 @@ default String getDownloadByNameUrl(String bucketName,
      */
     B2FileVersion finishLargeFile(B2FinishLargeFileRequest request) throws B2Exception;
 
+    /**
+     * Updates the legal hold configuration of the specified file as described by the request.
+     *
+     * @param request specifies which file to update and how to update it.
+     * @return the new state of the file
+     * @throws B2Exception if there's any trouble.
+     * @see <a href="https://www.backblaze.com/b2/docs/b2_update_file_legal_hold.html">b2_update_file_legal_hold</a>
+     */
+    B2UpdateFileLegalHoldResponse updateFileLegalHold(B2UpdateFileLegalHoldRequest request) throws B2Exception;
+
+    /**
+     * Updates the file retention configuration of the specified file as described by the request.
+     *
+     * @param request specifies which file to update and how to update it.
+     * @return the new state of the file
+     * @throws B2Exception if there's any trouble.
+     * @see <a href="https://www.backblaze.com/b2/docs/b2_update_file_retention.html">b2_update_file_retention</a>
+     */
+    B2UpdateFileRetentionResponse updateFileRetention(B2UpdateFileRetentionRequest request) throws B2Exception;
 
     /**
      * Closes this instance, releasing resources.