Merge pull request #318 from Backblaze/prepare-replication-prerelease

Replication prerelease

Author: ppolewicz

CHANGELOG.md

@@ -6,6 +6,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.16.0] - 2022-04-27
+
+This release contains a preview of replication support. It allows for basic
+usage of B2 replication feature (currently in closed beta).
+
+As the interface of the sdk (and the server api) may change, the replication
+support shall be considered PRIVATE interface and should be used with caution.
+Please consult the documentation on how to safely use the private api interface.
+
+Expect substantial amount of work on sdk interface:
+* The interface of `ReplicationConfiguration` WILL change
+* The interface of `FileVersion.replication_status` MIGHT change
+* The interface of `FileVersionDownload` MIGHT change
+
+### Added
+* Add basic replication support to `Bucket` and `FileVersion`
+* Add `is_master_key()` method to `AbstractAccountInfo`
+* Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES`
+* Add log tracing of `interpret_b2_error`
+* Add `ReplicationSetupHelper`
+
+### Fixed
+* Fix license test on Windows
+* Fix cryptic errors when running integration tests with a non-full key
+
 ## [1.15.0] - 2022-04-12
 
 ### Changed
@@ -22,9 +47,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add a parameter to customize `sync_policy_manager`
 * Add parameters to set the min/max part size for large file upload/copy methods
 * Add CopySourceTooBig exception
-* Add an option to set a custom file version class to FileVersionFactory
+* Add an option to set a custom file version class to `FileVersionFactory`
 * Add an option for B2Api to turn off hash checking for downloaded files
-* Add an option for B2Api to set write buffer size for DownloadedFile.save_to method
+* Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method
 * Add support for multiple profile files for SqliteAccountInfo
 
 ### Fixed

b2sdk/_v3/__init__.py

@@ -202,6 +202,15 @@
 from b2sdk.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider
 from b2sdk.sync.encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER
 
+# replication
+
+from b2sdk.replication.setting import ReplicationConfigurationFactory
+from b2sdk.replication.setting import ReplicationConfiguration
+from b2sdk.replication.setting import ReplicationSourceConfiguration
+from b2sdk.replication.setting import ReplicationRule
+from b2sdk.replication.setting import ReplicationDestinationConfiguration
+from b2sdk.replication.setup import ReplicationSetupHelper
+
 # other
 
 from b2sdk.b2http import B2Http

b2sdk/account_info/abstract.py

@@ -126,6 +126,9 @@ def is_same_account(self, account_id: str, realm: str) -> bool:
         except exception.MissingAccountData:
             return False
 
+    def is_master_key(self) -> bool:
+        return self.get_account_id() == self.get_application_key_id()
+
     @abstractmethod
     def get_account_id(self):
         """

b2sdk/api.py

@@ -16,6 +16,7 @@
 from .cache import AbstractCache
 from .bucket import Bucket, BucketFactory
 from .encryption.setting import EncryptionSetting
+from .replication.setting import ReplicationConfiguration
 from .exception import BucketIdNotFound, NonExistentBucket, RestrictedBucket
 from .file_lock import FileRetentionSetting, LegalHold
 from .file_version import DownloadVersionFactory, FileIdAndName, FileVersion, FileVersionFactory
@@ -211,6 +212,7 @@ def create_bucket(
         lifecycle_rules=None,
         default_server_side_encryption: Optional[EncryptionSetting] = None,
         is_file_lock_enabled: Optional[bool] = None,
+        replication: Optional[ReplicationConfiguration] = None,
     ):
         """
         Create a bucket.
@@ -222,6 +224,7 @@ def create_bucket(
         :param dict lifecycle_rules: bucket lifecycle rules to store with the bucket
         :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown)
         :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled
+        :param b2sdk.v2.ReplicationConfiguration replication: bucket replication rules or ``None``
         :return: a Bucket object
         :rtype: b2sdk.v2.Bucket
         """
@@ -236,6 +239,7 @@ def create_bucket(
             lifecycle_rules=lifecycle_rules,
             default_server_side_encryption=default_server_side_encryption,
             is_file_lock_enabled=is_file_lock_enabled,
+            replication=replication,
         )
         bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict(self, response)
         assert name == bucket.name, 'API created a bucket with different name\
@@ -323,7 +327,7 @@ def get_bucket_by_id(self, bucket_id: str) -> Bucket:
         # There is no such bucket.
         raise BucketIdNotFound(bucket_id)
 
-    def get_bucket_by_name(self, bucket_name):
+    def get_bucket_by_name(self, bucket_name: str):
         """
         Return the Bucket matching the given bucket_name.
 
@@ -526,12 +530,25 @@ def list_keys(self, start_application_key_id: Optional[str] = None
                 return
             start_application_key_id = next_application_key_id
 
+    def get_key(self, key_id: str) -> Optional[ApplicationKey]:
+        """
+        Gets information about a single key: it's capabilities, prefix, name etc
+
+        Returns `None` if the key does not exist.
+
+        Raises an exception if profile is not permitted to list keys.
+        """
+        return next(
+            self.list_keys(start_application_key_id=key_id),
+            None,
+        )
+
     # other
     def get_file_info(self, file_id: str) -> FileVersion:
         """
         Gets info about file version.
 
-        :param str file_id: the id of the file who's info will be retrieved.
+        :param str file_id: the id of the file whose info will be retrieved.
         """
         return self.file_version_factory.from_api_response(
             self.session.get_file_info_by_id(file_id)

b2sdk/application_key.py

@@ -65,6 +65,10 @@ def parse_response_dict(cls, response: dict):
                for key, value in optional_args.items() if value is not None},
         }
 
+    def has_capabilities(self, capabilities) -> bool:
+        """ checks whether the key has ALL of the given capabilities """
+        return len(set(capabilities) - set(self.capabilities)) == 0
+
     def as_dict(self):
         """Represent the key as a dict, like the one returned by B2 cloud"""
         mandatory_keys = {

b2sdk/bucket.py

@@ -9,27 +9,41 @@
 ######################################################################
 
 import logging
+
 from typing import Optional, Tuple
 
 from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
 from .encryption.types import EncryptionMode
-from .exception import BucketIdNotFound, CopySourceTooBig, FileNotPresent, FileOrBucketNotFound, UnexpectedCloudBehaviour, UnrecognizedBucketType
+from .exception import (
+    BucketIdNotFound,
+    CopySourceTooBig,
+    FileNotPresent,
+    FileOrBucketNotFound,
+    UnexpectedCloudBehaviour,
+    UnrecognizedBucketType,
+)
 from .file_lock import (
+    UNKNOWN_BUCKET_RETENTION,
     BucketRetentionSetting,
     FileLockConfiguration,
     FileRetentionSetting,
-    UNKNOWN_BUCKET_RETENTION,
     LegalHold,
 )
 from .file_version import DownloadVersion, FileVersion
 from .progress import AbstractProgressListener, DoNothingProgressListener
+from .replication.setting import ReplicationConfiguration, ReplicationConfigurationFactory
 from .transfer.emerge.executor import AUTO_CONTENT_TYPE
 from .transfer.emerge.write_intent import WriteIntent
 from .transfer.inbound.downloaded_file import DownloadedFile
 from .transfer.outbound.copy_source import CopySource
 from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile
-from .utils import B2TraceMeta, disable_trace, limit_trace_arguments
-from .utils import b2_url_encode, validate_b2_file_name
+from .utils import (
+    B2TraceMeta,
+    b2_url_encode,
+    disable_trace,
+    limit_trace_arguments,
+    validate_b2_file_name,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -58,6 +72,7 @@ def __init__(
         ),
         default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION,
         is_file_lock_enabled: Optional[bool] = None,
+        replication: Optional[ReplicationConfiguration] = None,
     ):
         """
         :param b2sdk.v2.B2Api api: an API object
@@ -73,6 +88,7 @@ def __init__(
         :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings
         :param b2sdk.v2.BucketRetentionSetting default_retention: default retention setting
         :param bool is_file_lock_enabled: whether file locking is enabled or not
+        :param b2sdk.v2.ReplicationConfiguration replication: replication rules for the bucket
         """
         self.api = api
         self.id_ = id_
@@ -87,6 +103,7 @@ def __init__(
         self.default_server_side_encryption = default_server_side_encryption
         self.default_retention = default_retention
         self.is_file_lock_enabled = is_file_lock_enabled
+        self.replication = replication
 
     def get_fresh_state(self) -> 'Bucket':
         """
@@ -98,15 +115,15 @@ def get_fresh_state(self) -> 'Bucket':
             raise BucketIdNotFound(self.id_)
         return buckets_found[0]
 
-    def get_id(self):
+    def get_id(self) -> str:
         """
         Return bucket ID.
 
         :rtype: str
         """
         return self.id_
 
-    def set_info(self, new_bucket_info, if_revision_is=None):
+    def set_info(self, new_bucket_info, if_revision_is=None) -> 'Bucket':
         """
         Update bucket info.
 
@@ -115,7 +132,7 @@ def set_info(self, new_bucket_info, if_revision_is=None):
         """
         return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is)
 
-    def set_type(self, bucket_type):
+    def set_type(self, bucket_type) -> 'Bucket':
         """
         Update bucket type.
 
@@ -132,7 +149,8 @@ def update(
         if_revision_is: Optional[int] = None,
         default_server_side_encryption: Optional[EncryptionSetting] = None,
         default_retention: Optional[BucketRetentionSetting] = None,
-    ):
+        replication: Optional[ReplicationConfiguration] = None,
+    ) -> 'Bucket':
         """
         Update various bucket parameters.
 
@@ -143,6 +161,7 @@ def update(
         :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is*
         :param default_server_side_encryption: default server side encryption settings (``None`` if unknown)
         :param default_retention: bucket default retention setting
+        :param replication: replication rules for the bucket;
         """
         account_id = self.api.account_info.get_account_id()
         return self.api.BUCKET_FACTORY_CLASS.from_api_bucket_dict(
@@ -157,6 +176,7 @@ def update(
                 if_revision_is=if_revision_is,
                 default_server_side_encryption=default_server_side_encryption,
                 default_retention=default_retention,
+                replication=replication,
             )
         )
 
@@ -936,6 +956,7 @@ def as_dict(self):
         result['defaultServerSideEncryption'] = self.default_server_side_encryption.as_dict()
         result['isFileLockEnabled'] = self.is_file_lock_enabled
         result['defaultRetention'] = self.default_retention.as_dict()
+        result['replication'] = self.replication and self.replication.as_dict()
 
         return result
 
@@ -967,32 +988,64 @@ def from_api_bucket_dict(cls, api, bucket_dict):
 
         .. code-block:: python
 
-           {
-               "bucketType": "allPrivate",
-               "bucketId": "a4ba6a39d8b6b5fd561f0010",
-               "bucketName": "zsdfrtsazsdfafr",
-               "accountId": "4aa9865d6f00",
-               "bucketInfo": {},
-               "options": [],
-               "revision": 1,
-               "defaultServerSideEncryption": {
-                   "isClientAuthorizedToRead" : true,
-                   "value": {
-                     "algorithm" : "AES256",
-                     "mode" : "SSE-B2"
-                   }
-               },
-               "fileLockConfiguration": {
-                   "isClientAuthorizedToRead": true,
-                   "value": {
-                       "defaultRetention": {
-                           "mode": null,
-                           "period": null
+            {
+                "bucketType": "allPrivate",
+                "bucketId": "a4ba6a39d8b6b5fd561f0010",
+                "bucketName": "zsdfrtsazsdfafr",
+                "accountId": "4aa9865d6f00",
+                "bucketInfo": {},
+                "options": [],
+                "revision": 1,
+                "defaultServerSideEncryption": {
+                    "isClientAuthorizedToRead" : true,
+                    "value": {
+                        "algorithm" : "AES256",
+                        "mode" : "SSE-B2"
+                    }
+                },
+                "fileLockConfiguration": {
+                    "isClientAuthorizedToRead": true,
+                    "value": {
+                        "defaultRetention": {
+                            "mode": null,
+                            "period": null
+                            },
+                            "isFileLockEnabled": false
+                        }
+                },
+                "replicationConfiguration": {
+                    "clientIsAllowedToRead": true,
+                    "value": {
+                        "asReplicationSource": {
+                            "replicationRules": [
+                                {
+                                    "destinationBucketId": "c5f35d53a90a7ea284fb0719",
+                                    "fileNamePrefix": "",
+                                    "includeExistingFiles": True,
+                                    "isEnabled": true,
+                                    "priority": 1,
+                                    "replicationRuleName": "replication-us-west"
+                                },
+                                {
+                                    "destinationBucketId": "55f34d53a96a7ea284fb0719",
+                                    "fileNamePrefix": "",
+                                    "includeExistingFiles": True,
+                                    "isEnabled": true,
+                                    "priority": 2,
+                                    "replicationRuleName": "replication-us-west-2"
+                                }
+                            ],
+                            "sourceApplicationKeyId": "10053d55ae26b790000000006"
                         },
-                        "isFileLockEnabled": false
+                        "asReplicationDestination": {
+                            "sourceToDestinationKeyMapping": {
+                                "10053d55ae26b790000000045": "10053d55ae26b790000000004",
+                                "10053d55ae26b790000000046": "10053d55ae26b790030000004"
+                            }
+                        }
                     }
-              }
-           }
+                }
+            }
 
         into a Bucket object.
 
@@ -1016,6 +1069,7 @@ def from_api_bucket_dict(cls, api, bucket_dict):
             raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`')
         default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict)
         file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict)
+        replication = ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value
         return cls.BUCKET_CLASS(
             api,
             bucket_id,
@@ -1030,4 +1084,5 @@ def from_api_bucket_dict(cls, api, bucket_dict):
             default_server_side_encryption,
             file_lock_configuration.default_retention,
             file_lock_configuration.is_file_lock_enabled,
+            replication,
         )