Merge branch 'master' into replication-monitoring

Author: ppolewicz

.github/workflows/ci.yml

@@ -21,9 +21,10 @@ jobs:
         with:
           ignore_words_list: datas
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
       - name: Install dependencies
         run: python -m pip install --upgrade nox pip setuptools
       - name: Run linters
@@ -41,9 +42,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
       - name: Install dependencies
         run: python -m pip install --upgrade nox pip setuptools
       - name: Build the distribution
@@ -61,9 +63,10 @@ jobs:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
       - name: Install dependencies
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
         run: python -m pip install --upgrade nox pip setuptools
@@ -95,9 +98,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
       - name: Install dependencies
         run: python -m pip install --upgrade nox pip setuptools
       - name: Run unit tests
@@ -115,9 +119,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
       - name: Install dependencies
         env:
           DEBIAN_FRONTEND: noninteractive

CHANGELOG.md

@@ -11,20 +11,26 @@ though no known major changes to it are planned at this point. Early adopters ma
 feasible to rely this implementation already.
 
 ### Added
+* Add included_sources module, for keeping track of included modified third-party libraries
 * Add `include_existing_files` parameter to `ReplicationSetupHelper`
 
 ### Changed
+* Change the per part retry limit from 5 to 20 for data transfer operations. Please note that the retry system is not considered to be a part of the public interface and is subject to be adjusted
+* Do not wait more than 64 seconds between retry attempts (unless server asks for it)
+* On longer failures wait an additional (random, up to 1s) amount of time to prevent client synchronization
+* Flatten `ReplicationConfiguration` interface
 * Reorder actions of `ReplicationSetupHelper` to avoid zombie rules
 
 ### Fixed
 * Fix `AccountInfo.is_master_key()`
 * Fix docstring of `SqliteAccountInfo`
+* Fix lifecycle rule type in the docs
 
 ### Infrastructure
 * Add 3.11.0-beta.1 to CI
 * Change Sphinx major version from 5 to 6
-* Extracted folder/bucket scanning into a new `scan` module
-* The interface of `ReplicationConfiguration` was flattened
+* Extract folder/bucket scanning into a new `scan` module
+* Enable pip cache in CI
 
 ## [1.16.0] - 2022-04-27
 

b2sdk/LICENSE

@@ -0,0 +1 @@
+../LICENSE

b2sdk/_v3/__init__.py

@@ -222,6 +222,7 @@
 
 # other
 
+from b2sdk.included_sources import get_included_sources
 from b2sdk.b2http import B2Http
 from b2sdk.api_config import B2HttpApiConfig
 from b2sdk.api_config import DEFAULT_HTTP_API_CONFIG

b2sdk/api.py

@@ -221,7 +221,7 @@ def create_bucket(
         :param str bucket_type: a bucket type, could be one of the following values: ``"allPublic"``, ``"allPrivate"``
         :param dict bucket_info: additional bucket info to store with the bucket
         :param dict cors_rules: bucket CORS rules to store with the bucket
-        :param dict lifecycle_rules: bucket lifecycle rules to store with the bucket
+        :param list lifecycle_rules: bucket lifecycle rules to store with the bucket
         :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown)
         :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled
         :param b2sdk.v2.ReplicationConfiguration replication: bucket replication rules or ``None``

b2sdk/b2http.py

@@ -8,6 +8,7 @@
 #
 ######################################################################
 
+from random import random
 import io
 import json
 import logging
@@ -17,7 +18,7 @@
 import requests
 import time
 
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from .exception import (
     B2Error, B2RequestTimeoutDuringUpload, BadDateFormat, BrokenPipe, B2ConnectionError,
@@ -151,10 +152,17 @@ class B2Http:
        except B2Error as e:
            ...
 
+    Please note that the timeout/retry system, including class-level variables,
+    is not a part of the interface and is subject to change.
     """
 
-    # timeout for HTTP GET/POST requests
-    TIMEOUT = 1200  # 20 minutes as server-side copy can take time
+    TIMEOUT = 128
+    TIMEOUT_FOR_COPY = 1200  # 20 minutes as server-side copy can take time
+    TIMEOUT_FOR_UPLOAD = 128
+    TRY_COUNT_DATA = 20
+    TRY_COUNT_DOWNLOAD = 20
+    TRY_COUNT_HEAD = 5
+    TRY_COUNT_OTHER = 5
 
     def __init__(self, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG):
         """
@@ -176,7 +184,15 @@ def add_callback(self, callback):
         """
         self.callbacks.append(callback)
 
-    def post_content_return_json(self, url, headers, data, try_count=5, post_params=None):
+    def post_content_return_json(
+        self,
+        url,
+        headers,
+        data,
+        try_count: int = TRY_COUNT_DATA,
+        post_params=None,
+        _timeout: Optional[int] = None,
+    ):
         """
         Use like this:
 
@@ -202,7 +218,10 @@ def do_post():
             data.seek(0)
             self._run_pre_request_hooks('POST', url, request_headers)
             response = self.session.post(
-                url, headers=request_headers, data=data, timeout=self.TIMEOUT
+                url,
+                headers=request_headers,
+                data=data,
+                timeout=_timeout or self.TIMEOUT_FOR_UPLOAD,
             )
             self._run_post_request_hooks('POST', url, request_headers, response)
             return response
@@ -223,7 +242,7 @@ def do_post():
         finally:
             response.close()
 
-    def post_json_return_json(self, url, headers, params, try_count=5):
+    def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER):
         """
         Use like this:
 
@@ -241,10 +260,24 @@ def post_json_return_json(self, url, headers, params, try_count=5):
         :return: the decoded JSON document
         :rtype: dict
         """
+
+        # This is not just b2_copy_file or b2_copy_part, but it would not
+        # be good to find out by analyzing the url.
+        # In the future a more generic system between raw_api and b2http
+        # to indicate the timeouts should be designed.
+        timeout = self.TIMEOUT_FOR_COPY
+
         data = io.BytesIO(json.dumps(params).encode())
-        return self.post_content_return_json(url, headers, data, try_count, params)
+        return self.post_content_return_json(
+            url,
+            headers,
+            data,
+            try_count,
+            params,
+            _timeout=timeout,
+        )
 
-    def get_content(self, url, headers, try_count=5):
+    def get_content(self, url, headers, try_count: int = TRY_COUNT_DOWNLOAD):
         """
         Fetches content from a URL.
 
@@ -282,7 +315,12 @@ def do_get():
         response = self._translate_and_retry(do_get, try_count, None)
         return ResponseContextManager(response)
 
-    def head_content(self, url: str, headers: Dict[str, Any], try_count: int = 5) -> Dict[str, Any]:
+    def head_content(
+        self,
+        url: str,
+        headers: Dict[str, Any],
+        try_count: int = TRY_COUNT_HEAD,
+    ) -> Dict[str, Any]:
         """
         Does a HEAD instead of a GET for the URL.
         The response's content is limited to the headers.
@@ -413,6 +451,7 @@ def _translate_and_retry(cls, fcn, try_count, post_params=None):
         """
         # For all but the last try, catch the exception.
         wait_time = 1.0
+        max_wait_time = 64
         for _ in range(try_count - 1):
             try:
                 return cls._translate_errors(fcn, post_params)
@@ -426,11 +465,20 @@ def _translate_and_retry(cls, fcn, try_count, post_params=None):
                 else:
                     sleep_duration = wait_time
                     sleep_reason = 'that is what the default exponential backoff is'
+
                 logger.info(
-                    'Pausing thread for %i seconds because %s', sleep_duration, sleep_reason
+                    'Pausing thread for %i seconds because %s',
+                    sleep_duration,
+                    sleep_reason,
                 )
                 time.sleep(sleep_duration)
+
+                # Set up wait time for the next iteration
                 wait_time *= 1.5
+                if wait_time > max_wait_time:
+                    # avoid clients synchronizing and causing a wave
+                    # of requests when connectivity is restored
+                    wait_time = max_wait_time + random()
 
         # If the last try gets an exception, it will be raised.
         return cls._translate_errors(fcn, post_params)

b2sdk/bucket.py

@@ -81,7 +81,7 @@ def __init__(
         :param str type_: a bucket type
         :param dict bucket_info: an info to store with a bucket
         :param dict cors_rules: CORS rules to store with a bucket
-        :param dict lifecycle_rules: lifecycle rules to store with a bucket
+        :param list lifecycle_rules: lifecycle rules of the bucket
         :param int revision: a bucket revision number
         :param dict bucket_dict: a dictionary which contains bucket parameters
         :param set options_set: set of bucket options strings
@@ -145,7 +145,7 @@ def update(
         bucket_type: Optional[str] = None,
         bucket_info: Optional[dict] = None,
         cors_rules: Optional[dict] = None,
-        lifecycle_rules: Optional[dict] = None,
+        lifecycle_rules: Optional[list] = None,
         if_revision_is: Optional[int] = None,
         default_server_side_encryption: Optional[EncryptionSetting] = None,
         default_retention: Optional[BucketRetentionSetting] = None,

b2sdk/included_sources.py

@@ -0,0 +1,32 @@
+######################################################################
+#
+# File: b2sdk/included_sources.py
+#
+# Copyright 2021 Backblaze Inc. All Rights Reserved.
+#
+# License https://www.backblaze.com/using_b2_code.html
+#
+######################################################################
+
+# This module provides a list of third party sources included and modified in b2sdk, so it can be exposed to
+# B2 Command Line Tool for printing, for legal compliance reasons
+
+import dataclasses
+from typing import Dict, List
+
+_included_sources: List['IncludedSourceMeta'] = []
+
+
+@dataclasses.dataclass
+class IncludedSourceMeta:
+    name: str
+    comment: str
+    files: Dict[str, str]
+
+
+def add_included_source(src: IncludedSourceMeta):
+    _included_sources.append(src)
+
+
+def get_included_sources() -> List['IncludedSourceMeta']:
+    return _included_sources

b2sdk/v1/bucket.py

@@ -205,7 +205,7 @@ def update(
         bucket_type: Optional[str] = None,
         bucket_info: Optional[dict] = None,
         cors_rules: Optional[dict] = None,
-        lifecycle_rules: Optional[dict] = None,
+        lifecycle_rules: Optional[list] = None,
         if_revision_is: Optional[int] = None,
         default_server_side_encryption: Optional[v2.EncryptionSetting] = None,
         default_retention: Optional[v2.BucketRetentionSetting] = None,
@@ -217,7 +217,7 @@ def update(
         :param bucket_type: a bucket type, e.g. ``allPrivate`` or ``allPublic``
         :param bucket_info: an info to store with a bucket
         :param cors_rules: CORS rules to store with a bucket
-        :param lifecycle_rules: lifecycle rules to store with a bucket
+        :param lifecycle_rules: lifecycle rules of the bucket
         :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is*
         :param default_server_side_encryption: default server side encryption settings (``None`` if unknown)
         :param default_retention: bucket default retention setting

test/integration/test_raw_api.py

@@ -153,7 +153,12 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets):
         api_url,
         account_auth_token,
         account_id,
-        ['listBuckets', 'listFiles', 'readFiles', 'writeFiles'],
+        [
+            'listBuckets',
+            'listFiles',
+            'readFiles',
+            'writeFiles',  # Pawel @ 2022-06-21: adding this to make tests pass with a weird server validator
+        ],
         'testReplicationSourceKey',
         None,
         None,