Merge pull request Backblaze#812 from Backblaze/fix-integration-tests

Fix integration tests

Author: ppolewicz

.github/workflows/ci.yml

@@ -47,8 +47,31 @@ jobs:
         run: python -m pip install --upgrade nox pip setuptools
       - name: Build the distribution
         run: nox -vs build
-  test:
+  cleanup_buckets:
     needs: lint
+    env:
+      B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
+      B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
+      - name: Install dependencies
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        run: python -m pip install --upgrade nox pip setuptools
+      - name: Find and remove old buckets
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        run: nox -vs cleanup_buckets
+  test:
+    needs: cleanup_buckets
     env:
       B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
       B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
@@ -79,7 +102,7 @@ jobs:
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
         run: nox -vs integration -- --cleanup
   test-linux-bundle:
-    needs: lint
+    needs: cleanup_buckets
     env:
       B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
       B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
@@ -111,7 +134,7 @@ jobs:
           if-no-files-found: warn
           retention-days: 7
   test-macos-and-windows-bundle:
-    needs: lint
+    needs: cleanup_buckets
     env:
       B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
       B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}

CHANGELOG.md

@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 As in 3.4.0, replication support may be unstable, however no backward-incompatible
 changes are currently planned.
-This version is pinned strictly to `b2-sdk-python==1.17.0` for the same reason.
+This version is pinned strictly to `b2-sdk-python==1.17.3` for the same reason.
 
 ### Added
 * Add `--write-buffer-size` parameter
@@ -17,6 +17,10 @@ This version is pinned strictly to `b2-sdk-python==1.17.0` for the same reason.
 
 ### Infrastructure
 * Try not to crash tests due to bucket name collision
+* Fix replication integration tests
+* Fix leaking buckets in integration tests
+* Limit number of workers for integration tests to 1 for now
+* Make integration tests remove buckets only based on name, not based on creation time
 * Add dependabot configuration
 
 ## [3.4.0] - 2022-05-04

b2/console_tool.py

@@ -2472,7 +2472,7 @@ def _setup_parser(cls, parser):
         super()._setup_parser(parser)
         parser.add_argument('source', metavar='SOURCE_BUCKET_NAME')
         parser.add_argument('--rule', metavar='REPLICATION_RULE_NAME', default=None)
-        parser.add_argument('--destination-profile', required=True)
+        parser.add_argument('--destination-profile')
         parser.add_argument('--dont-scan-destination', action='store_true')
         parser.add_argument(
             '--output-format', default='console', choices=('console', 'json', 'csv')

noxfile.py

@@ -13,7 +13,6 @@
 import subprocess
 
 from glob import glob
-from multiprocessing import cpu_count
 
 import nox
 
@@ -36,7 +35,7 @@
     "pytest==6.2.5",
     "pytest-cov==3.0.0",
     'pytest-xdist==2.5.0',
-    'filelock==3.6.0',
+    'backoff==2.1.2',
 ]
 REQUIREMENTS_BUILD = ['setuptools>=20.2']
 REQUIREMENTS_BUNDLE = [
@@ -73,13 +72,14 @@ def install_myself(session, extras=None):
     if extras:
         arg += '[%s]' % ','.join(extras)
 
+    session.install('-e', arg)
+
     if INSTALL_SDK_FROM:
         cwd = os.getcwd()
         os.chdir(INSTALL_SDK_FROM)
         session.run('pip', 'uninstall', 'b2sdk', '-y')
         session.run('python', 'setup.py', 'develop')
         os.chdir(cwd)
-    session.install('-e', arg)
 
 
 @nox.session(name='format', python=PYTHON_DEFAULT_VERSION)
@@ -156,9 +156,8 @@ def integration(session):
     """Run integration tests."""
     install_myself(session)
     session.install(*REQUIREMENTS_TEST)
-    session.run(
-        'pytest', '-s', '-n', str(min(cpu_count(), 8) * 5), *session.posargs, 'test/integration'
-    )
+    #session.run('pytest', '-s', '-x', '-v', '-n', '4', *session.posargs, 'test/integration')
+    session.run('pytest', '-s', '-x', '-v', *session.posargs, 'test/integration')
 
 
 @nox.session(python=PYTHON_VERSIONS)
@@ -172,6 +171,14 @@ def test(session):
         session.notify('integration')
 
 
+@nox.session(python=PYTHON_DEFAULT_VERSION)
+def cleanup_buckets(session):
+    """Remove buckets from previous test runs."""
+    install_myself(session)
+    session.install(*REQUIREMENTS_TEST)
+    session.run('pytest', '-s', '-x', *session.posargs, 'test/integration/cleanup_buckets.py')
+
+
 @nox.session
 def cover(session):
     """Perform coverage analysis."""

requirements.txt

@@ -1,5 +1,5 @@
 arrow>=1.0.2,<2.0.0
-b2sdk==1.17.2
+b2sdk==1.17.3
 docutils==0.16
 idna>=2.2.0; platform_system == 'Java'
 importlib-metadata>=3.3.0; python_version < '3.8'

test/integration/cleanup_buckets.py

@@ -0,0 +1,16 @@
+######################################################################
+#
+# File: test/integration/cleanup_buckets.py
+#
+# Copyright 2022 Backblaze Inc. All Rights Reserved.
+#
+# License https://www.backblaze.com/using_b2_code.html
+#
+######################################################################
+
+
+def test_cleanup_buckets(b2_api):
+    # this is not a test, but it is intended to be called
+    # via pytest because it reuses fixtures which have everything
+    # set up
+    b2_api.clean_buckets()

test/integration/conftest.py

@@ -7,19 +7,22 @@
 # License https://www.backblaze.com/using_b2_code.html
 #
 ######################################################################
+
+import contextlib
 import sys
 
 from os import environ, path
-from pathlib import Path
-from tempfile import TemporaryDirectory, gettempdir
+from tempfile import TemporaryDirectory
 
 import pytest
 
 from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR
-from filelock import FileLock
+from b2sdk.exception import BucketIdNotFound
 
 from .helpers import Api, CommandLine, bucket_name_part
 
+GENERAL_BUCKET_NAME_PREFIX = 'clitst'
+
 
 @pytest.hookimpl
 def pytest_addoption(parser):
@@ -50,17 +53,31 @@ def realm() -> str:
 
 @pytest.fixture(scope='function')
 def bucket_name(b2_api) -> str:
-    yield b2_api.create_bucket().name
+    bucket = b2_api.create_bucket()
+    yield bucket.name
+    with contextlib.suppress(BucketIdNotFound):
+        b2_api.clean_bucket(bucket)
+
+
+@pytest.fixture(scope='function')  # , autouse=True)
+def debug_print_buckets(b2_api):
+    print('-' * 30)
+    print('Buckets before test ' + environ['PYTEST_CURRENT_TEST'])
+    num_buckets = b2_api.count_and_print_buckets()
+    print('-' * 30)
+    try:
+        yield
+    finally:
+        print('-' * 30)
+        print('Buckets after test ' + environ['PYTEST_CURRENT_TEST'])
+        delta = b2_api.count_and_print_buckets() - num_buckets
+        print(f'DELTA: {delta}')
+        print('-' * 30)
 
 
 @pytest.fixture(scope='session')
-def general_bucket_name_prefix() -> str:
-    yield 'clitst'
-
-
-@pytest.fixture(scope='session')
-def this_run_bucket_name_prefix(general_bucket_name_prefix) -> str:
-    yield general_bucket_name_prefix + bucket_name_part(8)
+def this_run_bucket_name_prefix() -> str:
+    yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8)
 
 
 @pytest.fixture(scope='module')
@@ -91,45 +108,13 @@ def auto_change_account_info_dir(monkey_patch) -> dir:
 
 
 @pytest.fixture(scope='module')
-def b2_api(
-    application_key_id, application_key, realm, general_bucket_name_prefix,
-    this_run_bucket_name_prefix
-) -> Api:
+def b2_api(application_key_id, application_key, realm, this_run_bucket_name_prefix) -> Api:
     yield Api(
-        application_key_id, application_key, realm, general_bucket_name_prefix,
+        application_key_id, application_key, realm, GENERAL_BUCKET_NAME_PREFIX,
         this_run_bucket_name_prefix
     )
 
 
-@pytest.fixture(scope='module', autouse=True)
-def auto_clean_buckets(b2_api, request, testrun_uid):
-    """ Automatically clean buckets before and after the whole module testing """
-
-    lock_file = Path(gettempdir()) / f'{testrun_uid}.lock'
-
-    # lock file cannot be used to store info - use another file to track # of active workers
-    worker_count_file = Path(gettempdir()) / f'{testrun_uid}.active-workers-count.txt'
-
-    with FileLock(str(lock_file)):
-        if not worker_count_file.is_file():
-            b2_api.clean_buckets()
-            worker_count_file.write_text('1')
-
-        else:
-            worker_count = int(worker_count_file.read_text())
-            worker_count_file.write_text(str(worker_count + 1))
-
-    yield
-
-    if request.config.getoption('--cleanup'):
-        with FileLock(str(lock_file)):
-            worker_count = int(worker_count_file.read_text())
-            worker_count_file.write_text(str(worker_count - 1))
-            if worker_count == 1:
-                b2_api.clean_buckets()
-                worker_count_file.unlink()
-
-
 @pytest.fixture(scope='module')
 def b2_tool(
     request, application_key_id, application_key, realm, this_run_bucket_name_prefix

test/integration/helpers.py

@@ -20,16 +20,21 @@
 import threading
 
 from dataclasses import dataclass
+from datetime import datetime
 from os import environ, linesep, path
 from pathlib import Path
 from tempfile import gettempdir, mkdtemp
 from typing import List, Optional, Union
 
+import backoff
+
+from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound
 from b2sdk.v2 import ALL_CAPABILITIES, NO_RETENTION_FILE_SETTING, B2Api, Bucket, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, InMemoryAccountInfo, InMemoryCache, LegalHold, RetentionMode, SqliteAccountInfo, fix_windows_path_limit
-from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent
+from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests
 
 from b2.console_tool import Command, current_time_millis
 
+BUCKET_CLEANUP_PERIOD_MILLIS = 0
 ONE_HOUR_MILLIS = 60 * 60 * 1000
 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24
 
@@ -98,7 +103,7 @@ def _should_remove_bucket(self, bucket: Bucket):
         OLD_PATTERN = 'test-b2-cli-'
         if bucket.name.startswith(self.general_bucket_name_prefix) or bucket.name.startswith(OLD_PATTERN):  # yapf: disable
             if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info:
-                delete_older_than = current_time_millis() - ONE_HOUR_MILLIS
+                delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS
                 this_bucket_creation_time = bucket.bucket_info[BUCKET_CREATED_AT_MILLIS]
                 if int(this_bucket_creation_time) < delete_older_than:
                     return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}"
@@ -116,9 +121,24 @@ def clean_buckets(self):
                 continue
 
             print('Trying to remove bucket:', bucket.name, 'because', why)
-            self.clean_bucket(bucket)
+            try:
+                self.clean_bucket(bucket)
+            except (BucketIdNotFound, v3BucketIdNotFound):
+                print('It seems that bucket %s has already been removed' % (bucket.name,))
+        buckets = self.api.list_buckets()
+        print('Total bucket count after cleanup:', len(buckets))
+        for bucket in buckets:
+            print(bucket)
+
+    @backoff.on_exception(
+        backoff.expo,
+        TooManyRequests,
+        max_tries=8,
+    )
+    def clean_bucket(self, bucket: Union[Bucket, str]):
+        if isinstance(bucket, str):
+            bucket = self.api.get_bucket_by_name(bucket)
 
-    def clean_bucket(self, bucket: Bucket):
         files_leftover = False
         file_versions = bucket.ls(latest_only=False, recursive=True)
 
@@ -168,6 +188,14 @@ def clean_bucket(self, bucket: Bucket):
                 print('It seems that bucket %s has already been removed' % (bucket.name,))
         print()
 
+    def count_and_print_buckets(self) -> int:
+        buckets = self.api.list_buckets()
+        count = len(buckets)
+        print(f'Total bucket count at {datetime.now()}: {count}')
+        for i, bucket in enumerate(buckets, start=1):
+            print(f'- {i}\t{bucket.name} [{bucket.id_}]')
+        return count
+
 
 def print_text_indented(text):
     """