diff --git a/CHANGELOG.md b/CHANGELOG.md index 375f658d3..60be55792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Infrastructure * Autocomplete integration tests will now work properly even if tested package has not been installed * Automatically set copyright date when generating the docs +* Use ApiTestManager from b2-sdk in integration tests for common logic ## [3.9.0] - 2023-04-28 diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 45adf743a..0bdaec745 100644 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -13,4 +13,4 @@ def test_cleanup_buckets(b2_api): # this is not a test, but it is intended to be called # via pytest because it reuses fixtures which have everything # set up - b2_api.clean_buckets() + b2_api.clean_all_buckets() diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 5e7e3e7c4..96b321b86 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -8,21 +8,20 @@ # ###################################################################### -import base64 -import contextlib import sys +import uuid +from contextlib import suppress from os import environ, path from tempfile import TemporaryDirectory import pytest -from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR -from b2sdk.exception import BadRequest, BucketIdNotFound +from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, Bucket +from b2sdk.exception import BucketIdNotFound +from b2sdk.test.api_test_manager import ApiTestManager -from .helpers import Api, CommandLine, bucket_name_part - -GENERAL_BUCKET_NAME_PREFIX = 'clitst' +from test.integration.helpers import CommandLine @pytest.hookimpl @@ -53,17 +52,10 @@ def realm() -> str: @pytest.fixture(scope='function') -def bucket(b2_api) -> str: - try: - bucket = b2_api.create_bucket() - except BadRequest as e: - if e.code != 'too_many_buckets': - raise - num_buckets = b2_api.count_and_print_buckets() - print('current number of buckets:', num_buckets) - raise +def bucket(b2_api) -> Bucket: + bucket = b2_api.create_test_bucket() yield bucket - with contextlib.suppress(BucketIdNotFound): + with suppress(BucketIdNotFound): b2_api.clean_bucket(bucket) @@ -74,7 +66,7 @@ def bucket_name(bucket) -> str: @pytest.fixture(scope='function') def file_name(bucket) -> str: - file_ = bucket.upload_bytes(b'test_file', f'{bucket_name_part(8)}.txt') + file_ = bucket.upload_bytes(b'test_file', f'{uuid.uuid4()}.txt') yield file_.file_name @@ -94,11 +86,6 @@ def debug_print_buckets(b2_api): print('-' * 30) -@pytest.fixture(scope='session') -def this_run_bucket_name_prefix() -> str: - yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) - - @pytest.fixture(scope='module') def monkey_patch(): """ Module-scope monkeypatching (original `monkeypatch` is function-scope) """ @@ -127,23 +114,14 @@ def auto_change_account_info_dir(monkey_patch) -> dir: @pytest.fixture(scope='module') -def b2_api(application_key_id, application_key, realm, this_run_bucket_name_prefix) -> Api: - yield Api( - application_key_id, application_key, realm, GENERAL_BUCKET_NAME_PREFIX, - this_run_bucket_name_prefix - ) +def b2_api(application_key_id, application_key, realm) -> ApiTestManager: + yield ApiTestManager(application_key_id, application_key, realm) @pytest.fixture(scope='module') -def global_b2_tool( - request, application_key_id, application_key, realm, this_run_bucket_name_prefix -) -> CommandLine: +def global_b2_tool(request, application_key_id, application_key, realm) -> CommandLine: tool = CommandLine( - request.config.getoption('--sut'), - application_key_id, - application_key, - realm, - this_run_bucket_name_prefix, + request.config.getoption('--sut'), application_key_id, application_key, realm ) tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check) return tool diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 11ba84250..0ab7bc626 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -15,27 +15,21 @@ import random import re import shutil -import string import subprocess import sys import threading -import time -from dataclasses import dataclass -from datetime import datetime from os import environ, linesep, path from pathlib import Path from tempfile import gettempdir, mkdtemp from typing import List, Optional, Union -import backoff import pytest -from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound -from b2sdk.v2 import ALL_CAPABILITIES, NO_RETENTION_FILE_SETTING, B2Api, Bucket, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, InMemoryAccountInfo, InMemoryCache, LegalHold, RetentionMode, SqliteAccountInfo, fix_windows_path_limit -from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests +from b2sdk.v2 import ALL_CAPABILITIES, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, \ + SqliteAccountInfo, fix_windows_path_limit -from b2.console_tool import Command, current_time_millis +from b2.console_tool import Command logger = logging.getLogger(__name__) @@ -43,8 +37,6 @@ ONE_HOUR_MILLIS = 60 * 60 * 1000 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 -BUCKET_NAME_LENGTH = 50 -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' BUCKET_CREATED_AT_MILLIS = 'created_at_millis' SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,) @@ -63,171 +55,6 @@ key=EncryptionKey(secret=os.urandom(32), key_id='another-user-generated-key-id') ) -RNG_SEED = '_'.join( - [ - os.getenv('GITHUB_REPOSITORY', ''), - os.getenv('GITHUB_SHA', ''), - os.getenv('GITHUB_RUN_ID', ''), - os.getenv('GITHUB_RUN_ATTEMPT', ''), - os.getenv('GITHUB_JOB', ''), - os.getenv('GITHUB_ACTION', ''), - str(os.getpid()), # for local runs with xdist - str(time.time()), - ] -) - -RNG = random.Random(RNG_SEED) - -RNG_COUNTER = 0 - - -def bucket_name_part(length: int) -> str: - assert length >= 1 - global RNG_COUNTER - RNG_COUNTER += 1 - name_part = ''.join(RNG.choice(BUCKET_NAME_CHARS) for _ in range(length)) - logger.info('RNG_SEED: %s', RNG_SEED) - logger.info('RNG_COUNTER: %i, length: %i', RNG_COUNTER, length) - logger.info('name_part: %s', name_part) - return name_part - - -@dataclass -class Api: - account_id: str - application_key: str - realm: str - general_bucket_name_prefix: str - this_run_bucket_name_prefix: str - - api: B2Api = None - - def __post_init__(self): - info = InMemoryAccountInfo() - cache = InMemoryCache() - self.api = B2Api(info, cache=cache) - self.api.authorize_account(self.realm, self.account_id, self.application_key) - assert BUCKET_NAME_LENGTH - len( - self.this_run_bucket_name_prefix - ) > 5, self.this_run_bucket_name_prefix - - def create_bucket(self) -> Bucket: - for _ in range(10): - bucket_name = self.this_run_bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) - ) - print('Creating bucket:', bucket_name) - try: - return self.api.create_bucket( - bucket_name, - 'allPublic', - bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}, - ) - except DuplicateBucketName: - pass - print() - - raise ValueError('Failed to create bucket due to name collision') - - def _should_remove_bucket(self, bucket: Bucket): - if bucket.name.startswith(self.this_run_bucket_name_prefix): - return True, 'it is a bucket for this very run' - OLD_PATTERN = 'test-b2-cli-' - if bucket.name.startswith(self.general_bucket_name_prefix) or bucket.name.startswith(OLD_PATTERN): # yapf: disable - if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: - delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS - this_bucket_creation_time = bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] - if int(this_bucket_creation_time) < delete_older_than: - return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}" - else: - return True, 'undefined ' + BUCKET_CREATED_AT_MILLIS - return False, '' - - def clean_buckets(self): - buckets = self.api.list_buckets() - print('Total bucket count:', len(buckets)) - for bucket in buckets: - should_remove, why = self._should_remove_bucket(bucket) - if not should_remove: - print(f'Skipping bucket removal: "{bucket.name}"') - continue - - print('Trying to remove bucket:', bucket.name, 'because', why) - try: - self.clean_bucket(bucket) - except (BucketIdNotFound, v3BucketIdNotFound): - print('It seems that bucket %s has already been removed' % (bucket.name,)) - buckets = self.api.list_buckets() - print('Total bucket count after cleanup:', len(buckets)) - for bucket in buckets: - print(bucket) - - @backoff.on_exception( - backoff.expo, - TooManyRequests, - max_tries=8, - ) - def clean_bucket(self, bucket: Union[Bucket, str]): - if isinstance(bucket, str): - bucket = self.api.get_bucket_by_name(bucket) - - files_leftover = False - file_versions = bucket.ls(latest_only=False, recursive=True) - - for file_version_info, _ in file_versions: - if file_version_info.file_retention: - if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: - print('Removing retention from file version:', file_version_info.id_) - self.api.update_file_retention( - file_version_info.id_, file_version_info.file_name, - NO_RETENTION_FILE_SETTING, True - ) - elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: - if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable - print( - 'File version: %s cannot be removed due to compliance mode retention' % - (file_version_info.id_,) - ) - files_leftover = True - continue - elif file_version_info.file_retention.mode == RetentionMode.NONE: - pass - else: - raise ValueError( - 'Unknown retention mode: %s' % (file_version_info.file_retention.mode,) - ) - if file_version_info.legal_hold.is_on(): - print('Removing legal hold from file version:', file_version_info.id_) - self.api.update_file_legal_hold( - file_version_info.id_, file_version_info.file_name, LegalHold.OFF - ) - print('Removing file version:', file_version_info.id_) - try: - self.api.delete_file_version(file_version_info.id_, file_version_info.file_name) - except FileNotPresent: - print( - 'It seems that file version %s has already been removed' % - (file_version_info.id_,) - ) - - if files_leftover: - print('Unable to remove bucket because some retained files remain') - else: - print('Removing bucket:', bucket.name) - try: - self.api.delete_bucket(bucket) - except BucketIdNotFound: - print('It seems that bucket %s has already been removed' % (bucket.name,)) - print() - - def count_and_print_buckets(self) -> int: - buckets = self.api.list_buckets() - count = len(buckets) - print(f'Total bucket count at {datetime.now()}: {count}') - for i, bucket in enumerate(buckets, start=1): - print(f'- {i}\t{bucket.name} [{bucket.id_}]') - return count - def print_text_indented(text): """ @@ -383,20 +210,14 @@ class CommandLine: re.compile(r'Trying to print: .*'), ] - def __init__(self, command, account_id, application_key, realm, bucket_name_prefix): + def __init__(self, command, account_id, application_key, realm): self.command = command self.account_id = account_id self.application_key = application_key self.realm = realm - self.bucket_name_prefix = bucket_name_prefix self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename) self.account_info_file_name = SqliteAccountInfo().filename - def generate_bucket_name(self): - return self.bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.bucket_name_prefix) - ) - def run_command(self, args, additional_env: Optional[dict] = None): """ Runs the command with the given arguments, returns a tuple in form of @@ -426,7 +247,7 @@ def should_succeed( if expected_pattern is not None: assert re.search(expected_pattern, stdout), \ - f'did not match pattern="{expected_pattern}", stdout="{stdout}"' + f'did not match pattern="{expected_pattern}", stdout="{stdout}"' return stdout diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index fa604f931..ffdeead7e 100644 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -23,7 +23,7 @@ import pytest from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, SSE_C_KEY_ID_FILE_INFO_KEY_NAME, UNKNOWN_FILE_RETENTION_SETTING, EncryptionMode, EncryptionSetting, FileRetentionSetting, LegalHold, RetentionMode, fix_windows_path_limit - +from b2sdk.test.api_test_manager import generate_bucket_name from b2.console_tool import current_time_millis from .helpers import BUCKET_CREATED_AT_MILLIS, ONE_DAY_MILLIS, ONE_HOUR_MILLIS, SSE_B2_AES, SSE_C_AES, SSE_C_AES_2, SSE_NONE, TempDir, file_mod_time_millis, random_hex, read_file, set_file_mod_time_millis, should_equal, write_file @@ -170,7 +170,7 @@ def test_basic(b2_tool, bucket_name): file_to_upload, ), ) # \r? is for Windows, as $ doesn't match \r\n - to_be_removed_bucket_name = b2_tool.generate_bucket_name() + to_be_removed_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -239,7 +239,7 @@ def test_bucket(b2_tool, bucket_name): def test_key_restrictions(b2_api, b2_tool, bucket_name): - second_bucket_name = b2_tool.generate_bucket_name() + second_bucket_name = generate_bucket_name() b2_tool.should_succeed(['create-bucket', second_bucket_name, 'allPublic', *get_bucketinfo()],) # A single file for rm to fail on. b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, 'README.md', 'test']) @@ -311,7 +311,7 @@ def test_key_restrictions(b2_api, b2_tool, bucket_name): def test_account(b2_tool, bucket_name): # actually a high level operations test - we run bucket tests here since this test doesn't use it b2_tool.should_succeed(['delete-bucket', bucket_name]) - new_bucket_name = b2_tool.generate_bucket_name() + new_bucket_name = generate_bucket_name() # apparently server behaves erratically when we delete a bucket and recreate it right away b2_tool.should_succeed(['create-bucket', new_bucket_name, 'allPrivate', *get_bucketinfo()]) b2_tool.should_succeed(['update-bucket', new_bucket_name, 'allPublic']) @@ -339,7 +339,7 @@ def test_account(b2_tool, bucket_name): # first, let's make sure "create-bucket" doesn't work without auth data - i.e. that the sqlite file hs been # successfully removed - bucket_name = b2_tool.generate_bucket_name() + bucket_name = generate_bucket_name() b2_tool.should_fail( ['create-bucket', bucket_name, 'allPrivate'], r'ERROR: Missing account data: \'NoneType\' object is not subscriptable (\(key 0\) )? ' @@ -353,7 +353,7 @@ def test_account(b2_tool, bucket_name): os.environ['B2_APPLICATION_KEY_ID'] = os.environ['B2_TEST_APPLICATION_KEY_ID'] os.environ['B2_ENVIRONMENT'] = b2_tool.realm - bucket_name = b2_tool.generate_bucket_name() + bucket_name = generate_bucket_name() b2_tool.should_succeed(['create-bucket', bucket_name, 'allPrivate', *get_bucketinfo()]) b2_tool.should_succeed(['delete-bucket', bucket_name]) assert os.path.exists(new_creds), 'sqlite file not created' @@ -886,7 +886,7 @@ def prepare_and_run_sync_copy_tests( else: b2_file_prefix = '' - other_bucket_name = b2_tool.generate_bucket_name() + other_bucket_name = generate_bucket_name() success, _ = b2_tool.run_command( ['create-bucket', other_bucket_name, 'allPublic', *get_bucketinfo()] ) @@ -1066,7 +1066,7 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption']) # Set default encryption via create-bucket - second_bucket_name = b2_tool.generate_bucket_name() + second_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -1526,7 +1526,7 @@ def test_license(b2_tool, with_packages): def test_file_lock(b2_tool, application_key_id, application_key, b2_api): - lock_disabled_bucket_name = b2_tool.generate_bucket_name() + lock_disabled_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -1579,7 +1579,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): 'compliance', '--defaultRetentionPeriod', '7 days' ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' ) - lock_enabled_bucket_name = b2_tool.generate_bucket_name() + lock_enabled_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -1802,7 +1802,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): ) # b2_tool.reauthorize(check_key_capabilities=False) buckets = [ - bucket for bucket in b2_api.api.list_buckets() + bucket for bucket in b2_api.list_buckets() if bucket.name in {lock_enabled_bucket_name, lock_disabled_bucket_name} ] for bucket in buckets: @@ -2085,7 +2085,7 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): source_replication_configuration_json = json.dumps(source_replication_configuration) # create a source bucket and set up replication to destination bucket - source_bucket_name = b2_tool.generate_bucket_name() + source_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -2155,7 +2155,7 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): def test_replication_setup(b2_api, b2_tool, bucket_name): - source_bucket_name = b2_tool.generate_bucket_name() + source_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', @@ -2292,7 +2292,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): source_replication_configuration_json = json.dumps(source_replication_configuration) # create a source bucket and set up replication to destination bucket - source_bucket_name = b2_tool.generate_bucket_name() + source_bucket_name = generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket',