Skip to content

Commit efdaca6

Browse files
agoncharov-reefagoncharov-reef
committed
Add replication-inspect command
1 parent 0577690 commit efdaca6

File tree

4 files changed

+217
-1
lines changed

4 files changed

+217
-1
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1212
* Add `replication-pause` command
1313
* Add `replication-unpause` command
1414
* Add `--include-existing-files` to `replication-setup`
15+
* Add `replication-inspect` command
1516

1617
### Fixed
1718
* Fix `replication-setup` default priority setter

‎b2/console_tool.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@
7474
parse_sync_folder,
7575
ReplicationMonitor,
7676
ProgressReport,
77+
TwoWayReplicationCheckGenerator,
78+
CheckState,
7779
)
7880
from b2sdk.v2.exception import (
7981
B2Error,
@@ -2587,6 +2589,89 @@ def alter_one_rule(cls, rule: ReplicationRule) -> Optional[ReplicationRule]:
25872589
return rule
25882590

25892591

2592+
@B2.register_subcommand
2593+
class ReplicationInspect(Command):
2594+
"""
2595+
Detect possible misconfigurations of replication by analyzing
2596+
replication rules, buckets and keys.
2597+
2598+
--output-format
2599+
"Console" output format is meant to be human-readable and is subject to change
2600+
in any further release. One should use "json" for reliable "no-breaking-changes"
2601+
output format.
2602+
"""
2603+
2604+
@classmethod
2605+
def _setup_parser(cls, parser):
2606+
super()._setup_parser(parser)
2607+
parser.add_argument('--source-profile', metavar='SOURCE_PROFILE')
2608+
parser.add_argument('--destination-profile', metavar='DESTINATION_PROFILE')
2609+
parser.add_argument('--source-bucket', metavar='SOURCE_BUCKET_NAME')
2610+
parser.add_argument('--destination-bucket', metavar='DESTINATION_BUCKET_NAME')
2611+
parser.add_argument('--rule', metavar='REPLICATION_RULE_NAME')
2612+
parser.add_argument('--file-name-prefix', metavar='FILE_NAME_PREFIX')
2613+
parser.add_argument('--show-all-checks', action='store_true')
2614+
2615+
parser.add_argument('--output-format', default='console', choices=('console', 'json'))
2616+
2617+
def run(self, args):
2618+
source_api = _get_b2api_for_profile(args.source_profile)
2619+
destination_api = _get_b2api_for_profile(args.destination_profile or args.source_profile)
2620+
2621+
troubleshooter = TwoWayReplicationCheckGenerator(
2622+
source_api=source_api,
2623+
destination_api=destination_api,
2624+
filter_source_bucket_name=args.source_bucket,
2625+
filter_destination_bucket_name=args.destination_bucket,
2626+
filter_replication_rule_name=args.rule,
2627+
file_name_prefix=args.file_name_prefix,
2628+
)
2629+
2630+
results = [check.as_dict() for check in troubleshooter.iter_checks()]
2631+
2632+
if args.output_format == 'json':
2633+
self._print_json(
2634+
[
2635+
{key: to_human_readable(value)
2636+
for key, value in result.items()} for result in results
2637+
]
2638+
)
2639+
elif args.output_format == 'console':
2640+
self._print_console(results, show_all_checks=args.show_all_checks)
2641+
else:
2642+
self._print_stderr(f'ERROR: format "{args.output_format}" is not supported')
2643+
return 1
2644+
2645+
return 0
2646+
2647+
def _print_console(self, results: List[dict], show_all_checks: bool = False) -> None:
2648+
for result in results:
2649+
2650+
# print keys starting with `_` as text before table
2651+
self._print('Configuration:')
2652+
for key, value in result.items():
2653+
if key.startswith('_'):
2654+
self._print(
2655+
' ' * 2 + key[1:].replace('_', ' ') + ': ' + to_human_readable(value)
2656+
)
2657+
2658+
# print other keys as rows rows
2659+
rows = {
2660+
key.replace('_', ' '): to_human_readable(value)
2661+
for key, value in result.items()
2662+
if not key.startswith('_') and (value != CheckState.OK or show_all_checks)
2663+
}.items()
2664+
self._print('Checks:')
2665+
2666+
key = None
2667+
for key, value in rows:
2668+
self._print(' ' * 2 + key + ': ' + value)
2669+
if not key: # loop was not entered
2670+
self._print(' ' * 2 + '-')
2671+
2672+
self._print('')
2673+
2674+
25902675
@B2.register_subcommand
25912676
class ReplicationStatus(Command):
25922677
"""

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
arrow>=1.0.2,<2.0.0
2-
b2sdk==1.17.3
2+
b2sdk==1.17.4
33
docutils==0.19
44
idna>=2.2.0; platform_system == 'Java'
55
importlib-metadata>=3.3.0; python_version < '3.8'

‎test/integration/test_b2_command_line.py‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2343,6 +2343,136 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api):
23432343
b2_api.clean_bucket(source_bucket_name)
23442344

23452345

2346+
def test_replication_troubleshooting(b2_tool, bucket_name, b2_api):
2347+
key_one_name = 'clt-testKey-01' + random_hex(6)
2348+
created_key_stdout = b2_tool.should_succeed(
2349+
[
2350+
'create-key',
2351+
key_one_name,
2352+
'listBuckets,readFiles',
2353+
]
2354+
)
2355+
key_one_id, _ = created_key_stdout.split()
2356+
2357+
key_two_name = 'clt-testKey-02' + random_hex(6)
2358+
created_key_stdout = b2_tool.should_succeed(
2359+
[
2360+
'create-key',
2361+
key_two_name,
2362+
'listBuckets,writeFiles',
2363+
]
2364+
)
2365+
key_two_id, _ = created_key_stdout.split()
2366+
2367+
# ---------------- add test data ----------------
2368+
destination_bucket_name = bucket_name
2369+
_ = b2_tool.should_succeed_json(
2370+
['upload-file', '--noProgress', '--quiet', destination_bucket_name, 'README.md', 'one/a']
2371+
)
2372+
2373+
# ---------------- set up replication destination ----------------
2374+
2375+
# update destination bucket info
2376+
destination_replication_configuration = {
2377+
'asReplicationSource': None,
2378+
'asReplicationDestination': {
2379+
'sourceToDestinationKeyMapping': {
2380+
key_one_id: key_two_id,
2381+
},
2382+
},
2383+
}
2384+
destination_replication_configuration_json = json.dumps(destination_replication_configuration)
2385+
destination_bucket = b2_tool.should_succeed_json(
2386+
[
2387+
'update-bucket',
2388+
destination_bucket_name,
2389+
'allPublic',
2390+
'--replication',
2391+
destination_replication_configuration_json,
2392+
]
2393+
)
2394+
2395+
# ---------------- set up replication source ----------------
2396+
source_replication_configuration = {
2397+
"asReplicationSource":
2398+
{
2399+
"replicationRules":
2400+
[
2401+
{
2402+
"destinationBucketId": destination_bucket['bucketId'],
2403+
"fileNamePrefix": "one/",
2404+
"includeExistingFiles": False,
2405+
"isEnabled": True,
2406+
"priority": 1,
2407+
"replicationRuleName": "replication-one"
2408+
}, {
2409+
"destinationBucketId": destination_bucket['bucketId'],
2410+
"fileNamePrefix": "two/",
2411+
"includeExistingFiles": False,
2412+
"isEnabled": True,
2413+
"priority": 2,
2414+
"replicationRuleName": "replication-two"
2415+
}
2416+
],
2417+
"sourceApplicationKeyId": key_one_id,
2418+
},
2419+
}
2420+
source_replication_configuration_json = json.dumps(source_replication_configuration)
2421+
2422+
# create a source bucket and set up replication to destination bucket
2423+
source_bucket_name = b2_tool.generate_bucket_name()
2424+
b2_tool.should_succeed(
2425+
[
2426+
'create-bucket',
2427+
source_bucket_name,
2428+
'allPublic',
2429+
'--fileLockEnabled',
2430+
'--replication',
2431+
source_replication_configuration_json,
2432+
*get_bucketinfo(),
2433+
]
2434+
)
2435+
2436+
# make test data
2437+
_ = b2_tool.should_succeed_json(
2438+
['upload-file', '--noProgress', '--quiet', source_bucket_name, 'CHANGELOG.md', 'one/a']
2439+
)
2440+
2441+
# run troubleshooter
2442+
troubleshooter_results_json = b2_tool.should_succeed_json(
2443+
[
2444+
'replication-inspect',
2445+
'--source-bucket',
2446+
source_bucket_name,
2447+
'--rule',
2448+
'replication-two',
2449+
'--output-format',
2450+
'json',
2451+
]
2452+
)
2453+
2454+
assert troubleshooter_results_json == [
2455+
{
2456+
"_destination_application_key": key_two_id,
2457+
"_destination_bucket": destination_bucket_name,
2458+
"_source_application_key": key_one_id,
2459+
"_source_bucket": source_bucket_name,
2460+
"_source_rule_name": "replication-two",
2461+
"destination_key_bucket_match": "OK",
2462+
"destination_key_capabilities": "OK",
2463+
"destination_key_exists": "OK",
2464+
"destination_key_name_prefix_match": "OK",
2465+
"file_lock_match": "NOT_OK",
2466+
"source_is_enabled": "OK",
2467+
"source_key_accepted_in_target_bucket": "OK",
2468+
"source_key_bucket_match": "OK",
2469+
"source_key_capabilities": "OK",
2470+
"source_key_exists": "OK",
2471+
"source_key_name_prefix_match": "OK"
2472+
}
2473+
]
2474+
2475+
23462476
def _assert_file_lock_configuration(
23472477
b2_tool,
23482478
file_id,

0 commit comments

Comments
 (0)