Skip to content

Commit bdff818

Browse files
vshankarvshankar
committed
Merge PR ceph#55837 into main
* refs/pull/55837/head: qa: enhance labeled perf counters test for cephfs-mirror Reviewed-by: Venky Shankar <[email protected]>
2 parents cedca8e + 7b7e5d4 commit bdff818

File tree

1 file changed

+110
-1
lines changed

1 file changed

+110
-1
lines changed

‎qa/tasks/cephfs/test_mirroring.py‎

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,10 @@ def test_cephfs_mirror_stats(self):
546546
self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
547547
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
548548

549+
# dump perf counters
550+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
551+
first = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
552+
549553
# take a snapshot
550554
self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
551555

@@ -554,6 +558,11 @@ def test_cephfs_mirror_stats(self):
554558
"client.mirror_remote@ceph", '/d0', 'snap0', 1)
555559
self.verify_snapshot('d0', 'snap0')
556560

561+
# check snaps_synced
562+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
563+
second = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
564+
self.assertGreater(second["counters"]["snaps_synced"], first["counters"]["snaps_synced"])
565+
557566
# some more IO
558567
self.mount_a.run_shell(["mkdir", "d0/d00"])
559568
self.mount_a.run_shell(["mkdir", "d0/d01"])
@@ -569,6 +578,11 @@ def test_cephfs_mirror_stats(self):
569578
"client.mirror_remote@ceph", '/d0', 'snap1', 2)
570579
self.verify_snapshot('d0', 'snap1')
571580

581+
# check snaps_synced
582+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
583+
third = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
584+
self.assertGreater(third["counters"]["snaps_synced"], second["counters"]["snaps_synced"])
585+
572586
# delete a snapshot
573587
self.mount_a.run_shell(["rmdir", "d0/.snap/snap0"])
574588

@@ -577,6 +591,10 @@ def test_cephfs_mirror_stats(self):
577591
self.assertTrue('snap0' not in snap_list)
578592
self.check_peer_status_deleted_snap(self.primary_fs_name, self.primary_fs_id,
579593
"client.mirror_remote@ceph", '/d0', 1)
594+
# check snaps_deleted
595+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
596+
fourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
597+
self.assertGreater(fourth["counters"]["snaps_deleted"], third["counters"]["snaps_deleted"])
580598

581599
# rename a snapshot
582600
self.mount_a.run_shell(["mv", "d0/.snap/snap1", "d0/.snap/snap2"])
@@ -587,6 +605,10 @@ def test_cephfs_mirror_stats(self):
587605
self.assertTrue('snap2' in snap_list)
588606
self.check_peer_status_renamed_snap(self.primary_fs_name, self.primary_fs_id,
589607
"client.mirror_remote@ceph", '/d0', 1)
608+
# check snaps_renamed
609+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
610+
fifth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
611+
self.assertGreater(fifth["counters"]["snaps_renamed"], fourth["counters"]["snaps_renamed"])
590612

591613
self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
592614
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -626,6 +648,12 @@ def test_cephfs_mirror_cancel_sync(self):
626648

627649
snap_list = self.mount_b.ls(path='d0/.snap')
628650
self.assertTrue('snap0' not in snap_list)
651+
652+
# check sync_failures
653+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
654+
vmirror_peers = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
655+
self.assertGreater(vmirror_peers["counters"]["sync_failures"], 0)
656+
629657
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
630658

631659
def test_cephfs_mirror_restart_sync_on_blocklist(self):
@@ -655,6 +683,10 @@ def test_cephfs_mirror_restart_sync_on_blocklist(self):
655683
# fetch rados address for blacklist check
656684
rados_inst = self.get_mirror_rados_addr(self.primary_fs_name, self.primary_fs_id)
657685

686+
# dump perf counters
687+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
688+
vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
689+
658690
# take a snapshot
659691
self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
660692

@@ -683,6 +715,10 @@ def test_cephfs_mirror_restart_sync_on_blocklist(self):
683715
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
684716
"client.mirror_remote@ceph", '/d0', 'snap0', expected_snap_count=1)
685717
self.verify_snapshot('d0', 'snap0')
718+
# check snaps_synced
719+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
720+
vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
721+
self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
686722

687723
self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
688724
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -691,6 +727,10 @@ def test_cephfs_mirror_failed_sync_with_correction(self):
691727
self.enable_mirroring(self.primary_fs_name, self.primary_fs_id)
692728
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
693729

730+
# dump perf counters
731+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
732+
vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
733+
694734
# add a non-existent directory for synchronization
695735
self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
696736

@@ -707,6 +747,10 @@ def test_cephfs_mirror_failed_sync_with_correction(self):
707747
time.sleep(120)
708748
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
709749
"client.mirror_remote@ceph", '/d0', 'snap0', 1)
750+
# check snaps_synced
751+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
752+
vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
753+
self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
710754
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
711755

712756
def test_cephfs_mirror_service_daemon_status(self):
@@ -760,7 +804,7 @@ def test_mirroring_init_failure(self):
760804
self.disable_mirroring_module()
761805

762806
# enable mirroring through mon interface -- this should result in the mirror daemon
763-
# failing to enable mirroring due to absence of `cephfs_mirorr` index object.
807+
# failing to enable mirroring due to absence of `cephfs_mirror` index object.
764808
self.run_ceph_cmd("fs", "mirror", "enable", self.primary_fs_name)
765809

766810
with safe_while(sleep=5, tries=10, action='wait for failed state') as proceed:
@@ -888,6 +932,10 @@ def test_cephfs_mirror_symlink_sync(self):
888932
self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
889933
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
890934

935+
# dump perf counters
936+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
937+
vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
938+
891939
# take a snapshot
892940
self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
893941

@@ -896,6 +944,10 @@ def test_cephfs_mirror_symlink_sync(self):
896944
"client.mirror_remote@ceph", '/d0', 'snap0', 1)
897945
self.verify_snapshot('d0', 'snap0')
898946

947+
# check snaps_synced
948+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
949+
vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
950+
self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
899951
self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
900952
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
901953

@@ -907,12 +959,20 @@ def test_cephfs_mirror_with_parent_snapshot(self):
907959
self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3')
908960
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
909961

962+
# dump perf counters
963+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
964+
vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
965+
910966
# take a snapshot
911967
self.mount_a.run_shell(["mkdir", "d0/d1/d2/d3/.snap/snap0"])
912968

913969
time.sleep(30)
914970
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
915971
"client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap0', 1)
972+
# check snaps_synced
973+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
974+
vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
975+
self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
916976

917977
# create snapshots in parent directories
918978
self.mount_a.run_shell(["mkdir", "d0/.snap/snap_d0"])
@@ -924,12 +984,20 @@ def test_cephfs_mirror_with_parent_snapshot(self):
924984
time.sleep(30)
925985
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
926986
"client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap1', 2)
987+
# check snaps_synced
988+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
989+
vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
990+
self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
927991

928992
self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap0"])
929993
self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap1"])
930994
time.sleep(15)
931995
self.check_peer_status_deleted_snap(self.primary_fs_name, self.primary_fs_id,
932996
"client.mirror_remote@ceph", '/d0/d1/d2/d3', 2)
997+
# check snaps_deleted
998+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
999+
vfourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1000+
self.assertGreater(vfourth["counters"]["snaps_deleted"], vthird["counters"]["snaps_deleted"])
9331001

9341002
self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3')
9351003
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -1033,13 +1101,20 @@ def exec_git_cmd(cmd_list):
10331101
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
10341102

10351103
self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}')
1104+
# dump perf counters
1105+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1106+
vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
10361107
self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a'])
10371108

10381109
# full copy, takes time
10391110
time.sleep(500)
10401111
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
10411112
"client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1)
10421113
self.verify_snapshot(repo_path, 'snap_a')
1114+
# check snaps_synced
1115+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1116+
vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1117+
self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
10431118

10441119
# create some diff
10451120
num = random.randint(5, 20)
@@ -1052,6 +1127,9 @@ def exec_git_cmd(cmd_list):
10521127
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
10531128
"client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2)
10541129
self.verify_snapshot(repo_path, 'snap_b')
1130+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1131+
vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1132+
self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
10551133

10561134
# diff again, this time back to HEAD
10571135
log.debug('resetting to HEAD')
@@ -1063,6 +1141,9 @@ def exec_git_cmd(cmd_list):
10631141
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
10641142
"client.mirror_remote@ceph", f'/{repo_path}', 'snap_c', 3)
10651143
self.verify_snapshot(repo_path, 'snap_c')
1144+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1145+
vfourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1146+
self.assertGreater(vfourth["counters"]["snaps_synced"], vthird["counters"]["snaps_synced"])
10661147

10671148
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
10681149

@@ -1133,11 +1214,18 @@ def verify_types(dirname, fnames, snap_name):
11331214
while turns != len(typs):
11341215
snapname = f'snap_{turns}'
11351216
cleanup_and_create_with_type('d0', fnames)
1217+
# dump perf counters
1218+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1219+
vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
11361220
self.mount_a.run_shell(['mkdir', f'd0/.snap/{snapname}'])
11371221
time.sleep(30)
11381222
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
11391223
"client.mirror_remote@ceph", '/d0', snapname, turns+1)
11401224
verify_types('d0', fnames, snapname)
1225+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1226+
vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1227+
self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
1228+
11411229
# next type
11421230
typs.rotate(1)
11431231
turns += 1
@@ -1184,13 +1272,19 @@ def exec_git_cmd(cmd_list):
11841272
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
11851273

11861274
self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}')
1275+
# dump perf counters
1276+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1277+
vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
11871278
self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a'])
11881279

11891280
# full copy, takes time
11901281
time.sleep(500)
11911282
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
11921283
"client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1)
11931284
self.verify_snapshot(repo_path, 'snap_a')
1285+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1286+
vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1287+
self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
11941288

11951289
# create some diff
11961290
num = random.randint(60, 100)
@@ -1207,6 +1301,9 @@ def exec_git_cmd(cmd_list):
12071301
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
12081302
"client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2)
12091303
self.verify_snapshot(repo_path, 'snap_b')
1304+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1305+
vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1306+
self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
12101307

12111308
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
12121309

@@ -1265,6 +1362,9 @@ def test_cephfs_mirror_cancel_mirroring_and_readd(self):
12651362
self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d2')
12661363
self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
12671364

1365+
# dump perf counters
1366+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1367+
vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
12681368
# take snapshots
12691369
log.debug('taking snapshots')
12701370
self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
@@ -1326,6 +1426,10 @@ def test_cephfs_mirror_cancel_mirroring_and_readd(self):
13261426
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
13271427
"client.mirror_remote@ceph", '/d2', 'snap0', 1)
13281428
self.verify_snapshot('d2', 'snap0')
1429+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1430+
vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1431+
self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
1432+
self.assertGreater(vafter["counters"]["snaps_deleted"], vbefore["counters"]["snaps_deleted"])
13291433

13301434
self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
13311435

@@ -1354,6 +1458,11 @@ def test_local_and_remote_dir_root_mode(self):
13541458
time.sleep(60)
13551459
self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
13561460
"client.mirror_remote@ceph", '/l1', 'snap0', 1)
1461+
# dump perf counters
1462+
res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
1463+
vmirror_peers = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
1464+
snaps_synced = vmirror_peers["counters"]["snaps_synced"]
1465+
self.assertEqual(snaps_synced, 1, f"Mismatch snaps_synced: {snaps_synced} vs 1")
13571466

13581467
mode_local = self.mount_a.run_shell(["stat", "--format=%A", "l1"]).stdout.getvalue().strip()
13591468
mode_remote = self.mount_b.run_shell(["stat", "--format=%A", "l1"]).stdout.getvalue().strip()

0 commit comments

Comments
 (0)