Merge pull request ceph#56767 from adk3798/cephadm-log-only-match

adk3798 · web-flow · commit 3f7db60e5ba4 · 2024-04-15T10:29:43.000-04:00
qa/cephadm: only fail on CEPHADM_ error in logs

Reviewed-by: Adam King &lt;adking@redhat.com&gt;
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/17.2.0.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/17.2.0.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm:
     roleless: true
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/task/test_cephadm_timeout.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_cephadm_timeout.yaml
@@ -4,10 +4,14 @@ roles:
   - mgr.a
   - osd.0
   - client.0
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
 - workunit:
     clients:
       client.0:
-        - cephadm/test_cephadm_timeout.py
+        - cephadm/test_cephadm_timeout.py
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli.yaml
@@ -6,6 +6,10 @@ roles:
   - mon.a
   - mgr.a
   - client.0
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli_mon.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli_mon.yaml
@@ -2,6 +2,8 @@ overrides:
   ceph:
     log-ignorelist:
       - MON_DOWN
+    log-only-match:
+      - CEPHADM_
 roles:
 - - host.a
   - osd.0
diff --git a/qa/suites/orch/cephadm/osds/1-start.yaml b/qa/suites/orch/cephadm/osds/1-start.yaml
@@ -25,3 +25,5 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+    log-only-match:
+      - CEPHADM_
diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml
@@ -10,6 +10,10 @@ roles:
   # Reserve a host for acting as a test client
   - - host.b
     - cephadm.exclude
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
   # TODO: (jjm) I don't think `install` is necessary for this file. Remove?
   - install:
diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml
@@ -9,6 +9,10 @@ roles:
 # Reserve a host for acting as a domain controller
 - - host.b
   - cephadm.exclude
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm.deploy_samba_ad_dc:
     role: host.b
diff --git a/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml b/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml
@@ -22,3 +22,5 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+    log-only-match:
+      - CEPHADM_
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml b/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml
@@ -27,3 +27,5 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+    log-only-match:
+      - CEPHADM_
diff --git a/qa/suites/orch/cephadm/smoke-small/start.yaml b/qa/suites/orch/cephadm/smoke-small/start.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm:
     conf:
diff --git a/qa/suites/orch/cephadm/smoke/start.yaml b/qa/suites/orch/cephadm/smoke/start.yaml
@@ -5,6 +5,8 @@ overrides:
       - mons down
       - mon down
       - out of quorum
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm:
     conf:
diff --git a/qa/suites/orch/cephadm/thrash/1-start.yaml b/qa/suites/orch/cephadm/thrash/1-start.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm.shell:
     env: [sha1]
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - cephadm.shell:
     env: [sha1]
diff --git a/qa/suites/orch/cephadm/with-work/start.yaml b/qa/suites/orch/cephadm/with-work/start.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml b/qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml
@@ -12,6 +12,9 @@ roles:
 overrides:
   cephadm:
     use-ca-signed-key: True
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/workunits/task/test_extra_daemon_features.yaml b/qa/suites/orch/cephadm/workunits/task/test_extra_daemon_features.yaml
@@ -7,6 +7,10 @@ roles:
   - mon.b
   - mgr.b
   - osd.1
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
@@ -5,6 +5,8 @@ overrides:
       - mons down
       - mon down
       - out of quorum
+    log-only-match:
+      - CEPHADM_
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml
@@ -6,6 +6,10 @@ roles:
   - mon.a
   - mgr.a
   - client.0
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
 tasks:
 - install:
 - cephadm:
diff --git a/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml b/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
@@ -5,6 +5,8 @@ overrides:
       - mons down
       - mon down
       - out of quorum
+    log-only-match:
+      - CEPHADM_
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml b/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
@@ -5,6 +5,8 @@ overrides:
       - mons down
       - mon down
       - out of quorum
+    log-only-match:
+      - CEPHADM_
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml b/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
@@ -6,6 +6,8 @@ overrides:
       - mon down
       - mons down
       - out of quorum
+    log-only-match:
+      - CEPHADM_
 roles:
 - - host.a
   - osd.0
diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
@@ -430,7 +430,7 @@ def ceph_log(ctx, config):
 
     finally:
         log.info('Checking cluster log for badness...')
-        def first_in_ceph_log(pattern, excludes):
+        def first_in_ceph_log(pattern, excludes, only_match):
             """
             Find the first occurrence of the pattern specified in the Ceph log,
             Returns None if none found.
@@ -445,6 +445,8 @@ def first_in_ceph_log(pattern, excludes):
                 '/var/log/ceph/{fsid}/ceph.log'.format(
                     fsid=fsid),
             ]
+            if only_match:
+                args.extend([run.Raw('|'), 'egrep', '|'.join(only_match)])
             if excludes:
                 for exclude in excludes:
                     args.extend([run.Raw('|'), 'egrep', '-v', exclude])
@@ -460,14 +462,22 @@ def first_in_ceph_log(pattern, excludes):
                 return stdout
             return None
 
+        # NOTE: technically the first and third arg to first_in_ceph_log
+        # are serving a similar purpose here of being something we
+        # look for in the logs. The reason they are separate args is that
+        # we want '\[ERR\]|\[WRN\]|\[SEC\]' to always have to be in the thing
+        # we match even if the test yaml specifies nothing else, and then the
+        # log-only-match options are for when a test only wants to fail on
+        # a specific subset of log lines that '\[ERR\]|\[WRN\]|\[SEC\]' matches
         if first_in_ceph_log('\[ERR\]|\[WRN\]|\[SEC\]',
-                             config.get('log-ignorelist')) is not None:
+                             config.get('log-ignorelist'),
+                             config.get('log-only-match')) is not None:
             log.warning('Found errors (ERR|WRN|SEC) in cluster log')
             ctx.summary['success'] = False
             # use the most severe problem as the failure reason
             if 'failure_reason' not in ctx.summary:
                 for pattern in ['\[SEC\]', '\[ERR\]', '\[WRN\]']:
-                    match = first_in_ceph_log(pattern, config['log-ignorelist'])
+                    match = first_in_ceph_log(pattern, config['log-ignorelist'], config.get('log-only-match'))
                     if match is not None:
                         ctx.summary['failure_reason'] = \
                             '"{match}" in cluster log'.format(
