fix: block bcm-recommended-actions from smoke tests and improve test runner

Author: pulimsr

generated/smoke-tests/bcm-recommended-actions/BCMRecommendedActionsSmokeTests.cpp

@@ -39,6 +39,7 @@ def generate(self, clients_to_build: set):
             'iot-managed-integrations',
             'elastictranscoder',
             'freetier',
+            'bcm-recommended-actions',
         }
         clients_to_build = clients_to_build - blocked_services
 

generated/smoke-tests/bcm-recommended-actions/CMakeLists.txt

@@ -9,12 +9,7 @@
 def main():
     parser = argparse.ArgumentParser(description='Run smoke tests')
     parser.add_argument('--testDir', default='./build', help='Path to build directory')
-    parser.add_argument('--test-failures', action='store_true', help='Use invalid credentials to test failure messages')
     args = parser.parse_args()
-    
-    if args.test_failures:
-        os.environ['AWS_ACCESS_KEY_ID'] = 'INVALID_KEY'
-        os.environ['AWS_SECRET_ACCESS_KEY'] = 'INVALID_SECRET'
 
     smoke_tests_dir = os.path.join(args.testDir, "generated/smoke-tests")
 
@@ -33,8 +28,10 @@ def main():
 
     total_tests = 0
     all_results = defaultdict(list)
+    skipped_services = []
     service_id = os.environ.get('AWS_SMOKE_TEST_SERVICE_IDS', '').strip().lower().replace(' ', '-') if os.environ.get('AWS_SMOKE_TEST_SERVICE_IDS', '').strip() else []
     if service_id:
+        skipped_services = [s for s in services if s not in service_id]
         services = [s for s in services if s in service_id]
 
     for service in services:
@@ -49,6 +46,8 @@ def main():
 
             with open(json_output_path, 'r') as f:
                 test_results = json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError) as e:
+            test_results = {}
         finally:
             os.unlink(json_output_path)
 
@@ -65,6 +64,31 @@ def main():
                         'failure': failure_msg,
                     })
 
+    # Count skipped tests
+    for service in skipped_services:
+        executable = os.path.join(smoke_tests_dir, service, f"{service}-smoke-tests")
+        fd, json_output_path = tempfile.mkstemp(suffix='.json')
+        os.close(fd)
+        
+        try:
+            test_command = [executable, f'--gtest_output=json:{json_output_path}']
+            subprocess.run(test_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            with open(json_output_path, 'r') as f:
+                test_results = json.load(f)
+            for test_suite in test_results.get('testsuites', []):
+                total_tests += test_suite['tests']
+                for test in test_suite.get('testsuite', []):
+                    all_results[service].append({
+                        'service': service,
+                        'name': test['name'],
+                        'status': 'SKIPPED',
+                        'failure': None,
+                    })
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+        finally:
+            os.unlink(json_output_path)
+
     print(f"1..{total_tests}")
 
     for service in all_results.keys():
@@ -76,7 +100,9 @@ def main():
             failure = result['failure']
             error_expected = "error expected from service" if "Failure" in test_name or "Error" in test_name else "no error expected from service"
 
-            if status == 'RUN' and not failure:
+            if status == 'SKIPPED':
+                print(f"ok {service} {test_name} # SKIP")
+            elif status == 'RUN' and not failure:
                 print(f"ok {service} {test_name} - {error_expected}")
             else:
                 print(f"not ok {service} {test_name}")