test: reorg and print logs while waiting continue on other checks when ready

Author: samrose

testinfra/test_ami_nix.py

@@ -313,12 +313,6 @@ def run_detailed_checks(host):
         result = host.run("sudo cat /var/log/postgresql/*.log")
         logger.info(f"postgresql logs:\n{result.stdout}\n{result.stderr}")
 
-        # Try starting PostgreSQL directly with pg_ctl and capture output
-        logger.info("Attempting to start PostgreSQL directly with pg_ctl:")
-        startup_log = "/tmp/postgres-start.log"
-        result = host.run(f"sudo -u postgres /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start -l {startup_log}")
-        logger.info(f"pg_ctl start attempt:\n{result.stdout}\n{result.stderr}")
-
         # Check the startup log
         logger.info("PostgreSQL startup log:")
         result = host.run(f"sudo cat {startup_log}")
@@ -362,17 +356,35 @@ def is_healthy(host, instance_ip, ssh_identity_file) -> bool:
         for service, check in health_checks:
             try:
                 if service == "postgres":
-                    # For PostgreSQL, we need to check multiple things
                     pg_isready = check(host)
 
-                    if pg_isready.failed:
-                        logger.error("PostgreSQL is not ready")
-                        logger.error(f"pg_isready stdout: {pg_isready.stdout}")
-                        logger.error(f"pg_isready stderr: {pg_isready.stderr}")
+                    # Always read and log the PostgreSQL logs first
+                    logger.warning("PostgreSQL status check:")
+                    try:
+                        # Read both .log and .csv files
+                        log_files = [
+                            "/var/log/postgresql/*.log",
+                            "/var/log/postgresql/*.csv"
+                        ]
 
-                        # Run detailed checks since we know we have a working connection
-                        run_detailed_checks(host)
-                        return False
+                        for log_pattern in log_files:
+                            log_result = host.run(f"sudo cat {log_pattern}")
+                            if not log_result.failed:
+                                logger.error(f"PostgreSQL logs from {log_pattern}:")
+                                logger.error(log_result.stdout)
+                                if log_result.stderr:
+                                    logger.error(f"Log read errors: {log_result.stderr}")
+                            else:
+                                logger.error(f"Failed to read PostgreSQL logs from {log_pattern}: {log_result.stderr}")
+                    except Exception as e:
+                        logger.error(f"Error reading PostgreSQL logs: {str(e)}")
+
+                    # Then check the status and return
+                    if not pg_isready.failed:
+                        continue
+                    # Wait before next attempt
+                    sleep(5)
+                    return False
                 else:
                     cmd = check(host)
                     if cmd.failed is True: