test: more logging for healthcheck

Author: samrose

testinfra/test_ami_nix.py

@@ -296,7 +296,14 @@ def is_healthy(host, instance_ip, ssh_identity_file) -> bool:
         health_checks = [
             (
                 "postgres",
-                lambda h: h.run("sudo -u postgres /usr/bin/pg_isready -U postgres"),
+                lambda h: (
+                    # First check if PostgreSQL is running
+                    h.run("sudo systemctl is-active postgresql"),
+                    # Then check if the socket directory exists and has correct permissions
+                    h.run("sudo ls -la /run/postgresql"),
+                    # Then try pg_isready
+                    h.run("sudo -u postgres /usr/bin/pg_isready -U postgres")
+                ),
             ),
             (
                 "adminapi",
@@ -322,23 +329,56 @@ def is_healthy(host, instance_ip, ssh_identity_file) -> bool:
 
         for service, check in health_checks:
             try:
-                cmd = check(host)
-                if cmd.failed is True:
-                    logger.warning(f"{service} not ready")
-                    logger.error(f"{service} command failed with rc={cmd.rc}")
-                    logger.error(f"{service} stdout: {cmd.stdout}")
-                    logger.error(f"{service} stderr: {cmd.stderr}")
+                if service == "postgres":
+                    # For PostgreSQL, we need to check multiple things
+                    systemd_status, socket_check, pg_isready = check(host)
+                    
+                    if systemd_status.failed:
+                        logger.error("PostgreSQL systemd service is not active")
+                        logger.error(f"systemd status: {systemd_status.stdout}")
+                        logger.error(f"systemd error: {systemd_status.stderr}")
+                        
+                        # Check init script logs
+                        logger.error("Init script logs:")
+                        host.run("sudo journalctl -u cloud-init --no-pager")
+                        
+                        # Check cloud-init logs
+                        logger.error("Cloud-init logs:")
+                        host.run("sudo cat /var/log/cloud-init-output.log")
+                        
+                        # Check if init script exists and its contents
+                        logger.error("Init script status:")
+                        host.run("ls -la /tmp/init.sh")
+                        host.run("cat /tmp/init.sh")
+                    
+                    if socket_check.failed:
+                        logger.error("PostgreSQL socket directory check failed")
+                        logger.error(f"socket check: {socket_check.stdout}")
+                        logger.error(f"socket error: {socket_check.stderr}")
+                    
+                    if pg_isready.failed:
+                        logger.error("pg_isready check failed")
+                        logger.error(f"pg_isready output: {pg_isready.stdout}")
+                        logger.error(f"pg_isready error: {pg_isready.stderr}")
 
-                    # For PostgreSQL, also check the logs and systemd status
-                    if service == "postgres":
-                        logger.error("PostgreSQL logs:")
-                        host.run("sudo cat /var/log/postgresql/postgresql-*.log")
-                        logger.error("PostgreSQL systemd status:")
-                        host.run("sudo systemctl status postgresql")
-                        logger.error("PostgreSQL journal logs:")
-                        host.run("sudo journalctl -u postgresql --no-pager")
+                    # Check PostgreSQL logs for startup issues
+                    logger.error("PostgreSQL logs:")
+                    host.run("sudo cat /var/log/postgresql/postgresql-*.log")
+                    logger.error("PostgreSQL systemd status:")
+                    host.run("sudo systemctl status postgresql")
+                    logger.error("PostgreSQL journal logs:")
+                    host.run("sudo journalctl -u postgresql --no-pager")
 
-                    return False
+                    if any(cmd.failed for cmd in [systemd_status, socket_check, pg_isready]):
+                        return False
+                else:
+                    cmd = check(host)
+                    if cmd.failed is True:
+                        logger.warning(f"{service} not ready")
+                        logger.error(f"{service} command failed with rc={cmd.rc}")
+                        logger.error(f"{service} stdout: {cmd.stdout}")
+                        logger.error(f"{service} stderr: {cmd.stderr}")
+                        return False
             except Exception as e:
                 logger.warning(
                     f"Connection failed during {service} check, attempting reconnect..."