Skip to content

Commit 5c4d1a7

Browse files
samrosesamrose
committed
test: extend logging more to see what happens when pg starts
1 parent 6018247 commit 5c4d1a7

File tree

1 file changed

+77
-71
lines changed

1 file changed

+77
-71
lines changed

‎testinfra/test_ami_nix.py

Lines changed: 77 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -410,89 +410,95 @@ def is_healthy(ssh) -> bool:
410410
logger.error(f"systemd status: {systemd_status.stdout}")
411411
logger.error(f"systemd error: {systemd_status.stderr}")
412412

413-
# Get detailed systemd status
414-
logger.error("Detailed systemd status:")
415-
host.run("sudo systemctl status postgresql -l --no-pager")
413+
# Check systemd service unit file
414+
logger.error("PostgreSQL systemd service unit file:")
415+
result = host.run("sudo systemctl cat postgresql")
416+
logger.error(f"service unit file:\n{result.stdout}\n{result.stderr}")
416417

417-
# Check init script logs
418-
logger.error("Init script logs:")
419-
host.run("sudo journalctl -u cloud-init --no-pager")
418+
# Check systemd service environment
419+
logger.error("PostgreSQL systemd service environment:")
420+
result = host.run("sudo systemctl show postgresql")
421+
logger.error(f"service environment:\n{result.stdout}\n{result.stderr}")
420422

421-
# Check cloud-init logs
422-
logger.error("Cloud-init logs:")
423-
host.run("sudo cat /var/log/cloud-init-output.log")
423+
# Check systemd service dependencies
424+
logger.error("PostgreSQL systemd service dependencies:")
425+
result = host.run("sudo systemctl list-dependencies postgresql")
426+
logger.error(f"service dependencies:\n{result.stdout}\n{result.stderr}")
424427

425-
# Check if init script exists and its contents
426-
logger.error("Init script status:")
427-
host.run("ls -la /tmp/init.sh")
428-
host.run("cat /tmp/init.sh")
428+
# Check if service is enabled
429+
logger.error("PostgreSQL service enabled status:")
430+
result = host.run("sudo systemctl is-enabled postgresql")
431+
logger.error(f"service enabled status:\n{result.stdout}\n{result.stderr}")
429432

430-
# Check PostgreSQL configuration
431-
logger.error("PostgreSQL configuration:")
432-
host.run("sudo cat /etc/postgresql/*/main/postgresql.conf")
433-
host.run("sudo cat /etc/postgresql/*/main/pg_hba.conf")
433+
# Check systemd journal for service execution logs
434+
logger.error("Systemd journal entries for PostgreSQL service execution:")
435+
result = host.run("sudo journalctl -u postgresql -n 100 --no-pager")
436+
logger.error(f"systemd journal:\n{result.stdout}\n{result.stderr}")
434437

435-
# Check PostgreSQL data directory permissions
436-
logger.error("PostgreSQL data directory permissions:")
437-
host.run("sudo ls -la /var/lib/postgresql/*/main/")
438+
# Check systemd journal specifically for ExecStartPre and ExecStart
439+
logger.error("Systemd journal entries for ExecStartPre and ExecStart:")
440+
result = host.run("sudo journalctl -u postgresql -n 100 --no-pager | grep -E 'ExecStartPre|ExecStart'")
441+
logger.error(f"execution logs:\n{result.stdout}\n{result.stderr}")
438442

439-
# Check PostgreSQL startup logs
440-
logger.error("PostgreSQL startup logs:")
441-
host.run("sudo cat /var/log/postgresql/postgresql-*.log")
443+
# Check systemd journal for any errors
444+
logger.error("Systemd journal entries with error level:")
445+
result = host.run("sudo journalctl -u postgresql -n 100 --no-pager -p err")
446+
logger.error(f"error logs:\n{result.stdout}\n{result.stderr}")
442447

443-
# Check systemd journal for PostgreSQL
444-
logger.error("Systemd journal for PostgreSQL:")
445-
host.run("sudo journalctl -u postgresql -n 100 --no-pager")
448+
# Check pre-start script output
449+
logger.error("Checking pre-start script output:")
450+
result = host.run("sudo -u postgres /usr/local/bin/postgres_prestart.sh")
451+
logger.error(f"pre-start script output:\n{result.stdout}\n{result.stderr}")
446452

447-
# Check for any PostgreSQL-related errors in system logs
448-
logger.error("System logs with PostgreSQL errors:")
449-
host.run("sudo journalctl | grep -i postgres | tail -n 100")
453+
# Check PostgreSQL logs directory
454+
logger.error("Checking PostgreSQL logs directory:")
455+
result = host.run("sudo ls -la /var/log/postgresql/")
456+
logger.error(f"log directory contents:\n{result.stdout}\n{result.stderr}")
450457

451-
# Check for any disk space issues
452-
logger.error("Disk space information:")
453-
host.run("df -h")
454-
host.run("sudo du -sh /var/lib/postgresql/*")
458+
# Check any existing PostgreSQL logs
459+
logger.error("Checking existing PostgreSQL logs:")
460+
result = host.run("sudo cat /var/log/postgresql/*.log")
461+
logger.error(f"postgresql logs:\n{result.stdout}\n{result.stderr}")
455462

456-
# Check for any memory issues
457-
logger.error("Memory information:")
458-
host.run("free -h")
463+
# Try starting PostgreSQL directly with pg_ctl and capture output
464+
logger.error("Attempting to start PostgreSQL directly with pg_ctl:")
465+
startup_log = "/tmp/postgres-start.log"
466+
result = host.run(f"sudo -u postgres /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start -l {startup_log}")
467+
logger.error(f"pg_ctl start attempt:\n{result.stdout}\n{result.stderr}")
459468

460-
# Check for any process conflicts
461-
logger.error("Running processes:")
462-
host.run("ps aux | grep postgres")
469+
# Check the startup log
470+
logger.error("PostgreSQL startup log:")
471+
result = host.run(f"sudo cat {startup_log}")
472+
logger.error(f"startup log contents:\n{result.stdout}\n{result.stderr}")
463473

464-
# Check for any port conflicts
465-
logger.error("Port usage:")
466-
host.run("sudo netstat -tulpn | grep 5432")
467-
468-
if socket_check.failed:
469-
logger.error("PostgreSQL socket directory check failed")
470-
logger.error(f"socket check: {socket_check.stdout}")
471-
logger.error(f"socket error: {socket_check.stderr}")
472-
473-
if pg_isready.failed:
474-
logger.error("pg_isready check failed")
475-
logger.error(f"pg_isready output: {pg_isready.stdout}")
476-
logger.error(f"pg_isready error: {pg_isready.stderr}")
477-
478-
# Check PostgreSQL logs for startup issues
479-
logger.error("PostgreSQL logs:")
480-
host.run("sudo cat /var/log/postgresql/postgresql-*.log")
481-
logger.error("PostgreSQL systemd status:")
482-
host.run("sudo systemctl status postgresql")
483-
logger.error("PostgreSQL journal logs:")
484-
host.run("sudo journalctl -u postgresql --no-pager")
485-
486-
if any(cmd.failed for cmd in [systemd_status, socket_check, pg_isready]):
487-
return False
488-
else:
489-
cmd = check(host)
490-
if cmd.failed is True:
491-
logger.warning(f"{service} not ready")
492-
logger.error(f"{service} command failed with rc={cmd.rc}")
493-
logger.error(f"{service} stdout: {cmd.stdout}")
494-
logger.error(f"{service} stderr: {cmd.stderr}")
495-
return False
474+
# Clean up the startup log
475+
result = host.run(f"sudo rm -f {startup_log}")
476+
477+
# Check PostgreSQL configuration
478+
logger.error("PostgreSQL configuration:")
479+
result = host.run("sudo cat /etc/postgresql/postgresql.conf")
480+
logger.error(f"postgresql.conf:\n{result.stdout}\n{result.stderr}")
481+
482+
# Check PostgreSQL authentication configuration
483+
logger.error("PostgreSQL authentication configuration:")
484+
result = host.run("sudo cat /etc/postgresql/pg_hba.conf")
485+
logger.error(f"pg_hba.conf:\n{result.stdout}\n{result.stderr}")
486+
487+
# Check PostgreSQL environment
488+
logger.error("PostgreSQL environment:")
489+
result = host.run("sudo -u postgres env | grep POSTGRES")
490+
logger.error(f"postgres environment:\n{result.stdout}\n{result.stderr}")
491+
492+
if any(cmd.failed for cmd in [systemd_status, socket_check, pg_isready]):
493+
return False
494+
else:
495+
cmd = check(host)
496+
if cmd.failed is True:
497+
logger.warning(f"{service} not ready")
498+
logger.error(f"{service} command failed with rc={cmd.rc}")
499+
logger.error(f"{service} stdout: {cmd.stdout}")
500+
logger.error(f"{service} stderr: {cmd.stderr}")
501+
return False
496502
except Exception as e:
497503
logger.warning(
498504
f"Connection failed during {service} check, attempting reconnect..."

0 commit comments

Comments
 (0)