From fa9bb6e4df7d8fe9b5fe561a4af1e0fd7a35aef3 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Wed, 23 Jul 2025 15:47:41 -0400 Subject: [PATCH] feat(reporting): Add database resource report command Adds a new management command, `database_resource_report`, to the `resource_registry` application to generate reports on database usage and query activity for PostgreSQL. The command provides the following information: - Table sizes and row counts. - Object counts for all Django models. - A real-time report from `pg_stat_activity`, with sorted lists for: - Longest running active queries - Oldest "idle in transaction" connections - Connections with active wait events - Oldest open connections - Historical query performance statistics from `pg_stat_statements`, including most frequent, slowest, and resource-intensive queries. It also advises on enabling the extension if it is not active. --- .../commands/database_resource_report.py | 171 ++++++++ docs/database_report_example.txt | 381 ++++++++++++++++++ docs/database_reporting.md | 96 +++++ 3 files changed, 648 insertions(+) create mode 100644 ansible_base/resource_registry/management/commands/database_resource_report.py create mode 100644 docs/database_report_example.txt create mode 100644 docs/database_reporting.md diff --git a/ansible_base/resource_registry/management/commands/database_resource_report.py b/ansible_base/resource_registry/management/commands/database_resource_report.py new file mode 100644 index 000000000..6b1122814 --- /dev/null +++ b/ansible_base/resource_registry/management/commands/database_resource_report.py @@ -0,0 +1,171 @@ +import datetime +import textwrap + +from django.apps import apps +from django.core.management.base import BaseCommand +from django.db import connection, utils +from django.utils import timezone + + +class Command(BaseCommand): + help = 'Generates a report on database table sizes and query activity for PostgreSQL.' + + def handle(self, *args, **options): + if connection.vendor != 'postgresql': + self.stdout.write(self.style.ERROR("This command is only supported for PostgreSQL databases.")) + return + + self.stdout.write(self.style.SUCCESS('Database Diagnostic Report')) + self.stdout.write(self.style.NOTICE(f"Report generated at: {timezone.now().isoformat()}")) + + self.print_table_sizes() + self.print_model_report() + self.print_pg_stat_activity() + self.print_pg_stat_statements() + + def print_table_sizes(self): + self.stdout.write("\n" + self.style.SUCCESS('Table Sizes and Row Counts')) + self.stdout.write(self.style.NOTICE("Based on database schema inspection.")) + + query = textwrap.dedent(""" + SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), + (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I', table_name), false, true, '')))[1]::text::bigint + FROM information_schema.tables + WHERE table_schema = 'public' AND table_name NOT LIKE 'pg_%%' ORDER BY 3 DESC; + """).strip() + + self.stdout.write(self.style.SQL_KEYWORD(f"Running query:\n{query}")) + + with connection.cursor() as cursor: + cursor.execute(query) + self.stdout.write(f"{'Table Name':<50} {'Size':<20} {'Rows':<10}") + self.stdout.write(f"{'--'*50} {'--'*20} {'--'*10}") + for row in cursor: + self.stdout.write(f"{row[0]:<50} {row[1]:<20} {row[2]:<10}") + + def print_model_report(self): + self.stdout.write("\n" + self.style.SUCCESS('Model Object Counts')) + self.stdout.write(self.style.NOTICE("Based on the Django model layer.")) + all_models = apps.get_models() + self.stdout.write(f"\n{'Model (app.model)':<50} {'Total Objects':<15}") + self.stdout.write(f"{'--'*50} {'--'*15}") + for model in sorted(all_models, key=lambda m: m._meta.label): + if model._meta.proxy: + continue + try: + self.stdout.write(f"{model._meta.label:<50} {model.objects.count():<15}") + except Exception: + self.stdout.write(f"{model._meta.label:<50} {'Error counting':<15}") + + def print_pg_stat_activity(self): + self.stdout.write("\n" + self.style.SUCCESS('PostgreSQL Activity Report (pg_stat_activity)')) + + def run_and_print_report(title, query, age_logic): + self.stdout.write("\n" + self.style.NOTICE(title)) + self.stdout.write(self.style.SQL_KEYWORD(f"Running query:\n{textwrap.dedent(query).strip()}")) + header = f"{'PID':<10} {'User':<15} {'Client':<20} {'State':<15} {'Wait Event':<25} {age_logic['header']:<25} {'Query Snippet'}" + header_underline = f"{'--'*10} {'--'*15} {'--'*20} {'--'*15} {'--'*25} {'--'*25} {'--'*80}" + with connection.cursor() as cursor: + try: + cursor.execute(query) + rows = cursor.fetchall() + if not rows: + self.stdout.write("No matching activity found.") + return + self.stdout.write(header) + self.stdout.write(header_underline) + for row in rows: + pid, user, client, state, wait_type, wait_event, query_start, state_change, backend_start, query_text = row + now = timezone.now() + age = "N/A" + ts = row[age_logic['timestamp_index']] + if ts: + age = now - ts + age_str = str(age).split('.')[0] if isinstance(age, datetime.timedelta) else "N/A" + wait_full = f"{wait_type or ''}:{wait_event or ''}".strip(':') + self.stdout.write( + f"{pid:<10} {user or '':<15} {str(client) or '':<20} {state or '':<15} {wait_full:<25} {age_str:<25} {query_text or ''}" + ) + except utils.DatabaseError as e: + self.stdout.write(self.style.ERROR(f"Could not query pg_stat_activity for '{title}': {e}")) + + base_query = ( + "SELECT pid, usename, client_addr, state, wait_event_type, wait_event, " + "query_start, state_change, backend_start, left(query, 100) as query_snippet " + "FROM pg_stat_activity WHERE state IS NOT NULL" + ) + reports = [ + ( + 'Top 10 Longest Running Active Queries', + f"{base_query} AND state = 'active' ORDER BY query_start ASC LIMIT 10", + {'header': 'Duration', 'timestamp_index': 6}, + ), + ( + 'Top 10 Oldest "Idle in Transaction" Connections', + f"{base_query} AND state = 'idle in transaction' ORDER BY state_change ASC LIMIT 10", + {'header': 'Idle Time', 'timestamp_index': 7}, + ), + ( + 'Top 10 Connections by Wait Time', + f"{base_query} AND wait_event IS NOT NULL ORDER BY state_change ASC LIMIT 10", + {'header': 'Wait Time', 'timestamp_index': 7}, + ), + ('Top 10 Oldest Open Connections', f"{base_query} ORDER BY backend_start ASC LIMIT 10", {'header': 'Connection Age', 'timestamp_index': 8}), + ] + for title, query, age_logic in reports: + run_and_print_report(title, query, age_logic) + + def print_pg_stat_statements(self): + self.stdout.write("\n" + self.style.SUCCESS('PostgreSQL Statement Statistics (pg_stat_statements)')) + with connection.cursor() as cursor: + try: + cursor.execute("SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements';") + if cursor.fetchone() is None: + self.stdout.write(self.style.ERROR("The 'pg_stat_statements' extension is not enabled.")) + self.stdout.write( + "To enable it, add 'pg_stat_statements' to 'shared_preload_libraries' in your postgresql.conf, restart the database, and then run 'CREATE EXTENSION pg_stat_statements;'" + ) + return + cursor.execute("SELECT count(*) FROM pg_stat_statements;") + if cursor.fetchone()[0] == 0: + self.stdout.write(self.style.WARNING("The 'pg_stat_statements' view is empty.")) + return + self.print_statement_report(cursor, "Top 10 Most Frequent Queries", "calls DESC", "Calls", "calls") + self.print_statement_report( + cursor, "Top 10 Slowest Queries (by mean execution time)", "mean_exec_time DESC", "Mean Time (ms)", "mean_exec_time" + ) + self.print_statement_report( + cursor, "Top 10 Queries by Total Execution Time (CPU)", "total_exec_time DESC", "Total Time (ms)", "total_exec_time" + ) + self.print_statement_report( + cursor, + "Top 10 Queries by I/O Read", + "(shared_blks_read + local_blks_read + temp_blks_read) DESC", + "Blocks Read", + "(shared_blks_read + local_blks_read + temp_blks_read)", + ) + self.print_statement_report( + cursor, + "Top 10 Queries by Buffer Cache Usage (Memory)", + "(shared_blks_hit + shared_blks_read) DESC", + "Shared Blocks Hit+Read", + "(shared_blks_hit + shared_blks_read)", + ) + except utils.DatabaseError as e: + self.stdout.write(self.style.ERROR(f"Could not query pg_stat_statements: {e}")) + + def print_statement_report(self, cursor, title, order_by, metric_name, metric_col): + self.stdout.write("\n" + self.style.NOTICE(title)) + query = f"SELECT {metric_col}, query FROM pg_stat_statements ORDER BY {order_by} LIMIT 10;" + self.stdout.write(self.style.SQL_KEYWORD(f"Running query:\n{query}")) + try: + cursor.execute(query) + self.stdout.write(f"{metric_name:<25} {'Query Snippet'}") + self.stdout.write(f"{'--'*25} {'--'*100}") + for row in cursor: + metric, query_text = row + query_snippet = textwrap.shorten(' '.join(query_text.split()), width=100, placeholder="...") + metric_str = f"{metric:,.2f}" if isinstance(metric, float) else f"{metric:,}" + self.stdout.write(f"{metric_str:<25} {query_snippet}") + except utils.DatabaseError as e: + self.stdout.write(self.style.ERROR(f"Could not generate report '{title}': {e}")) \ No newline at end of file diff --git a/docs/database_report_example.txt b/docs/database_report_example.txt new file mode 100644 index 000000000..180b7ff62 --- /dev/null +++ b/docs/database_report_example.txt @@ -0,0 +1,381 @@ +Database Diagnostic Report +Report generated at: 2025-07-24T18:48:44.435595+00:00 + +Table Sizes and Row Counts +Based on database schema inspection. +Running query: +SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), + (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I', table_name), false, true, '')))[1]::text::bigint +FROM information_schema.tables +WHERE table_schema = 'public' AND table_name NOT LIKE 'pg_%%' ORDER BY 3 DESC; +Table Name Size Rows +---------------------------------------------------------------------------------------------------- ---------------------------------------- -------------------- +auth_permission 136 kB 357 +django_migrations 72 kB 293 +dab_rbac_roledefinition_permissions 104 kB 228 +django_content_type 40 kB 90 +main_rbac_roles_parents 72 kB 65 +dab_rbac_dabpermission 56 kB 49 +main_rbac_roles 96 kB 39 +dab_rbac_roledefinition 112 kB 37 +main_hostmetricsummarymonthly 40 kB 36 +main_activitystream 48 kB 20 +main_credentialtype 112 kB 8 +main_unifiedjobtemplate 176 kB 5 +main_activitystream_credential 72 kB 5 +dab_rbac_roleevaluation 96 kB 4 +main_activitystream_instance_group 72 kB 4 +main_systemjobtemplate 24 kB 3 +main_schedule 128 kB 3 +dab_resource_registry_resourcetype 72 kB 3 +conf_setting 48 kB 3 +main_instancegroup 128 kB 2 +main_activitystream_job_template 72 kB 2 +main_credential 160 kB 2 +main_activitystream_user 72 kB 2 +main_activitystream_organization 72 kB 2 +main_instancegroup_instances 72 kB 2 +dab_resource_registry_resource 128 kB 2 +main_executionenvironment 128 kB 2 +main_activitystream_execution_environment 72 kB 2 +django_site 56 kB 1 +auth_user 64 kB 1 +main_inventory 176 kB 1 +main_organization 320 kB 1 +main_project 144 kB 1 +main_jobtemplate 128 kB 1 +main_organizationgalaxycredentialmembership 72 kB 1 +main_host 152 kB 1 +main_instance 72 kB 1 +main_activitystream_host 72 kB 1 +main_activitystream_inventory 72 kB 1 +main_activitystream_project 72 kB 1 +main_rbac_roles_members 72 kB 1 +main_activitystream_role 72 kB 1 +main_towerschedulestate 56 kB 1 +main_unifiedjobtemplate_credentials 72 kB 1 +dab_rbac_roleuserassignment 128 kB 1 +dab_rbac_objectrole 96 kB 1 +dab_resource_registry_serviceid 24 kB 1 +main_activitystream_instance 72 kB 1 +main_receptoraddress 64 kB 1 +auth_group_permissions 32 kB 0 +main_joblaunchconfig 40 kB 0 +main_custominventoryscript 32 kB 0 +main_activitystream_schedule 32 kB 0 +main_activitystream_team 32 kB 0 +main_activitystream_unified_job 32 kB 0 +main_activitystream_unified_job_template 32 kB 0 +main_group 48 kB 0 +main_jobevent 0 bytes 0 +main_host_inventory_sources 32 kB 0 +main_group_inventory_sources 32 kB 0 +main_activitystream_ad_hoc_command 32 kB 0 +main_activitystream_inventory_source 32 kB 0 +main_activitystream_inventory_update 32 kB 0 +main_activitystream_job 32 kB 0 +main_workflowjobtemplate_notification_templates_approvals 32 kB 0 +main_unifiedjob 136 kB 0 +main_activitystream_project_update 32 kB 0 +main_notificationtemplate 48 kB 0 +main_notification 24 kB 0 +main_inventoryupdateevent 0 bytes 0 +main_organization_notification_templates_approvals 32 kB 0 +main_activitystream_notification 32 kB 0 +main_activitystream_notification_template 32 kB 0 +main_organization_notification_templates_error 32 kB 0 +main_organization_notification_templates_success 32 kB 0 +main_unifiedjob_notifications 32 kB 0 +main_unifiedjobtemplate_notification_templates_error 32 kB 0 +main_unifiedjobtemplate_notification_templates_success 32 kB 0 +main_projectupdateevent 0 bytes 0 +main_workflowjob 48 kB 0 +main_team 72 kB 0 +main_rbac_role_ancestors 56 kB 0 +main_label 48 kB 0 +main_activitystream_label 32 kB 0 +main_unifiedjob_labels 32 kB 0 +main_unifiedjobtemplate_labels 32 kB 0 +main_workflowjobnode 72 kB 0 +main_adhoccommandevent 0 bytes 0 +auth_group 24 kB 0 +main_workflowjobtemplate 64 kB 0 +main_workflowjobnode_always_nodes 32 kB 0 +main_workflowjobnode_failure_nodes 32 kB 0 +main_workflowjobnode_success_nodes 32 kB 0 +main_workflowjobtemplatenode_always_nodes 32 kB 0 +_unpartitioned_main_jobevent 80 kB 0 +main_workflowjobtemplatenode 64 kB 0 +main_workflowjobtemplatenode_failure_nodes 32 kB 0 +main_workflowjobtemplatenode_success_nodes 32 kB 0 +main_activitystream_workflow_job 32 kB 0 +main_activitystream_workflow_job_node 32 kB 0 +main_activitystream_workflow_job_template 32 kB 0 +main_activitystream_workflow_job_template_node 32 kB 0 +main_systemjobevent 0 bytes 0 +main_activitystream_workflow_approval_template 32 kB 0 +main_smartinventorymembership 32 kB 0 +main_activitystream_workflow_approval 32 kB 0 +main_workflowapproval 24 kB 0 +main_activitystream_credential_type 32 kB 0 +main_workflowapprovaltemplate 8192 bytes 0 +main_unifiedjobtemplate_notification_templates_started 32 kB 0 +main_inventory_labels 32 kB 0 +main_instancelink 32 kB 0 +main_unifiedjob_credentials 32 kB 0 +main_activitystream_receptor_address 32 kB 0 +main_job 56 kB 0 +main_eventquery 24 kB 0 +dab_rbac_roleteamassignment 64 kB 0 +main_organization_notification_templates_started 32 kB 0 +main_unifiedjobtemplateinstancegroupmembership 32 kB 0 +main_organizationinstancegroupmembership 32 kB 0 +main_inventoryinstancegroupmembership 32 kB 0 +dab_rbac_objectrole_provides_teams 32 kB 0 +dab_rbac_roleevaluationuuid 48 kB 0 +main_indirectmanagednodeaudit 56 kB 0 +main_credentialinputsource 56 kB 0 +auth_user_user_permissions 32 kB 0 +flags_flagstate 16 kB 0 +django_session 32 kB 0 +main_schedule_credentials 32 kB 0 +main_workflowjobnode_credentials 32 kB 0 +main_workflowjobtemplatenode_credentials 32 kB 0 +main_joblaunchconfig_credentials 32 kB 0 +main_joblaunchconfig_labels 32 kB 0 +main_schedule_labels 32 kB 0 +main_workflowjobnode_labels 32 kB 0 +main_workflowjobtemplatenode_labels 32 kB 0 +main_usersessionmembership 32 kB 0 +_unpartitioned_main_inventoryupdateevent 48 kB 0 +_unpartitioned_main_projectupdateevent 64 kB 0 +_unpartitioned_main_systemjobevent 48 kB 0 +main_workflowjobtemplatenodebaseinstancegroupmembership 32 kB 0 +main_workflowjobnodebaseinstancegroupmembership 32 kB 0 +main_inventorygroupvariableswithhistory 40 kB 0 +main_workflowjobinstancegroupmembership 32 kB 0 +main_scheduleinstancegroupmembership 32 kB 0 +main_joblaunchconfiginstancegroupmembership 32 kB 0 +_unpartitioned_main_adhoccommandevent 64 kB 0 +auth_user_groups 32 kB 0 +main_adhoccommand 32 kB 0 +main_inventorysource 32 kB 0 +main_inventoryupdate 40 kB 0 +main_projectupdate 32 kB 0 +main_hostmetric 56 kB 0 +main_inventoryconstructedinventorymembership 32 kB 0 +main_systemjob 24 kB 0 +main_unifiedjob_dependent_jobs 32 kB 0 +main_group_hosts 32 kB 0 +main_group_parents 32 kB 0 +main_jobhostsummary 56 kB 0 +main_activitystream_group 32 kB 0 + +Model Object Counts +Based on the Django model layer. + +Model (app.model) Total Objects +---------------------------------------------------------------------------------------------------- ------------------------------ +auth.Group 0 +auth.Permission 357 +auth.User 1 +conf.Setting 3 +contenttypes.ContentType 90 +dab_rbac.DABPermission 49 +dab_rbac.ObjectRole 1 +dab_rbac.RoleDefinition 37 +dab_rbac.RoleEvaluation 4 +dab_rbac.RoleEvaluationUUID 0 +dab_rbac.RoleTeamAssignment 0 +dab_rbac.RoleUserAssignment 1 +dab_resource_registry.Resource 2 +dab_resource_registry.ResourceType 3 +dab_resource_registry.ServiceID 1 +flags.FlagState 0 +main.ActivityStream 20 +main.AdHocCommand 0 +main.AdHocCommandEvent 0 +main.Credential 2 +main.CredentialInputSource 0 +main.CredentialType 8 +main.CustomInventoryScript 0 +main.EventQuery 0 +main.ExecutionEnvironment 2 +main.Group 0 +main.Host 1 +main.HostMetric 0 +main.HostMetricSummaryMonthly 36 +main.IndirectManagedNodeAudit 0 +main.Instance 1 +main.InstanceGroup 2 +main.InstanceLink 0 +main.Inventory 1 +main.InventoryConstructedInventoryMembership 0 +main.InventoryGroupVariablesWithHistory 0 +main.InventoryInstanceGroupMembership 0 +main.InventorySource 0 +main.InventoryUpdate 0 +main.InventoryUpdateEvent 0 +main.Job 0 +main.JobEvent 0 +main.JobHostSummary 0 +main.JobLaunchConfig 0 +main.JobLaunchConfigInstanceGroupMembership 0 +main.JobTemplate 1 +main.Label 0 +main.Notification 0 +main.NotificationTemplate 0 +main.Organization 1 +main.OrganizationGalaxyCredentialMembership 1 +main.OrganizationInstanceGroupMembership 0 +main.Project 1 +main.ProjectUpdate 0 +main.ProjectUpdateEvent 0 +main.ReceptorAddress 1 +main.Role 39 +main.RoleAncestorEntry Error counting +main.Schedule 3 +main.ScheduleInstanceGroupMembership 0 +main.SmartInventoryMembership 0 +main.SystemJob 0 +main.SystemJobEvent 0 +main.SystemJobTemplate 3 +main.Team 0 +main.TowerScheduleState 1 +main.UnifiedJob 0 +main.UnifiedJobDeprecatedStdout 0 +main.UnifiedJobTemplate 5 +main.UnifiedJobTemplateInstanceGroupMembership 0 +main.UserSessionMembership 0 +main.WorkflowApproval 0 +main.WorkflowApprovalTemplate 0 +main.WorkflowJob 0 +main.WorkflowJobInstanceGroupMembership 0 +main.WorkflowJobNode 0 +main.WorkflowJobNodeBaseInstanceGroupMembership 0 +main.WorkflowJobTemplate 0 +main.WorkflowJobTemplateNode 0 +main.WorkflowJobTemplateNodeBaseInstanceGroupMembership 0 +sessions.Session 0 +sites.Site 1 + +PostgreSQL Activity Report (pg_stat_activity) + +Top 10 Longest Running Active Queries +Running query: +SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, backend_start, left(query, 100) as query_snippet FROM pg_stat_activity WHERE state IS NOT NULL AND state = 'active' ORDER BY query_start ASC LIMIT 10 +PID User Client State Wait Event Duration Query Snippet +-------------------- ------------------------------ ---------------------------------------- ------------------------------ -------------------------------------------------- -------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------- +4760 awx 172.23.0.4 active 0:00:00 SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, bac + +Top 10 Oldest "Idle in Transaction" Connections +Running query: +SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, backend_start, left(query, 100) as query_snippet FROM pg_stat_activity WHERE state IS NOT NULL AND state = 'idle in transaction' ORDER BY state_change ASC LIMIT 10 +No matching activity found. + +Top 10 Connections by Wait Time +Running query: +SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, backend_start, left(query, 100) as query_snippet FROM pg_stat_activity WHERE state IS NOT NULL AND wait_event IS NOT NULL ORDER BY state_change ASC LIMIT 10 +PID User Client State Wait Event Wait Time Query Snippet +-------------------- ------------------------------ ---------------------------------------- ------------------------------ -------------------------------------------------- -------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------- +49 awx 172.23.0.4 idle Client:ClientRead 23:38:03 SELECT "conf_setting"."id", "conf_setting"."created", "conf_setting"."modified", "conf_setting"."key +50 awx 172.23.0.4 idle Client:ClientRead 23:38:02 LISTEN "tower_settings_change"; +56 awx 172.23.0.4 idle Client:ClientRead 23:37:46 LISTEN web_ws_heartbeat +4758 awx 172.23.0.4 idle Client:ClientRead 0:00:13 SET idle_session_timeout = '0' +52 awx 172.23.0.4 idle Client:ClientRead 0:00:00 SELECT pg_notify('web_ws_heartbeat', '{"hostname": "awx-1", "ip": null, "action": "online"}'); +48 awx 172.23.0.4 idle Client:ClientRead 0:00:00 SELECT pg_notify($1, $2); + +Top 10 Oldest Open Connections +Running query: +SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, backend_start, left(query, 100) as query_snippet FROM pg_stat_activity WHERE state IS NOT NULL ORDER BY backend_start ASC LIMIT 10 +PID User Client State Wait Event Connection Age Query Snippet +-------------------- ------------------------------ ---------------------------------------- ------------------------------ -------------------------------------------------- -------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------- +48 awx 172.23.0.4 idle Client:ClientRead 23:38:06 SELECT pg_notify($1, $2); +49 awx 172.23.0.4 idle Client:ClientRead 23:38:04 SELECT "conf_setting"."id", "conf_setting"."created", "conf_setting"."modified", "conf_setting"."key +50 awx 172.23.0.4 idle Client:ClientRead 23:38:03 LISTEN "tower_settings_change"; +52 awx 172.23.0.4 idle Client:ClientRead 23:37:59 SELECT pg_notify('web_ws_heartbeat', '{"hostname": "awx-1", "ip": null, "action": "online"}'); +56 awx 172.23.0.4 idle Client:ClientRead 23:37:46 LISTEN web_ws_heartbeat +4758 awx 172.23.0.4 idle Client:ClientRead 0:00:13 SET idle_session_timeout = '0' +4760 awx 172.23.0.4 active 0:00:01 SELECT pid, usename, client_addr, state, wait_event_type, wait_event, query_start, state_change, bac + +PostgreSQL Statement Statistics (pg_stat_statements) + +Top 10 Most Frequent Queries +Running query: +SELECT calls, query FROM pg_stat_statements ORDER BY calls DESC LIMIT 10; +Calls Query Snippet +-------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +3,945 SELECT set_config($1, $2, $3) +3,473 SELECT pg_advisory_unlock($1) +3,473 SELECT pg_try_advisory_lock($1) +3,075 SHOW idle_session_timeout +3,075 SHOW idle_in_transaction_session_timeout +3,075 SET idle_in_transaction_session_timeout = 360000 +3,075 SET idle_session_timeout = '0' +3,075 SET idle_in_transaction_session_timeout = '0' +3,075 SET idle_session_timeout = 360000 +2,312 BEGIN + +Top 10 Slowest Queries (by mean execution time) +Running query: +SELECT mean_exec_time, query FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10; +Mean Time (ms) Query Snippet +-------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +71.08 SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), (xpath($1,... +8.37 +4.89 SELECT c.relname, CASE WHEN c.relispartition THEN $1 WHEN c.relkind IN ($2, $3) THEN $4 ELSE $5... +3.50 DECLARE "_django_curs_140363913213760_sync_775" NO SCROLL CURSOR WITH HOLD FOR SELECT... +3.28 DECLARE "_django_curs_140363913213760_sync_10" NO SCROLL CURSOR WITH HOLD FOR SELECT... +2.48 DECLARE "_django_curs_140363913213760_sync_5" NO SCROLL CURSOR WITH HOLD FOR SELECT... +2.46 DECLARE "_django_curs_140363913213760_sync_8" NO SCROLL CURSOR WITH HOLD FOR SELECT... +2.30 DECLARE "_django_curs_140363913213760_sync_11" NO SCROLL CURSOR WITH HOLD FOR SELECT... +2.25 DECLARE "_django_curs_140363913213760_sync_7" NO SCROLL CURSOR WITH HOLD FOR SELECT... +2.18 DECLARE "_django_curs_140363913213760_sync_9" NO SCROLL CURSOR WITH HOLD FOR SELECT... + +Top 10 Queries by Total Execution Time (CPU) +Running query: +SELECT total_exec_time, query FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 10; +Total Time (ms) Query Snippet +-------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +824.04 SELECT set_config($1, $2, $3) +568.67 SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), (xpath($1,... +152.22 SELECT ("main_instancegroup_instances"."instancegroup_id") AS... +102.31 SELECT "main_unifiedjob"."id", "main_unifiedjob"."polymorphic_ctype_id",... +96.46 SELECT "main_unifiedjob"."id", "main_unifiedjob"."polymorphic_ctype_id",... +83.52 UPDATE "main_instance" SET "errors" = $1, "last_seen" = $2::timestamptz, "last_health_check" =... +81.43 SELECT pg_try_advisory_lock($1) +73.42 SELECT c.relname, CASE WHEN c.relispartition THEN $1 WHEN c.relkind IN ($2, $3) THEN $4 ELSE $5... +58.63 SELECT "main_schedule"."id", "main_schedule"."created", "main_schedule"."modified",... +58.05 SELECT "main_instance"."id", "main_instance"."hostname", "main_instance"."capacity",... + +Top 10 Queries by I/O Read +Running query: +SELECT (shared_blks_read + local_blks_read + temp_blks_read), query FROM pg_stat_statements ORDER BY (shared_blks_read + local_blks_read + temp_blks_read) DESC LIMIT 10; +Blocks Read Query Snippet +-------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +985 SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), (xpath($1,... +58 SELECT c.relname, CASE WHEN c.relispartition THEN $1 WHEN c.relkind IN ($2, $3) THEN $4 ELSE $5... +5 UPDATE "main_credentialtype" SET "created" = $1::timestamptz, "modified" = $2::timestamptz,... +2 SELECT "django_migrations"."id", "django_migrations"."app", "django_migrations"."name",... +2 SELECT "conf_setting"."id", "conf_setting"."created", "conf_setting"."modified",... +2 SELECT "django_migrations"."id", "django_migrations"."app", "django_migrations"."name",... +2 +1 SELECT $1 FROM pg_extension WHERE extname = $2 +1 UPDATE "main_instance" SET "errors" = $1, "last_seen" = $2::timestamptz, "last_health_check" =... +1 SELECT "main_credentialtype"."id", "main_credentialtype"."created",... + +Top 10 Queries by Buffer Cache Usage (Memory) +Running query: +SELECT (shared_blks_hit + shared_blks_read), query FROM pg_stat_statements ORDER BY (shared_blks_hit + shared_blks_read) DESC LIMIT 10; +Shared Blocks Hit+Read Query Snippet +-------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +140,543 SELECT table_name, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))), (xpath($1,... +33,563 SELECT c.relname, CASE WHEN c.relispartition THEN $1 WHEN c.relkind IN ($2, $3) THEN $4 ELSE $5... +8,071 SELECT ("main_instancegroup_instances"."instancegroup_id") AS... +7,680 UPDATE "main_instance" SET "errors" = $1, "last_seen" = $2::timestamptz, "last_health_check" =... +5,765 SELECT "main_unifiedjob"."id", "main_unifiedjob"."polymorphic_ctype_id",... +5,377 SELECT "main_schedule"."id", "main_schedule"."created", "main_schedule"."modified",... +4,612 SELECT "main_instance"."id", "main_instance"."hostname", "main_instance"."capacity",... +3,459 SELECT "main_unifiedjob"."id", "main_unifiedjob"."polymorphic_ctype_id",... +3,444 SELECT "main_unifiedjob"."id", "main_unifiedjob"."polymorphic_ctype_id",... +2,321 UPDATE "main_towerschedulestate" SET "schedule_last_run" = $1::timestamptz WHERE... diff --git a/docs/database_reporting.md b/docs/database_reporting.md new file mode 100644 index 000000000..04e4bc538 --- /dev/null +++ b/docs/database_reporting.md @@ -0,0 +1,96 @@ +# PostgreSQL Diagnostic Reporting + +This document describes the `database_resource_report` management command, a tool for analyzing PostgreSQL database table sizes and query activity. + +**Note:** This command is only compatible with PostgreSQL. + +## How to Run the Command + +To make the command available in your project, ensure that the `ansible_base.resource_registry` application is included in your `INSTALLED_APPS` setting in your project's `settings.py`. + +```python +# settings.py +INSTALLED_APPS = [ + ... + 'ansible_base.resource_registry', + ... +] +``` + +Once the app is installed, you can run the command using your project's `manage.py` script. The exact invocation may vary depending on the consuming application. + +**AWX Development Environment:** +```bash +# Run from within the tools_awx_1 container +awx-manage database_resource_report +``` + +**AAP Gateway Environment:** +```bash +# Run from within the aap-gateway container +aap-gateway-manage database_resource_report +``` + +The command accepts no arguments and will run all available reports sequentially. + +--- + +## Understanding the Report Output + +The report is divided into several sections, providing a comprehensive overview of the database from different perspectives. Each section also includes the literal SQL query that was run to generate the data. + +### 1. Table Sizes and Row Counts + +This section provides a raw, physical overview of the database tables. + +- **What it shows:** A list of all tables in the `public` schema, their total size on disk (including indexes and TOAST data), and the estimated number of rows. +- **How it's sorted:** By the number of rows, in descending order. +- **Usefulness:** Helps you quickly identify the largest tables in your database, which are often the most important ones to monitor for bloat and performance. + +### 2. Model Object Counts + +This section provides an application-level view of the data, based on Django's models. + +- **What it shows:** A list of all registered Django models in the application and the total number of objects (rows) for each. +- **How it's sorted:** Alphabetically by the model's label (`app_name.ModelName`). +- **Usefulness:** Helps you understand data distribution from the perspective of the application's logic. It can also highlight tables that are not managed by a Django model. + +### 3. PostgreSQL Activity Report (`pg_stat_activity`) + +This section provides a **real-time snapshot** of all current connections and their activity. It is only available for PostgreSQL databases. It is broken down into several sorted views to help diagnose immediate issues. + +#### Top 10 Longest Running Active Queries +- **What it shows:** Queries that are currently in the `active` state, sorted by how long they have been running. +- **Usefulness:** This is the most critical report for diagnosing live performance problems. It immediately shows you which queries are currently consuming resources and may be stuck or running inefficiently. + +#### Top 10 Oldest "Idle in Transaction" Connections +- **What it shows:** Connections that have an open transaction (`BEGIN` has been issued) but are not currently running a query. +- **Usefulness:** These connections can be dangerous. They can hold locks for long periods, blocking other queries, and prevent database cleanup processes (`VACUUM`) from working, which leads to table bloat. This report helps you find and investigate them. + +#### Top 10 Connections by Wait Time +- **What it shows:** Connections that are currently waiting for a specific event (e.g., waiting for a lock, for disk I/O, or for the client to send data). +- **Usefulness:** This report directly identifies bottlenecks. If many queries are waiting on the same type of event, it points to a specific area of contention. + +#### Top 10 Oldest Open Connections +- **What it shows:** All connections sorted by the time they were first established. +- **Usefulness:** A general health check that can help you spot problems with connection pooling (applications not closing connections properly) or find very old, forgotten sessions. + +### 4. PostgreSQL Statement Statistics (`pg_stat_statements`) + +This section provides **historical performance data** about all queries that have been run against the database. It is only available for PostgreSQL and requires the `pg_stat_statements` extension to be enabled. + +#### Enabling `pg_stat_statements` +If the extension is not enabled, the report will provide instructions on how to do so. You must be a database superuser. The order of operations is important. + +1. **Configure the server:** Add or modify the `shared_preload_libraries` line in your `postgresql.conf` file. This tells PostgreSQL to load the extension into memory on the next startup. + ```ini + shared_preload_libraries = 'pg_stat_statements' + ``` +2. **Restart the PostgreSQL server:** This is required to apply the configuration change and load the library. +3. **Create the extension:** Connect to your database and run `CREATE EXTENSION pg_stat_statements;`. This command initializes the views and functions for the extension, which can only be done after the library has been loaded. + +#### The Reports +- **Top 10 Most Frequent Queries:** Shows which queries are executed most often. Optimizing these can have a large impact on overall performance. +- **Top 10 Slowest Queries (by mean execution time):** Shows the queries that take the longest on average to complete. +- **Top 10 Queries by Total Execution Time (CPU):** This is often the most important report. It shows which queries have consumed the most total database time (i.e., `frequency * average_time`). These are the best candidates for optimization. +- **Top 10 Queries by I/O Read & Buffer Cache Usage:** These reports show which queries are the most demanding on disk and memory resources, respectively. They can help identify queries that would benefit from new indexes or memory tuning.