Skip to content

Commit f851f55

Browse files
samrosesamrose
committed
tests: create a tmp image that has coredumps enabled for testing
these changes will NOT be merged into develop or promoted to production
1 parent 24430ee commit f851f55

File tree

3 files changed

+227
-3
lines changed

3 files changed

+227
-3
lines changed

‎ansible/tasks/setup-coredumps.yml

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
---
2+
# Setup coredump collection for debugging PostgreSQL crashes
3+
# This configuration is temporary and should be cleaned up after debugging
4+
5+
- name: Create coredump directory
6+
become: yes
7+
file:
8+
path: /pg_coredump_debug
9+
state: directory
10+
mode: '0777'
11+
owner: root
12+
group: root
13+
when: stage2_nix
14+
15+
- name: Configure kernel core pattern
16+
become: yes
17+
ansible.posix.sysctl:
18+
name: kernel.core_pattern
19+
value: '/pg_coredump_debug/core.%e.%p.%t'
20+
state: present
21+
sysctl_file: /etc/sysctl.d/99-coredump.conf
22+
reload: yes
23+
when: stage2_nix
24+
25+
- name: Enable core dumps with PID
26+
become: yes
27+
ansible.posix.sysctl:
28+
name: kernel.core_uses_pid
29+
value: '1'
30+
state: present
31+
sysctl_file: /etc/sysctl.d/99-coredump.conf
32+
reload: yes
33+
when: stage2_nix
34+
35+
- name: Enable SUID dumpable
36+
become: yes
37+
ansible.posix.sysctl:
38+
name: fs.suid_dumpable
39+
value: '1'
40+
state: present
41+
sysctl_file: /etc/sysctl.d/99-coredump.conf
42+
reload: yes
43+
when: stage2_nix
44+
45+
- name: Create systemd system.conf.d directory
46+
become: yes
47+
file:
48+
path: /etc/systemd/system.conf.d
49+
state: directory
50+
mode: '0755'
51+
owner: root
52+
group: root
53+
when: stage2_nix
54+
55+
- name: Configure systemd for coredumps
56+
become: yes
57+
copy:
58+
content: |
59+
# Temporary coredump configuration - remove after debugging
60+
[Manager]
61+
DefaultLimitCORE=infinity
62+
DumpCore=yes
63+
dest: /etc/systemd/system.conf.d/50-coredump.conf
64+
mode: '0644'
65+
owner: root
66+
group: root
67+
when: stage2_nix
68+
69+
- name: Configure security limits for coredumps
70+
become: yes
71+
blockinfile:
72+
path: /etc/security/limits.conf
73+
block: |
74+
# Temporary coredump limits - remove after debugging
75+
* hard core 50000
76+
* soft core 50000
77+
marker: "# {mark} ANSIBLE MANAGED BLOCK - COREDUMP"
78+
state: present
79+
when: stage2_nix
80+
81+
- name: Check current GRUB_CMDLINE_LINUX_DEFAULT
82+
become: yes
83+
shell: grep '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub || echo 'GRUB_CMDLINE_LINUX_DEFAULT=""'
84+
register: current_grub_cmdline
85+
changed_when: false
86+
when: stage2_nix
87+
88+
- name: Check for GRUB_CMDLINE_LINUX_DEFAULT in grub.d files
89+
become: yes
90+
shell: grep -l '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub.d/* 2>/dev/null || true
91+
register: grub_d_files
92+
changed_when: false
93+
when: stage2_nix
94+
95+
- name: Update GRUB configuration for coredump_filter in main grub file
96+
become: yes
97+
lineinfile:
98+
path: /etc/default/grub
99+
regexp: '^GRUB_CMDLINE_LINUX_DEFAULT='
100+
line: 'GRUB_CMDLINE_LINUX_DEFAULT="{{ current_grub_cmdline.stdout | regex_replace("^GRUB_CMDLINE_LINUX_DEFAULT=\"(.*)\"$", "\\1") | regex_replace("coredump_filter=[0-9]+", "") | trim }} coredump_filter=49"'
101+
backrefs: no
102+
when:
103+
- stage2_nix
104+
- grub_d_files.stdout == ""
105+
notify: update grub
106+
107+
- name: Update GRUB configuration for coredump_filter in grub.d files
108+
become: yes
109+
lineinfile:
110+
path: "{{ item }}"
111+
regexp: '^GRUB_CMDLINE_LINUX_DEFAULT='
112+
line: 'GRUB_CMDLINE_LINUX_DEFAULT="{{ lookup("pipe", "grep \"^GRUB_CMDLINE_LINUX_DEFAULT=\" " + item + " | sed \"s/^GRUB_CMDLINE_LINUX_DEFAULT=\\\"\\(.*\\)\\\"$/\\1/\" | sed \"s/coredump_filter=[0-9]*//g\"") | trim }} coredump_filter=49"'
113+
backrefs: no
114+
with_items: "{{ grub_d_files.stdout_lines }}"
115+
when:
116+
- stage2_nix
117+
- grub_d_files.stdout != ""
118+
notify: update grub
119+
120+
- name: Install gdb for debugging
121+
become: yes
122+
apt:
123+
pkg:
124+
- gdb
125+
state: present
126+
update_cache: yes
127+
cache_valid_time: 3600
128+
when: stage2_nix
129+
130+
- name: Create root .gdbinit configuration
131+
become: yes
132+
copy:
133+
content: |
134+
# GDB configuration for PostgreSQL debugging
135+
# Note: debug-file-directory and substitute-path need to be set manually
136+
# after installing debug symbols and source files from nix
137+
set auto-load safe-path /
138+
# Example paths - will be set dynamically when debugging:
139+
# set debug-file-directory /nix/store/<hash>-postgresql-<version>-debug/lib/debug/
140+
# set substitute-path ./ /nix/store/<hash>-postgresql-<version>-src-<version>/
141+
dest: /root/.gdbinit
142+
mode: '0644'
143+
owner: root
144+
group: root
145+
when: stage2_nix
146+
147+
- name: Create postgres user .gdbinit configuration
148+
become: yes
149+
become_user: postgres
150+
copy:
151+
content: |
152+
# GDB configuration for PostgreSQL debugging
153+
# Note: debug-file-directory and substitute-path need to be set manually
154+
# after installing debug symbols and source files from nix
155+
set auto-load safe-path /
156+
# Example paths - will be set dynamically when debugging:
157+
# set debug-file-directory /nix/store/<hash>-postgresql-<version>-debug/lib/debug/
158+
# set substitute-path ./ /nix/store/<hash>-postgresql-<version>-src-<version>/
159+
dest: /var/lib/postgresql/.gdbinit
160+
mode: '0644'
161+
owner: postgres
162+
group: postgres
163+
when: stage2_nix
164+
165+
- name: Create coredump setup documentation
166+
become: yes
167+
copy:
168+
content: |
169+
# PostgreSQL Coredump Configuration
170+
171+
This AMI has been configured to collect PostgreSQL coredumps for debugging.
172+
173+
## Configuration Files Modified:
174+
- /etc/sysctl.d/99-coredump.conf - Kernel coredump settings
175+
- /etc/systemd/system.conf.d/50-coredump.conf - Systemd coredump settings
176+
- /etc/security/limits.conf - User limits for coredump size
177+
- /etc/default/grub - GRUB configuration for coredump_filter
178+
179+
## Coredump Location:
180+
Coredumps are saved to: /pg_coredump_debug/
181+
182+
## Debug Symbols:
183+
Debug symbols and source files are already installed via nix during the build.
184+
To find the paths:
185+
```
186+
sudo -u postgres nix profile list | grep postgresql_.*_debug
187+
sudo -u postgres nix profile list | grep postgresql_.*_src
188+
```
189+
190+
## Using GDB:
191+
To analyze a coredump:
192+
```
193+
sudo gdb postmaster -c /pg_coredump_debug/<core_file>
194+
```
195+
196+
Then in gdb, set the debug symbols path:
197+
```
198+
symbol-file /nix/store/<hash>-postgresql-<version>-debug/lib/debug/postgres
199+
```
200+
201+
## Cleanup:
202+
To disable coredump collection after debugging:
203+
1. Remove /etc/sysctl.d/99-coredump.conf
204+
2. Remove /etc/systemd/system.conf.d/50-coredump.conf
205+
3. Remove coredump block from /etc/security/limits.conf
206+
4. Delete coredumps from /pg_coredump_debug/
207+
5. Run: sudo sysctl -p && sudo systemctl daemon-reload
208+
dest: /pg_coredump_debug/README.md
209+
mode: '0644'
210+
owner: root
211+
group: root
212+
when: stage2_nix
213+
214+
# Handler for updating GRUB
215+
- name: Define update grub handler
216+
become: yes
217+
command: update-grub
218+
listen: update grub
219+
when: stage2_nix

‎ansible/tasks/setup-postgres.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,11 @@
175175
import_tasks: tasks/stage2-setup-postgres.yml
176176
when: stage2_nix
177177

178+
# Setup coredump collection for debugging
179+
- name: Setup coredump collection
180+
import_tasks: tasks/setup-coredumps.yml
181+
when: stage2_nix
182+
178183
# init DB
179184
- name: Create directory on data volume
180185
file:

‎ansible/vars.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ postgres_major:
99

1010
# Full version strings for each major version
1111
postgres_release:
12-
postgresorioledb-17: "17.5.1.017-orioledb"
13-
postgres17: "17.4.1.074"
14-
postgres15: "15.8.1.131"
12+
postgresorioledb-17: "17.5.1.017-orioledb-coredump-1"
13+
postgres17: "17.4.1.074-coredump-1"
14+
postgres15: "15.8.1.131-coredump-1"
1515

1616
# Non Postgres Extensions
1717
pgbouncer_release: "1.19.0"

0 commit comments

Comments
 (0)