Skip to content

Commit 6622cdb

Browse files
pantierrapantierra
authored
feat: Add more pgstac config options. (#340)
1 parent 904ed78 commit 6622cdb

File tree

10 files changed

+491
-13
lines changed

10 files changed

+491
-13
lines changed

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Added
11+
12+
- Expose PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are being dynamically applied via templated SQL during bootstrap.
13+
- Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning
14+
- Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`)
15+
- Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`)
16+
- Automatic extent updater CronJob created when `update_collection_extent` is "false" (configurable schedule via `extentUpdater.schedule`)
17+
1018
## [0.7.13] - 2025-11-04
1119

1220
### Added

‎charts/eoapi/initdb-data/settings/pgstac-settings.sql‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Apply pgstac settings
2+
-- These settings are configured via Helm values at pgstacBootstrap.settings.pgstacSettings
3+
4+
-- Queue settings
5+
DELETE FROM pgstac.pgstac_settings WHERE name = 'queue_timeout';
6+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('queue_timeout', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.queue_timeout }}');
7+
8+
DELETE FROM pgstac.pgstac_settings WHERE name = 'use_queue';
9+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('use_queue', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.use_queue }}');
10+
11+
-- Collection extent management
12+
DELETE FROM pgstac.pgstac_settings WHERE name = 'update_collection_extent';
13+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('update_collection_extent', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent }}');
14+
15+
-- Context settings
16+
DELETE FROM pgstac.pgstac_settings WHERE name = 'context';
17+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context }}');
18+
19+
DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count';
20+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_count }}');
21+
22+
DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost';
23+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_cost }}');
24+
25+
DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl';
26+
INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_stats_ttl }}');

‎charts/eoapi/templates/pgstacbootstrap/configmap.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ metadata:
1313
helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
1414
data:
1515
pgstac-settings.sql: |
16-
{{ $.Files.Get "initdb-data/settings/pgstac-settings.sql" | nindent 4 }}
16+
{{- tpl ($.Files.Get "initdb-data/settings/pgstac-settings.sql.tpl") $ | nindent 4 }}
1717
{{- if (index .Values "eoapi-notifier").enabled }}
1818
{{ $.Files.Get "initdb-data/settings/pgstac-notification-triggers.sql" | nindent 4 }}
1919
{{- end }}

‎charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{{- if .Values.pgstacBootstrap.enabled }}
2+
{{- if .Values.pgstacBootstrap.settings.pgstacSettings }}
3+
{{- if eq (default "false" .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent) "false" }}
4+
---
5+
apiVersion: batch/v1
6+
kind: CronJob
7+
metadata:
8+
name: {{ .Release.Name }}-pgstac-extent-updater
9+
labels:
10+
{{- include "eoapi.labels" . | nindent 4 }}
11+
app.kubernetes.io/component: pgstac-extents
12+
spec:
13+
schedule: {{ .Values.pgstacBootstrap.settings.extentUpdater.schedule | quote }}
14+
concurrencyPolicy: Forbid
15+
successfulJobsHistoryLimit: 1
16+
failedJobsHistoryLimit: 1
17+
jobTemplate:
18+
spec:
19+
template:
20+
metadata:
21+
labels:
22+
{{- include "eoapi.labels" . | nindent 12 }}
23+
app.kubernetes.io/component: pgstac-extents
24+
spec:
25+
restartPolicy: OnFailure
26+
containers:
27+
- name: extent-updater
28+
image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }}
29+
imagePullPolicy: IfNotPresent
30+
command:
31+
- "/bin/sh"
32+
- "-c"
33+
- |
34+
psql -c "SELECT update_collection_extents();"
35+
env:
36+
{{- include "eoapi.postgresqlEnv" . | nindent 14 }}
37+
resources:
38+
limits:
39+
cpu: "512m"
40+
memory: "1024Mi"
41+
requests:
42+
cpu: "256m"
43+
memory: "512Mi"
44+
{{- end }}
45+
{{- end }}
46+
{{- end }}

‎charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{{- if .Values.pgstacBootstrap.enabled }}
2+
{{- if .Values.pgstacBootstrap.settings.pgstacSettings }}
3+
{{- if eq (.Values.pgstacBootstrap.settings.pgstacSettings.use_queue | default "true") "true" }}
4+
---
5+
apiVersion: batch/v1
6+
kind: CronJob
7+
metadata:
8+
name: {{ .Release.Name }}-pgstac-queue-processor
9+
labels:
10+
{{- include "eoapi.labels" . | nindent 4 }}
11+
app.kubernetes.io/component: pgstac-queue
12+
spec:
13+
schedule: {{ .Values.pgstacBootstrap.settings.queueProcessor.schedule | quote }}
14+
concurrencyPolicy: Forbid
15+
successfulJobsHistoryLimit: 1
16+
failedJobsHistoryLimit: 1
17+
jobTemplate:
18+
spec:
19+
template:
20+
metadata:
21+
labels:
22+
{{- include "eoapi.labels" . | nindent 12 }}
23+
app.kubernetes.io/component: pgstac-queue
24+
spec:
25+
restartPolicy: OnFailure
26+
containers:
27+
- name: queue-processor
28+
image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }}
29+
imagePullPolicy: IfNotPresent
30+
command:
31+
- "/bin/sh"
32+
- "-c"
33+
- |
34+
psql -c "RUN run_queued_queries();"
35+
env:
36+
{{- include "eoapi.postgresqlEnv" . | nindent 14 }}
37+
resources:
38+
limits:
39+
cpu: "256m"
40+
memory: "512Mi"
41+
requests:
42+
cpu: "128m"
43+
memory: "256Mi"
44+
{{- end }}
45+
{{- end }}
46+
{{- end }}

‎charts/eoapi/tests/pgstac_config_tests.yaml‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
suite: pgstac configuration tests
2+
templates:
3+
- templates/pgstacbootstrap/configmap.yaml
4+
- templates/pgstacbootstrap/queue-processor.yaml
5+
- templates/pgstacbootstrap/extent-updater.yaml
6+
tests:
7+
# PgSTAC Settings Tests
8+
- it: should apply custom pgstac settings
9+
set:
10+
pgstacBootstrap:
11+
enabled: true
12+
settings:
13+
pgstacSettings:
14+
queue_timeout: "30 minutes"
15+
use_queue: "false"
16+
update_collection_extent: "true"
17+
context: "on"
18+
context_estimated_count: "50000"
19+
template: templates/pgstacbootstrap/configmap.yaml
20+
documentIndex: 0
21+
asserts:
22+
- matchRegex:
23+
path: data["pgstac-settings.sql"]
24+
pattern: "VALUES \\('queue_timeout', '30 minutes'\\)"
25+
- matchRegex:
26+
path: data["pgstac-settings.sql"]
27+
pattern: "VALUES \\('use_queue', 'false'\\)"
28+
- matchRegex:
29+
path: data["pgstac-settings.sql"]
30+
pattern: "VALUES \\('update_collection_extent', 'true'\\)"
31+
- matchRegex:
32+
path: data["pgstac-settings.sql"]
33+
pattern: "VALUES \\('context_estimated_count', '50000'\\)"
34+
35+
- it: should use default pgstac settings
36+
set:
37+
pgstacBootstrap:
38+
enabled: true
39+
template: templates/pgstacbootstrap/configmap.yaml
40+
documentIndex: 0
41+
asserts:
42+
- matchRegex:
43+
path: data["pgstac-settings.sql"]
44+
pattern: "VALUES \\('queue_timeout', '10 minutes'\\)"
45+
- matchRegex:
46+
path: data["pgstac-settings.sql"]
47+
pattern: "VALUES \\('use_queue', 'false'\\)"
48+
- matchRegex:
49+
path: data["pgstac-settings.sql"]
50+
pattern: "VALUES \\('update_collection_extent', 'true'\\)"
51+
52+
# Queue Processor Tests
53+
- it: should create queue processor when use_queue is true
54+
set:
55+
pgstacBootstrap:
56+
enabled: true
57+
settings:
58+
pgstacSettings:
59+
use_queue: "true"
60+
template: templates/pgstacbootstrap/queue-processor.yaml
61+
asserts:
62+
- hasDocuments:
63+
count: 1
64+
- isKind:
65+
of: CronJob
66+
- equal:
67+
path: metadata.name
68+
value: RELEASE-NAME-pgstac-queue-processor
69+
- equal:
70+
path: spec.schedule
71+
value: "0 * * * *"
72+
73+
- it: should NOT create queue processor when use_queue is false
74+
set:
75+
pgstacBootstrap:
76+
enabled: true
77+
settings:
78+
pgstacSettings:
79+
use_queue: "false"
80+
template: templates/pgstacbootstrap/queue-processor.yaml
81+
asserts:
82+
- hasDocuments:
83+
count: 0
84+
85+
- it: should use custom queue processor schedule
86+
set:
87+
pgstacBootstrap:
88+
enabled: true
89+
settings:
90+
pgstacSettings:
91+
use_queue: "true"
92+
queueProcessor:
93+
schedule: "*/15 * * * *"
94+
template: templates/pgstacbootstrap/queue-processor.yaml
95+
asserts:
96+
- equal:
97+
path: spec.schedule
98+
value: "*/15 * * * *"
99+
- matchRegex:
100+
path: spec.jobTemplate.spec.template.spec.containers[0].command[2]
101+
pattern: "RUN run_queued_queries\\(\\);"
102+
103+
# Extent Updater Tests
104+
- it: should create extent updater when update_collection_extent is false
105+
set:
106+
pgstacBootstrap:
107+
enabled: true
108+
settings:
109+
pgstacSettings:
110+
update_collection_extent: "false"
111+
template: templates/pgstacbootstrap/extent-updater.yaml
112+
asserts:
113+
- hasDocuments:
114+
count: 1
115+
- isKind:
116+
of: CronJob
117+
- equal:
118+
path: metadata.name
119+
value: RELEASE-NAME-pgstac-extent-updater
120+
- equal:
121+
path: spec.schedule
122+
value: "0 2 * * *"
123+
124+
- it: should NOT create extent updater when update_collection_extent is true
125+
set:
126+
pgstacBootstrap:
127+
enabled: true
128+
settings:
129+
pgstacSettings:
130+
update_collection_extent: "true"
131+
template: templates/pgstacbootstrap/extent-updater.yaml
132+
asserts:
133+
- hasDocuments:
134+
count: 0
135+
136+
- it: should use custom extent updater schedule
137+
set:
138+
pgstacBootstrap:
139+
enabled: true
140+
settings:
141+
pgstacSettings:
142+
update_collection_extent: "false"
143+
extentUpdater:
144+
schedule: "0 */6 * * *"
145+
template: templates/pgstacbootstrap/extent-updater.yaml
146+
asserts:
147+
- equal:
148+
path: spec.schedule
149+
value: "0 */6 * * *"
150+
- matchRegex:
151+
path: spec.jobTemplate.spec.template.spec.containers[0].command[2]
152+
pattern: "SELECT update_collection_extents\\(\\);"
153+
154+
# Combined scenario tests
155+
- it: should create both cronjobs with proper settings
156+
set:
157+
pgstacBootstrap:
158+
enabled: true
159+
settings:
160+
pgstacSettings:
161+
use_queue: "true"
162+
update_collection_extent: "false"
163+
templates:
164+
- templates/pgstacbootstrap/queue-processor.yaml
165+
- templates/pgstacbootstrap/extent-updater.yaml
166+
asserts:
167+
- hasDocuments:
168+
count: 1
169+
- isKind:
170+
of: CronJob
171+
172+
- it: should not create any cronjobs when disabled
173+
set:
174+
pgstacBootstrap:
175+
enabled: false
176+
templates:
177+
- templates/pgstacbootstrap/queue-processor.yaml
178+
- templates/pgstacbootstrap/extent-updater.yaml
179+
asserts:
180+
- hasDocuments:
181+
count: 0
182+
183+
# Resource limits tests
184+
- it: should set correct resource limits for queue processor
185+
set:
186+
pgstacBootstrap:
187+
enabled: true
188+
settings:
189+
pgstacSettings:
190+
use_queue: "true"
191+
template: templates/pgstacbootstrap/queue-processor.yaml
192+
asserts:
193+
- equal:
194+
path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu
195+
value: "256m"
196+
- equal:
197+
path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory
198+
value: "512Mi"
199+
200+
- it: should set correct resource limits for extent updater
201+
set:
202+
pgstacBootstrap:
203+
enabled: true
204+
settings:
205+
pgstacSettings:
206+
update_collection_extent: "false"
207+
template: templates/pgstacbootstrap/extent-updater.yaml
208+
asserts:
209+
- equal:
210+
path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu
211+
value: "512m"
212+
- equal:
213+
path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory
214+
value: "1024Mi"

0 commit comments

Comments
 (0)