Skip to content

Commit 24341ba

Browse files
mario-winklermario-winkler
committed
Add cronjobs for update collection extents and run queued queries
1 parent 0d7946d commit 24341ba

File tree

6 files changed

+117
-31
lines changed

6 files changed

+117
-31
lines changed

‎helm-chart/eoapi-data-management/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ kubeVersion: ">=1.23.0-0"
1515
# This is the chart version. This version number should be incremented each time you make changes
1616
# to the chart and its templates, including the app version.
1717
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18-
version: "0.3.0"
18+
version: "0.4.0"
1919

2020
# This is the version number of the application being deployed. This version number should be
2121
# incremented each time you make changes to the application. Versions are not expected to

‎helm-chart/eoapi-data-management/fixtures/collections.json‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎helm-chart/eoapi-data-management/fixtures/settings.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@ INSERT INTO pgstac.pgstac_settings (name, value)
1515
('queue_timeout', '10 minutes'),
1616
('format_cache', 'false'),
1717
('readonly', 'false'),
18-
('use_queue', 'false')
18+
('use_queue', 'true')
1919
ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;
2020
COMMIT;

‎helm-chart/eoapi-data-management/templates/configmap.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if .Values.collectionSetup.enabled }}
1+
{{- if .Values.load_collections.enabled }}
22
---
33
apiVersion: v1
44
kind: ConfigMap
@@ -28,7 +28,7 @@ data:
2828
#!/bin/bash
2929
apt update -y && apt install python3 python3-pip -y
3030
pip install pypgstac[psycopg]
31-
run-collection-setup.sh: |
31+
run-load-collections.sh: |
3232
#!/bin/bash
3333
bash /opt/initdb/apt-and-pip-install.sh
3434

‎helm-chart/eoapi-data-management/templates/job.yaml‎

Lines changed: 81 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
1-
{{- if .Values.collectionSetup.enabled }}
1+
{{- if .Values.load_collections.enabled }}
22
---
33
apiVersion: batch/v1
44
kind: Job
55
metadata:
6-
name: collection-setup-{{ $.Release.Name }}
6+
name: load-collections-{{ $.Release.Name }}
77
labels:
8-
app: collection-setup-{{ $.Release.Name }}
8+
app: load-collections-{{ $.Release.Name }}
99
spec:
1010
template:
1111
metadata:
1212
labels:
13-
app: collection-setup-{{ $.Release.Name }}
13+
app: load-collections-{{ $.Release.Name }}
1414
spec:
1515
restartPolicy: Never
1616
containers:
17-
- name: collectionsetup
18-
image: {{ .Values.collectionSetup.image.name }}:{{ .Values.collectionSetup.image.tag }}
19-
command: {{- toYaml .Values.collectionSetup.command | nindent 12 }}
17+
- name: load-collections
18+
image: {{ .Values.common.image.name }}:{{ .Values.common.image.tag }}
19+
command: {{- toYaml .Values.load_collections.command | nindent 12 }}
2020
args:
21-
{{- toYaml .Values.collectionSetup.args | nindent 12 }}
21+
{{- toYaml .Values.load_collections.args | nindent 12 }}
2222
ports:
2323
- containerPort: 5432
2424
resources:
25-
{{- toYaml .Values.collectionSetup.settings.resources | nindent 12 }}
25+
{{- toYaml .Values.common.resources | nindent 12 }}
2626
volumeMounts:
2727
- mountPath: /opt/initdb/sql-data
2828
name: initdb-sql-volume-{{ $.Release.Name }}
@@ -34,7 +34,7 @@ spec:
3434
- name: PGADMIN_URI
3535
valueFrom:
3636
secretKeyRef:
37-
name: {{ .Values.collectionSetup.settings.eoapiReleaseName }}-pguser-postgres
37+
name: {{ .Values.common.eoapiReleaseName }}-pguser-postgres
3838
key: uri
3939
volumes:
4040
- name: initdb-sql-volume-{{ $.Release.Name }}
@@ -46,13 +46,81 @@ spec:
4646
- name: initdb-sh-volume-{{ $.Release.Name }}
4747
configMap:
4848
name: initdb-sh-config-{{ $.Release.Name }}
49-
{{- with .Values.collectionSetup.settings.affinity }}
49+
{{- with .Values.common.affinity }}
5050
affinity:
5151
{{- toYaml . | nindent 8 }}
5252
{{- end }}
53-
{{- with .Values.collectionSetup.settings.tolerations }}
53+
{{- with .Values.common.tolerations }}
5454
tolerations:
5555
{{- toYaml . | nindent 8 }}
5656
{{- end }}
5757
backoffLimit: 1
58-
{{- end }}
58+
{{- end }}
59+
{{- if .Values.update_collection_extents.enabled }}
60+
---
61+
apiVersion: batch/v1
62+
kind: CronJob
63+
metadata:
64+
name: update-collection-extents-{{ $.Release.Name }}
65+
labels:
66+
app: update-collection-extents-{{ $.Release.Name }}
67+
spec:
68+
# 30 minutes past the hour, every 12 hours
69+
schedule: "30 */12 * * *"
70+
jobTemplate:
71+
spec:
72+
template:
73+
spec:
74+
containers:
75+
- name: update-collection-extents
76+
image: {{ .Values.common.image.name }}:{{ .Values.common.image.tag }}
77+
imagePullPolicy: IfNotPresent
78+
command: {{- toYaml .Values.update_collection_extents.command | nindent 16 }}
79+
args:
80+
{{- toYaml .Values.update_collection_extents.args | nindent 16 }}
81+
ports:
82+
- containerPort: 5432
83+
resources:
84+
{{- toYaml .Values.common.resources | nindent 16 }}
85+
env:
86+
- name: PGADMIN_URI
87+
valueFrom:
88+
secretKeyRef:
89+
name: {{ .Values.common.eoapiReleaseName }}-pguser-postgres
90+
key: uri
91+
restartPolicy: OnFailure
92+
{{- end }}
93+
{{- if .Values.run_queued_queries.enabled }}
94+
---
95+
apiVersion: batch/v1
96+
kind: CronJob
97+
metadata:
98+
name: run-queued-queries-{{ $.Release.Name }}
99+
labels:
100+
app: run-queued-queries-{{ $.Release.Name }}
101+
spec:
102+
# every hour
103+
schedule: "0 * * * *"
104+
jobTemplate:
105+
spec:
106+
template:
107+
spec:
108+
containers:
109+
- name: run-queued-queries
110+
image: {{ .Values.common.image.name }}:{{ .Values.common.image.tag }}
111+
imagePullPolicy: IfNotPresent
112+
command: {{- toYaml .Values.run_queued_queries.command | nindent 16 }}
113+
args:
114+
{{- toYaml .Values.run_queued_queries.args | nindent 16 }}
115+
ports:
116+
- containerPort: 5432
117+
resources:
118+
{{- toYaml .Values.common.resources | nindent 16 }}
119+
env:
120+
- name: PGADMIN_URI
121+
valueFrom:
122+
secretKeyRef:
123+
name: {{ .Values.common.eoapiReleaseName }}-pguser-postgres
124+
key: uri
125+
restartPolicy: OnFailure
126+
{{- end }}

‎helm-chart/eoapi-data-management/values.yaml‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,36 @@
1-
collectionSetup:
2-
enabled: true
1+
2+
common:
33
image:
44
name: ghcr.io/stac-utils/pgstac
55
tag: v0.9.2
6+
eoapiReleaseName: eoapi
7+
resources:
8+
requests:
9+
cpu: "512m"
10+
memory: "1024Mi"
11+
limits:
12+
cpu: "512m"
13+
memory: "1024Mi"
14+
15+
load_collections:
16+
enabled: true
17+
command:
18+
- "sh"
19+
args:
20+
- "/opt/initdb/run-load-collections.sh"
21+
22+
update_collection_extents:
23+
enabled: true
24+
command:
25+
- "sh"
26+
args:
27+
- "-c"
28+
- "psql $PGADMIN_URI -c 'SET search_path = pgstac,public; SELECT update_collection_extents();'"
29+
30+
run_queued_queries:
31+
enabled: true
632
command:
733
- "sh"
834
args:
9-
- "/opt/initdb/run-collection-setup.sh"
10-
settings:
11-
eoapiReleaseName: eoapi
12-
resources:
13-
requests:
14-
cpu: "512m"
15-
memory: "1024Mi"
16-
limits:
17-
cpu: "512m"
18-
memory: "1024Mi"
35+
- "-c"
36+
- "psql $PGADMIN_URI -c 'SET search_path = pgstac,public; CALL run_queued_queries();'"

0 commit comments

Comments
 (0)