feat: variable postgresql connection pools (#265)

* feat: make workflow db pool configurable

* simplify

* bump semver

Author: marcinkubica

charts/retool/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: retool
 description: A Helm chart for Kubernetes
 type: application
-version: 6.8.9
+version: 6.9.0
 maintainers:
   - name: Retool Engineering
     email: [email protected]

charts/retool/templates/_workers.tpl

@@ -29,6 +29,13 @@
   {{- $workerValues = $parentValues.evalWorker -}}
 {{- end -}}
 
+{{- $workerPoolMaxSize := 100 -}}
+{{- if $workerValues }}
+  {{- if $workerValues.postgresPoolMaxSize }}
+    {{- $workerPoolMaxSize = $workerValues.postgresPoolMaxSize }}
+  {{- end }}
+{{- end -}}
+
 {{- $healthcheckPort := ternary 3012 3005 (eq $workerType "agentEval") -}}
 {{- $serviceType := ternary "AGENT_EVAL_TEMPORAL_WORKER" "WORKFLOW_TEMPORAL_WORKER" (eq $workerType "agentEval") -}}
 {{- $taskqueue := ternary "agent-eval" (ternary "agent" "" (eq $workerType "agent")) (eq $workerType "agentEval") -}}
@@ -119,7 +126,7 @@ spec:
             value: {{ $taskqueue }}
           {{- end }}
           - name: DBCONNECTOR_POSTGRES_POOL_MAX_SIZE
-            value: "100"
+            value: {{ $workerPoolMaxSize | quote }}
           {{- if $.Values.dbconnector.enabled }}
           - name: DB_CONNECTOR_HOST
             value: http://{{ template "retool.fullname" $ }}-dbconnector

charts/retool/templates/deployment_workflows.yaml

@@ -62,6 +62,14 @@ spec:
         {{- if .Values.commandline.args }}
 {{ toYaml .Values.commandline.args | indent 10 }}
         {{- end }}
+        {{- $workflowBackendPoolMaxSize := 100 -}}
+        {{- with .Values.workflows }}
+          {{- with .backend }}
+            {{- if .postgresPoolMaxSize }}
+              {{- $workflowBackendPoolMaxSize = .postgresPoolMaxSize }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
         env:
           - name: DEPLOYMENT_TEMPLATE_TYPE
             value: {{ template "retool.deploymentTemplateType" . }}
@@ -107,7 +115,7 @@ spec:
             value: "true"
           {{ end }}
           - name: DBCONNECTOR_POSTGRES_POOL_MAX_SIZE
-            value: "100"
+            value: {{ $workflowBackendPoolMaxSize | quote }}
           - name: DBCONNECTOR_QUERY_TIMEOUT_MS
           {{- if .Values.workflows.dbConnectorTimeout }}
             value: {{ .Values.workflows.dbConnectorTimeout | quote}}

charts/retool/values.yaml

@@ -379,12 +379,16 @@ workflows:
     # Scaling this number will increase the number of workflow workers, e.g. a replicaCount of 4
     # will launch 9 pods -- 1 workflow backend, 4 workflow workers, and 4 for temporal cluster
     replicaCount: 1
+    # Override the connection pool size used by workflow workers when calling the dbconnector service
+    postgresPoolMaxSize: 100
 
   backend:
     # A replicaCount of 1 will launch 6 pods -- 1 workflow backend, 1 workflow worker, and 4 pods that make up the executor temporal cluster
     # Scaling this number will increase the number of workflow backends, e.g. a replicaCount of 4
     # will launch 9 pods -- 4 workflow backend, 1 workflow workers, and 4 for temporal cluster
     replicaCount: 1
+    # Override the connection pool size used by workflow backends when calling the dbconnector service
+    postgresPoolMaxSize: 100
 
     # If necessary, specify the resources to provision the workflows-backend pod separately from the main backend pods.
     # resources:

values.yaml

@@ -379,12 +379,16 @@ workflows:
     # Scaling this number will increase the number of workflow workers, e.g. a replicaCount of 4
     # will launch 9 pods -- 1 workflow backend, 4 workflow workers, and 4 for temporal cluster
     replicaCount: 1
+    # Override the connection pool size used by workflow workers when calling the dbconnector service
+    postgresPoolMaxSize: 100
 
   backend:
     # A replicaCount of 1 will launch 6 pods -- 1 workflow backend, 1 workflow worker, and 4 pods that make up the executor temporal cluster
     # Scaling this number will increase the number of workflow backends, e.g. a replicaCount of 4
     # will launch 9 pods -- 4 workflow backend, 1 workflow workers, and 4 for temporal cluster
     replicaCount: 1
+    # Override the connection pool size used by workflow backends when calling the dbconnector service
+    postgresPoolMaxSize: 100
 
     # If necessary, specify the resources to provision the workflows-backend pod separately from the main backend pods.
     # resources: