Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/retool/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ apiVersion: v2
name: retool
description: A Helm chart for Kubernetes
type: application
version: 6.3.6
version: 6.4.0
maintainers:
- name: Retool Engineering
email: engineering+helm@retool.com
Expand Down
2 changes: 2 additions & 0 deletions charts/retool/ci/standalone-dbconnector-enabled-option.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dbconnector:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

glad to see this feature of testing with random extra charts was useful lol

enabled: true
24 changes: 24 additions & 0 deletions charts/retool/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,23 @@ app.kubernetes.io/name: {{ include "retool.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Selector labels for standalone dbconnector. Note changes here will require manual
deployment recreation and incur downtime, so should be avoided.
*/}}
{{- define "retool.dbconnector.selectorLabels" -}}
retoolService: {{ include "retool.dbconnector.name" . }}
{{- end }}

{{/*
Extra (non-selector) labels for standalone dbconnector.
*/}}
{{- define "retool.dbconnector.labels" -}}
app.kubernetes.io/name: {{ include "retool.dbconnector.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
telemetry.retool.com/service-name: dbconnector
{{- end }}

{{/*
Selector labels for workflow backend. Note changes here will require manual
deployment recreation and incur downtime, so should be avoided.
Expand Down Expand Up @@ -286,6 +303,13 @@ Set Temporal namespace
{{- end -}}
{{- end -}}

{{/*
Set dbconnector service name
*/}}
{{- define "retool.dbconnector.name" -}}
{{ template "retool.fullname" . }}-dbconnector
{{- end -}}

{{/*
Set workflow backend service name
*/}}
Expand Down
15 changes: 14 additions & 1 deletion charts/retool/templates/deployment_backend.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,21 @@ spec:
value: {{ template "retool.deploymentTemplateVersion" . }}
- name: NODE_ENV
value: production
{{- $serviceType := list "MAIN_BACKEND" "DB_CONNECTOR" "DB_SSH_CONNECTOR" }}
{{- $serviceType := list "MAIN_BACKEND" }}
{{- /*
JAVA_DBCONNECTOR in the service type only applies before the version of Retool that changes it to opt-out (3.93.0-edge),
and only if the Java dbconnector is enabled in values.yaml.
*/}}
{{- if and ( not ( include "retool_version_with_java_dbconnector_opt_out" . ) ) ( $.Values.dbconnector.java.enabled ) }}
{{- $serviceType = append $serviceType "JAVA_DBCONNECTOR" }}
{{- end }}
{{- /*
The dbconnector.enabled setting is to enable a standalone dbconnector deployment.
If it is not enabled (default), the main backend acts as its own dbconnector.
*/ -}}
{{- if not $.Values.dbconnector.enabled -}}
{{- $serviceType = concat $serviceType (list "DB_CONNECTOR" "DB_SSH_CONNECTOR") -}}
{{- end }}
{{- /*
It may seem counterintuitive to add the JOBS_RUNNER service type only without a jobs runner.
The reason for this is that the backend needs to act as a jobs runner, if the jobs runner is not enabled.
Expand Down Expand Up @@ -112,6 +119,12 @@ spec:
value: {{ template "retool.postgresql.user" . }}
- name: POSTGRES_SSL_ENABLED
value: {{ template "retool.postgresql.ssl_enabled" . }}
{{- if .Values.dbconnector.enabled }}
- name: DB_CONNECTOR_HOST
value: http://{{ template "retool.fullname" . }}-dbconnector
- name: DB_CONNECTOR_PORT
value: {{ .Values.dbconnector.port | quote }}
{{- end }}
{{- if .Values.config.dbConnectorTimeout }}
- name: DBCONNECTOR_QUERY_TIMEOUT_MS
value: {{ .Values.config.dbConnectorTimeout | quote }}
Expand Down
270 changes: 270 additions & 0 deletions charts/retool/templates/deployment_dbconnector.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
{{- if .Values.dbconnector.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "retool.dbconnector.name" . }}
labels:
{{- include "retool.dbconnector.selectorLabels" . | nindent 4 }}
{{- include "retool.dbconnector.labels" . | nindent 4 }}
{{- include "retool.labels" . | nindent 4 }}
{{- if .Values.deployment.annotations }}
annotations:
{{- toYaml .Values.deployment.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.dbconnector.replicas }}
selector:
matchLabels:
{{- include "retool.dbconnector.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.podAnnotations }}
{{ toYaml .Values.podAnnotations | indent 8 }}
{{- end }}
{{- if .Values.backend.annotations }}
{{ toYaml .Values.backend.annotations | indent 8 }}
{{- end }}
labels:
{{- include "retool.dbconnector.selectorLabels" . | nindent 8 }}
{{- include "retool.dbconnector.labels" . | nindent 8 }}
{{- include "retool.labels" . | nindent 8 }}
spec:
{{- if .Values.dbconnector.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.dbconnector.terminationGracePeriodSeconds }}
{{- end }}
containers:
- args:
- bash
- -c
- chmod -R +x ./docker_scripts; sync; ./docker_scripts/start_api.sh
image: "{{ .Values.image.repository }}:{{ required "Please set a value for .Values.image.tag" .Values.image.tag }}"
name: {{ include "retool.dbconnector.name" . }}
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path }}
port: {{ .Values.dbconnector.port }}
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
successThreshold: {{ .Values.readinessProbe.successThreshold }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path }}
port: {{ .Values.dbconnector.port }}
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
resources:
{{- if .Values.dbconnector.resources }}
{{- toYaml .Values.dbconnector.resources | nindent 10 }}
{{- else }}
{{- toYaml .Values.resources | nindent 10 }}
{{- end }}
env:
{{- if $.Values.dbconnector.java.enabled }}
- name: SERVICE_TYPE
value: DB_CONNECTOR,DB_SSH_CONNECTOR,JAVA_DBCONNECTOR
{{- else }}
- name: SERVICE_TYPE
value: DB_CONNECTOR,DB_SSH_CONNECTOR
{{- end }}
- name: DEPLOYMENT_TEMPLATE_TYPE
value: {{ template "retool.deploymentTemplateType" . }}
- name: DEPLOYMENT_TEMPLATE_VERSION
value: {{ template "retool.deploymentTemplateVersion" . }}
- name: NODE_ENV
value: production
- name: NODE_OPTIONS
value: {{ .Values.dbconnector.config.nodeOptions }}
- name: DBCONNECTOR_HTTP_AGENT_MAX_SOCKETS
value: {{ .Values.dbconnector.config.httpAgentMaxSockets | quote }}
- name: DBCONNECTOR_POSTGRES_POOL_MAX_SIZE
value: {{ .Values.dbconnector.config.postgresPoolMaxSize | quote }}
- name: DISABLE_DATABASE_MIGRATIONS
value: "true"
- name: WORKFLOW_BACKEND_HOST
value: http://{{ template "retool.fullname" . }}-workflow-backend
- name: CLIENT_ID
value: {{ default "" .Values.config.auth.google.clientId }}
- name: COOKIE_INSECURE
value: {{ .Values.config.useInsecureCookies | quote }}
{{- if and (not .Values.externalSecrets.enabled) (not .Values.externalSecrets.externalSecretsOperator.enabled) }}
- name: LICENSE_KEY
valueFrom:
secretKeyRef:
{{- if .Values.config.licenseKeySecretName }}
name: {{ .Values.config.licenseKeySecretName }}
key: {{ .Values.config.licenseKeySecretKey | default "license-key" }}
{{- else }}
name: {{ template "retool.fullname" . }}
key: license-key
{{- end }}
- name: JWT_SECRET
valueFrom:
secretKeyRef:
{{- if .Values.config.jwtSecretSecretName }}
name: {{ .Values.config.jwtSecretSecretName }}
key: {{ .Values.config.jwtSecretSecretKey | default "jwt-secret" }}
{{- else }}
name: {{ template "retool.fullname" . }}
key: jwt-secret
{{- end }}
- name: ENCRYPTION_KEY
valueFrom:
secretKeyRef:
{{- if .Values.config.encryptionKeySecretName }}
name: {{ .Values.config.encryptionKeySecretName }}
key: {{ .Values.config.encryptionKeySecretKey | default "encryption-key" }}
{{- else }}
name: {{ template "retool.fullname" . }}
key: encryption-key
{{- end }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
name: {{ template "retool.postgresql.fullname" . }}
# `postgres` is the default admin username for postgres in the subchart we use, so it needs the admin password
# if a different username is picked, then it needs the custom password instead.
{{- if eq .Values.postgresql.auth.username "postgres" }}
key: postgres-password
{{- else }}
key: password
{{- end }}
{{- else }}
{{- if .Values.config.postgresql.passwordSecretName }}
name: {{ .Values.config.postgresql.passwordSecretName }}
key: {{ .Values.config.postgresql.passwordSecretKey | default "postgresql-password" }}
{{- else }}
name: {{ template "retool.fullname" . }}
key: postgresql-password
{{- end }}
{{- end }}
- name: CLIENT_SECRET
valueFrom:
secretKeyRef:
{{- if .Values.config.auth.google.clientSecretSecretName }}
name: {{ .Values.config.auth.google.clientSecretSecretName }}
key: {{ .Values.config.auth.google.clientSecretSecretKey | default "google-client-secret" }}
{{- else }}
name: {{ template "retool.fullname" . }}
key: google-client-secret
{{- end }}
{{- end }}
{{- range $key, $value := .Values.env }}
- name: "{{ $key }}"
value: "{{ $value }}"
{{- end }}
{{- range .Values.environmentSecrets }}
- name: {{ .name }}
valueFrom:
secretKeyRef:
name: {{ .secretKeyRef.name }}
key: {{ .secretKeyRef.key }}
{{- end }}
{{- with .Values.environmentVariables }}
{{ toYaml . | indent 10 }}
{{- end }}
{{- if .Values.externalSecrets.enabled }}
envFrom:
- secretRef:
name: {{ .Values.externalSecrets.name }}
{{- range .Values.externalSecrets.secrets }}
- secretRef:
name: {{ .name }}
{{- end }}
{{- end }}
{{- if .Values.externalSecrets.externalSecretsOperator.enabled }}
envFrom:
{{- range .Values.externalSecrets.externalSecretsOperator.secretRef }}
- secretRef:
name: {{ .name }}
{{- end }}
{{- end }}
volumeMounts:
{{- range $configFile := (keys .Values.files) }}
- name: {{ template "retool.name" $ }}
mountPath: "/usr/share/retool/config/{{ $configFile }}"
subPath: {{ $configFile }}
{{- end }}
{{- if and .Values.persistentVolumeClaim.enabled .Values.persistentVolumeClaim.mountPath }}
- name: retool-pv
mountPath: {{ .Values.persistentVolumeClaim.mountPath }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{- toYaml .Values.extraVolumeMounts | nindent 8 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- tpl . $ | indent 8 }}
{{- end }}
{{- range .Values.extraConfigMapMounts }}
- name: {{ .name }}
mountPath: {{ .mountPath }}
subPath: {{ .subPath }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
{{- toYaml .Values.image.pullSecrets | nindent 8 }}
{{- end }}
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
{{- if .Values.nodeSelector }}
nodeSelector:
{{- toYaml .Values.nodeSelector | nindent 8 }}
{{- end }}
tolerations:
{{- toYaml .Values.tolerations | nindent 8 }}
{{- if .Values.securityContext.enabled }}
securityContext:
runAsUser: {{ .Values.securityContext.runAsUser }}
fsGroup: {{ .Values.securityContext.fsGroup }}
{{- end }}
volumes:
{{- range .Values.extraConfigMapMounts }}
- name: {{ .name }}
configMap:
name: {{ .configMap }}
{{- end }}
{{- if .Values.persistentVolumeClaim.enabled }}
- name: retool-pv
persistentVolumeClaim:
claimName: {{ default (include "retool.fullname" .) .Values.persistentVolumeClaim.existingClaim }}
{{- end }}
{{- if .Values.extraVolumes }}
{{- toYaml .Values.extraVolumes | nindent 8 }}
{{- end }}
---

{{- if .Values.podDisruptionBudget }}
{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version -}}
apiVersion: policy/v1
{{- else -}}
apiVersion: policy/v1beta1
{{- end }}
kind: PodDisruptionBudget
metadata:
name: {{ include "retool.dbconnector.name" . }}
spec:
{{ toYaml .Values.podDisruptionBudget }}
selector:
matchLabels:
{{- include "retool.dbconnector.selectorLabels" . | nindent 6 }}
---
{{- end }}

apiVersion: v1
kind: Service
metadata:
name: {{ include "retool.dbconnector.name" . }}
spec:
selector:
retoolService: {{ include "retool.dbconnector.name" . }}
ports:
- protocol: TCP
name: http-server
port: {{ .Values.dbconnector.port }}
targetPort: {{ .Values.dbconnector.port }}
{{- end }}
15 changes: 14 additions & 1 deletion charts/retool/templates/deployment_workflows.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,29 @@ spec:
value: {{ template "retool.deploymentTemplateVersion" . }}
- name: NODE_ENV
value: production
{{- $serviceType := list "WORKFLOW_BACKEND" "DB_CONNECTOR" "DB_SSH_CONNECTOR" }}
{{- $serviceType := list "WORKFLOW_BACKEND" }}
{{- /*
JAVA_DBCONNECTOR in the service type only applies before the version of Retool that changes it to opt-out (3.93.0-edge),
and only if the Java dbconnector is enabled in values.yaml.
*/}}
{{- if and ( not ( include "retool_version_with_java_dbconnector_opt_out" . ) ) ( $.Values.dbconnector.java.enabled ) }}
{{- $serviceType = append $serviceType "JAVA_DBCONNECTOR" }}
{{- end }}
{{- /*
The dbconnector.enabled setting is to enable a standalone dbconnector deployment.
If it is not enabled (default), the main backend acts as its own dbconnector.
*/ -}}
{{- if not $.Values.dbconnector.enabled -}}
{{- $serviceType = concat $serviceType (list "DB_CONNECTOR" "DB_SSH_CONNECTOR") -}}
{{- end }}
- name: SERVICE_TYPE
value: {{ join "," $serviceType }}
{{- if $.Values.dbconnector.enabled }}
- name: DB_CONNECTOR_HOST
value: http://{{ template "retool.fullname" . }}-dbconnector
Copy link
Contributor

@jjlgao jjlgao Jan 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this would point workflows-backend at the generic dbc deployment -- is it intentional that we aren't creating a workflows-dbconnector deployment here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's intentional, yes, but not unchangeable. I think splitting dbconnector out of main backend is likely the main scalability unlock for large single-tenant deployments, and we're only just now adding it to retool-helm. going even further and splitting out a workflows-dbconnector from the main dbconnector seems premature, given that workflows doesn't need its own dbconnector to function and we do that mainly for scale reasons in our multi-tenant cloud.

- name: DB_CONNECTOR_PORT
value: {{ .Values.dbconnector.port | quote }}
{{- end }}
{{ if and ( include "retool_version_with_java_dbconnector_opt_out" . ) ( not $.Values.dbconnector.java.enabled ) }}
- name: DISABLE_JAVA_DBCONNECTOR
value: "true"
Expand Down
Loading
Loading