feat: revamp public helm charts (#2581)

* feat: add opentaco-platform-reference chart

* update helm release and test workflows

* handle statesman object storage secret in platform-reference

* handle separate public vs internal urls, add taco-sidecar

* standardize image conditionals for token-service to match other charts

* fix helm unittest, enable workload identity in taco-orchestrator

* fix broken unit tests

* remove mysql mssql and sqlite from token service to simplify / standardize

* set publicURL, signing secret, and enable x forwarding for statesman

* match cloudsql credential volume to internal chart

* update docker compose with new env vars

* add cronjob for drift execution and notification triggers

* bump chart versions

* use oci reference for taco-sidecar chart

Author: sidpalas

.github/workflows/helm-release.yml

@@ -5,8 +5,8 @@ on:
     branches:
       - develop
     paths:
-      - 'helm-charts/**'
-  workflow_dispatch:  # Allow manual triggering
+      - "helm-charts/**"
+  workflow_dispatch: # Allow manual triggering
 
 permissions:
   contents: read
@@ -24,7 +24,9 @@ jobs:
           - taco-statesman
           - taco-token-service
           - taco-drift
-          - taco-ui 
+          - taco-ui
+          - taco-sidecar
+          - opentaco-platform-reference
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -41,7 +43,7 @@ jobs:
           cd helm-charts/${{ matrix.chart }}
           helm package .
           helm push ${{ matrix.chart }}-*.tgz oci://ghcr.io/diggerhq/helm-charts
-  
+
   # Then release umbrella chart after dependencies are available
   release-umbrella:
     needs: release-charts
@@ -66,4 +68,4 @@ jobs:
         run: |
           cd helm-charts/opentaco
           helm package .
-          helm push opentaco-*.tgz oci://ghcr.io/diggerhq/helm-charts
+          helm push opentaco-*.tgz oci://ghcr.io/diggerhq/helm-charts

.github/workflows/helm-test.yml

@@ -9,6 +9,19 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        chart:
+          - digger-backend
+          - taco-orchestrator
+          - taco-statesman
+          - taco-token-service
+          - taco-drift
+          - taco-ui
+          - taco-sidecar
+          - opentaco-platform-reference
+          - opentaco
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -18,12 +31,16 @@ jobs:
 
       - name: Install helm-unittest
         run: |
-          helm plugin install https://github.com/helm-unittest/helm-unittest.git
+          helm plugin install https://github.com/helm-unittest/helm-unittest.git --verify=false
 
       - name: Lint chart
         run: |
-          helm lint helm-charts/digger-backend
+          helm lint helm-charts/${{ matrix.chart }}
 
-      - name: Run unit tests
+      - name: Run unit tests (if present)
         run: |
-          helm unittest helm-charts/digger-backend
+          if [ -d "helm-charts/${{ matrix.chart }}/tests" ]; then
+            helm unittest helm-charts/${{ matrix.chart }}
+          else
+            echo "No helm-unittest tests found for ${{ matrix.chart }}, skipping"
+          fi

backend/controllers/cache.go

@@ -110,7 +110,10 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 }
 
 func sendProcessCacheRequest(repoFullName string, branch string, installationId int64) error {
-	diggerHostname := os.Getenv("HOSTNAME")
+	diggerHostname := utils.GetInternalBaseURL()
+	if diggerHostname == "" {
+		return fmt.Errorf("INTERNAL_BASE_URL (or legacy HOSTNAME) is not set")
+	}
 	webhookSecret := os.Getenv("DIGGER_INTERNAL_SECRET")
 
 	installationLink, err := models.DB.GetGithubInstallationLinkForInstallationId(installationId)

backend/controllers/github_setup.go

@@ -36,25 +36,11 @@ func GithubAppSetup(c *gin.Context) {
 		Webhook               *githubWebhook    `json:"hook_attributes"`
 	}
 
-	host := os.Getenv("HOSTNAME")
-	// When the backend is deployed behind a reverse proxy (or behind the UI proxy),
-	// the inbound request Host/TLS reflects the internal hop, not the public origin.
-	// The GitHub App manifest flow requires public callback/webhook URLs, so we
-	// prefer X-Forwarded-Host/Proto when present to construct externally reachable
-	// URLs.
-	forwardedHost := c.Request.Header.Get("X-Forwarded-Host")
-	forwardedProto := c.Request.Header.Get("X-Forwarded-Proto")
-	if forwardedHost != "" {
-		if forwardedProto == "" {
-			forwardedProto = "https"
-		}
-		host = fmt.Sprintf("%s://%s", forwardedProto, forwardedHost)
-	} else if host == "" {
-		scheme := "http"
-		if c.Request.TLS != nil {
-			scheme = "https"
-		}
-		host = fmt.Sprintf("%s://%s", scheme, c.Request.Host)
+	host := utils.GetPublicBaseURL()
+	if host == "" {
+		slog.Error("PUBLIC_BASE_URL and HOSTNAME are not set")
+		c.String(http.StatusInternalServerError, "PUBLIC_BASE_URL (or legacy HOSTNAME) must be set to the public URL (for example: https://app.example.com)")
+		return
 	}
 	publicPrefix := utils.NormalizePublicPathPrefix(os.Getenv("DIGGER_PUBLIC_PATH_PREFIX"))
 	manifest := &githubAppRequest{

backend/utils/base_urls.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func GetPublicBaseURL() string {
+	return getBaseURL("PUBLIC_BASE_URL", "HOSTNAME", "https")
+}
+
+func GetInternalBaseURL() string {
+	return getBaseURL("INTERNAL_BASE_URL", "HOSTNAME", "http")
+}
+
+func getBaseURL(primaryEnv string, fallbackEnv string, defaultScheme string) string {
+	// Historically this codebase used HOSTNAME for both public URL generation
+	// (for example GitHub App callback/webhook manifest URLs) and internal
+	// service-to-self calls. We now prefer explicit PUBLIC_BASE_URL and
+	// INTERNAL_BASE_URL, but keep HOSTNAME as a compatibility fallback so older
+	// deployments and existing Helm secrets continue to work during migration.
+	raw := strings.TrimSpace(os.Getenv(primaryEnv))
+	if raw == "" {
+		raw = strings.TrimSpace(os.Getenv(fallbackEnv))
+	}
+	if raw == "" {
+		return ""
+	}
+
+	raw = strings.TrimRight(raw, "/")
+	if strings.Contains(raw, "://") {
+		return raw
+	}
+
+	return fmt.Sprintf("%s://%s", defaultScheme, raw)
+}

backend/utils/graphs.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	"os"
 
 	"github.com/diggerhq/digger/backend/models"
 	configuration "github.com/diggerhq/digger/libs/digger_config"
@@ -41,7 +40,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, jobReporterType st
 	}
 	organisationName := organisation.Name
 
-	backendHostName := os.Getenv("HOSTNAME")
+	backendHostName := GetPublicBaseURL()
 
 	slog.Debug("Processing jobs", "count", len(jobsMap))
 	marshalledJobsMap := map[string][]byte{}

docker-compose.yml

@@ -78,7 +78,8 @@ services:
       # - GITHUB_APP_CLIENT_SECRET=
       # - GITHUB_APP_PRIVATE_KEY_BASE64=
       # - GITHUB_WEBHOOK_SECRET=
-      - HOSTNAME=public-url-including-scheme-CHANGE_ME
+      - PUBLIC_BASE_URL=public-url-including-scheme-CHANGE_ME
+      - INTERNAL_BASE_URL=http://orchestrator:3000
       - HTTP_BASIC_AUTH=true
       - HTTP_BASIC_AUTH_PASSWORD=basic-auth-password-CHANGE_ME
       - HTTP_BASIC_AUTH_USERNAME=admin
@@ -194,7 +195,6 @@ services:
     env_file:
       - backend/.env
     environment:
-      - BACKGROUND_JOBS_CLIENT_TYPE=local-exec
       - DATABASE_URL=postgres://postgres:postgres-password-CHANGE_ME@postgres-orchestrator:5432/orchestrator?sslmode=disable
       # Embedded in job specs so CI can report drift results back.
       - DIGGER_DRIFT_REPORTER_HOSTNAME=public-url-including-scheme-CHANGE_ME
@@ -203,7 +203,6 @@ services:
       # Used for drift service internal scheduling triggers.
       - DIGGER_HOSTNAME=http://drift:3000
       - DIGGER_INTERNAL_SECRET=orchestrator-secret-CHANGE_ME
-      - DIGGER_LOAD_PROJECTS_ON_PUSH=true
       - DIGGER_PUBLIC_PATH_PREFIX=/orchestrator
       # Auth for drift service internal endpoints (/ _internal/*).
       - DIGGER_WEBHOOK_SECRET=drift-secret-CHANGE_ME
@@ -214,11 +213,11 @@ services:
       # - GITHUB_APP_CLIENT_SECRET=
       # - GITHUB_APP_PRIVATE_KEY_BASE64=
       # - GITHUB_WEBHOOK_SECRET=
-      - HOSTNAME=https://hyperphysical-alyse-metagnathous.ngrok-free.dev
+      - PUBLIC_BASE_URL=public-url-including-scheme-CHANGE_ME
+      - INTERNAL_BASE_URL=http://drift:3000
       - HTTP_BASIC_AUTH=true
       - HTTP_BASIC_AUTH_PASSWORD=github-setup-basic-auth-password-CHANGE_ME
       - HTTP_BASIC_AUTH_USERNAME=admin
-      - PROJECTS_REFRESH_BIN=/app/projects_refesh_main
     ports:
       - "3001:3000"
     depends_on:

helm-charts/opentaco-platform-reference/Chart.yaml

@@ -0,0 +1,21 @@
+apiVersion: v2
+name: opentaco-platform-reference
+description: Reference-only platform add-ons for quick OpenTaco bootstrap
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+
+# This chart is intentionally a reference implementation to bootstrap a working
+# environment quickly. It is not a production platform blueprint.
+
+dependencies:
+  - name: traefik
+    version: "~34.4.0"
+    repository: https://traefik.github.io/charts
+    condition: traefik.enabled
+
+  - name: cloudnative-pg
+    alias: cnpg
+    version: "~0.22.0"
+    repository: https://cloudnative-pg.github.io/charts
+    condition: cnpg.enabled

helm-charts/opentaco-platform-reference/README.md

@@ -0,0 +1,31 @@
+# OpenTaco Platform Reference Chart
+
+This chart is a reference implementation to get to a working OpenTaco setup quickly.
+
+It is not intended as a production blueprint. Teams should use their own platform approach for ingress, database lifecycle/operations, and object storage.
+
+It installs:
+- Traefik ingress controller
+- CloudNativePG operator
+- Shared CNPG cluster and application DB credentials
+- MinIO (StatefulSet) for statesman object storage
+- Bucket init job (creates `opentaco` bucket by default)
+- Statesman object storage secret (`statesman-object-storage` by default)
+
+CNPG note:
+- CNPG can auto-generate the bootstrap app secret (`<cluster>-app`) when no bootstrap secret is provided.
+- This chart creates explicit per-service app secrets so the `opentaco` subcharts can reference stable, service-specific credentials.
+- Secrets include structured postgres keys (`host`, `port`, `database`, `username`, `password`, `sslmode`) and are intended to be consumed via each service chart's `database.existingSecret` + `database.secretKeys` settings.
+
+MinIO defaults:
+- Service: `minio.opentaco.svc.cluster.local:9000`
+- Console: `minio.opentaco.svc.cluster.local:9001`
+- Bucket: `opentaco`
+
+For statesman S3 backend, configure OpenTaco with:
+- `OPENTACO_STORAGE=s3`
+- `taco-statesman.taco.storage.s3.secretRef.name=statesman-object-storage`
+
+Install `opentaco` separately after this chart. This chart now owns the CNPG `Cluster` resource and app database credentials.
+
+Use this chart for demos and rapid validation. For production, consume the `opentaco` chart directly and manage ingress, database management, and object storage with your own standards and tooling.

helm-charts/opentaco-platform-reference/templates/_helpers.tpl

@@ -0,0 +1,35 @@
+{{- define "opentaco-platform-reference.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "opentaco-platform-reference.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "opentaco-platform-reference.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "opentaco-platform-reference.labels" -}}
+helm.sh/chart: {{ include "opentaco-platform-reference.chart" . }}
+app.kubernetes.io/name: {{ include "opentaco-platform-reference.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{- define "opentaco-platform-reference.minio.fullname" -}}
+{{- if .Values.minio.fullnameOverride }}
+{{- .Values.minio.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-minio" .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}