feat: Add secret support for default connections and sources (#103)

Fixes: HDX-2206

Fixes #93

Author: teeohhem

.changeset/curly-onions-promise.md

@@ -0,0 +1,5 @@
+---
+"helm-charts": patch
+---
+
+feat: Add secret support for default connections and sources

README.md

@@ -69,7 +69,9 @@ helm install my-hyperdx hyperdx/hdx-oss-v2
 
 ### External ClickHouse
 
-If you have an existing ClickHouse cluster:
+If you have an existing ClickHouse cluster, you have two options for configuring connections:
+
+#### Option 1: Inline Configuration (Simple)
 
 ```yaml
 # values-external-clickhouse.yaml
@@ -93,6 +95,93 @@ hyperdx:
     ]
 ```
 
+#### Option 2: External Secret (Recommended for Production)
+
+For production deployments where you want to keep credentials separate from your Helm configuration:
+
+```yaml
+# values-external-clickhouse-secret.yaml
+clickhouse:
+  enabled: false  # Disable the built-in ClickHouse
+
+otel:
+  clickhouseEndpoint: "tcp://your-clickhouse-server:9000"
+  clickhousePrometheusEndpoint: "http://your-clickhouse-server:9363"  # Optional
+
+hyperdx:
+  # Use an existing secret for complete configuration (connections + sources)
+  useExistingConfigSecret: true
+  existingConfigSecret: "hyperdx-external-config"
+  existingConfigConnectionsKey: "connections.json"
+  existingConfigSourcesKey: "sources.json"
+```
+
+Create your configuration secret:
+
+```bash
+# Create the connections JSON
+cat <<EOF > connections.json
+[
+  {
+    "name": "Production ClickHouse",
+    "host": "https://your-production-clickhouse.com:8123",
+    "port": 8123,
+    "username": "hyperdx_user",
+    "password": "your-secure-password"
+  }
+]
+EOF
+
+# Create the sources JSON
+cat <<EOF > sources.json
+[
+  {
+    "from": {
+      "databaseName": "default",
+      "tableName": "otel_logs"
+    },
+    "kind": "log",
+    "name": "Logs",
+    "connection": "Production ClickHouse",
+    "timestampValueExpression": "TimestampTime",
+    "displayedTimestampValueExpression": "Timestamp",
+    "implicitColumnExpression": "Body",
+    "serviceNameExpression": "ServiceName",
+    "bodyExpression": "Body",
+    "eventAttributesExpression": "LogAttributes",
+    "resourceAttributesExpression": "ResourceAttributes",
+    "severityTextExpression": "SeverityText",
+    "traceIdExpression": "TraceId",
+    "spanIdExpression": "SpanId"
+  },
+  {
+    "from": {
+      "databaseName": "default",
+      "tableName": "otel_traces"
+    },
+    "kind": "trace",
+    "name": "Traces",
+    "connection": "Production ClickHouse",
+    "timestampValueExpression": "Timestamp",
+    "displayedTimestampValueExpression": "Timestamp",
+    "implicitColumnExpression": "SpanName",
+    "serviceNameExpression": "ServiceName",
+    "traceIdExpression": "TraceId",
+    "spanIdExpression": "SpanId",
+    "durationExpression": "Duration"
+  }
+]
+EOF
+
+# Create the Kubernetes secret
+kubectl create secret generic hyperdx-external-config \
+  --from-file=connections.json=connections.json \
+  --from-file=sources.json=sources.json
+
+# Clean up the local files
+rm connections.json sources.json
+```
+
 ### External OTEL Collector
 
 If you have an existing OTEL collector setup:
@@ -125,6 +214,7 @@ otel:
 
 hyperdx:
   otelExporterEndpoint: "http://your-otel-collector:4318"
+  # Option 1: Inline configuration (for testing/development)
   defaultConnections: |
     [
       {
@@ -135,6 +225,12 @@ hyperdx:
         "password": "your-password"
       }
     ]
+  
+  # Option 2: External secret (recommended for production)
+  # useExistingConfigSecret: true
+  # existingConfigSecret: "my-external-config"
+  # existingConfigConnectionsKey: "connections.json"
+  # existingConfigSourcesKey: "sources.json"
 ```
 
 ## Configuration

charts/hdx-oss-v2/templates/hyperdx-deployment.yaml

@@ -97,6 +97,20 @@ spec:
                 secretKeyRef:
                   name: {{ include "hdx-oss.fullname" . }}-app-secrets
                   key: api-key
+            {{- if .Values.hyperdx.useExistingConfigSecret }}
+            - name: DEFAULT_CONNECTIONS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.hyperdx.existingConfigSecret | quote }}
+                  key: {{ .Values.hyperdx.existingConfigConnectionsKey | quote }}
+                  optional: false
+            - name: DEFAULT_SOURCES
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.hyperdx.existingConfigSecret | quote }}
+                  key: {{ .Values.hyperdx.existingConfigSourcesKey | quote }}
+                  optional: false
+            {{- else }}
             {{- if .Values.hyperdx.defaultConnections }}
             - name: DEFAULT_CONNECTIONS
               value: {{ tpl .Values.hyperdx.defaultConnections . | quote }}
@@ -105,6 +119,7 @@ spec:
             - name: DEFAULT_SOURCES
               value: {{ tpl .Values.hyperdx.defaultSources . | quote }}
             {{- end }}
+            {{- end }}
             {{- with .Values.hyperdx.env }}
             {{- toYaml . | nindent 12 }}
             {{- end }}