feat(storage): add configuration for external object storage provider

charts/cryostat/templates/cryostat_deployment.yaml

@@ -91,27 +91,43 @@ spec:
           - name: QUARKUS_DATASOURCE_JDBC_URL
             value: jdbc:postgresql://{{ $fullName }}-db:5432/cryostat
           - name: STORAGE_BUCKETS_ARCHIVES_NAME
-            value: archivedrecordings
+            value: {{ .Values.storage.buckets.names.archivedRecordings }}
+          - name: STORAGE_BUCKETS_EVENT_TEMPLATES_NAME
+            value: {{ .Values.storage.buckets.names.eventTemplates }}
+          - name: STORAGE_BUCKETS_PROBE_TEMPLATES_NAME
+            value: {{ .Values.storage.buckets.names.jmcAgentProbeTemplates }}
+          - name: STORAGE_BUCKETS_METADATA_NAME
+            value: {{ .Values.storage.buckets.names.metadata }}
+          - name: CRYOSTAT_SERVICES_REPORTS_STORAGE_CACHE_NAME
+            value: {{ .Values.storage.buckets.names.archivedReports }}
+          - name: STORAGE_METADATA_STORAGE_MODE
+            value: {{ .Values.storage.provider.metadata.storageMode }}
+          {{- if (.Values.storage.provider.tls).trustAll }}
+          - name: QUARKUS_S3_SYNC_CLIENT_TLS_TRUST_MANAGERS_PROVIDER_TYPE
+            value: trust-all
+          {{- end }}
           - name: QUARKUS_S3_ENDPOINT_OVERRIDE
-            value: http://{{ $fullName }}-storage:8333
+            value: {{ default (printf "http://%s-storage:8333" $fullName) .Values.storage.provider.url }}
           - name: QUARKUS_S3_PATH_STYLE_ACCESS
-            value: "true"
+            value: "{{ .Values.storage.provider.usePathStyleAccess }}"
           - name: QUARKUS_S3_AWS_REGION
-            value: us-east-1
+            # if an external provider URL is supplied then a region must also be supplied.
+            # Otherwise we are deploying a managed storage instance and can set a default value
+            value: {{ ternary .Values.storage.provider.region "us-east-1" (not (empty .Values.storage.provider.url)) }}
           - name: QUARKUS_S3_AWS_CREDENTIALS_TYPE
-            value: static
-          - name: QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_ACCESS_KEY_ID
-            value: cryostat
+            value: {{ .Values.storage.provider.authentication.credentialsType }}
           - name: AWS_ACCESS_KEY_ID
-            value: $(QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_ACCESS_KEY_ID)
-          - name: QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY
             valueFrom:
               secretKeyRef:
                 name: {{ default (printf "%s-storage-secret" .Release.Name) .Values.storage.storageSecretName }}
-                key: STORAGE_ACCESS_KEY
+                key: STORAGE_ACCESS_KEY_ID
                 optional: false
           - name: AWS_SECRET_ACCESS_KEY
-            value: $(QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY)
+            valueFrom:
+              secretKeyRef:
+                name: {{ default (printf "%s-storage-secret" .Release.Name) .Values.storage.storageSecretName }}
+                key: STORAGE_ACCESS_KEY
+                optional: false
           - name: GRAFANA_DATASOURCE_URL
             value: http://localhost:8800
           - name: GRAFANA_DASHBOARD_URL

charts/cryostat/templates/networkpolicy_ingress.yaml

@@ -62,6 +62,7 @@ spec:
       ports:
         - protocol: TCP
           port: 5432
+{{- if (empty (.Values.storage.provider).url) }}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -92,3 +93,4 @@ spec:
         - protocol: TCP
           port: 8333
 {{- end }}
+{{- end }}

charts/cryostat/templates/storage_deployment.yaml

@@ -1,5 +1,6 @@
 {{- $fullName := include "cryostat.fullname" . -}}
 ---
+{{- if (empty (.Values.storage.provider).url) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -42,9 +43,13 @@ spec:
           imagePullPolicy: {{ (.Values.storage).image.pullPolicy }}
           env:
           - name: CRYOSTAT_BUCKETS
-            value: archivedrecordings,archivedreports,eventtemplates,probes
+            value: {{ join "," (.Values.storage.buckets.names | values | uniq | sortAlpha) }}
           - name: CRYOSTAT_ACCESS_KEY
-            value: cryostat
+            valueFrom:
+              secretKeyRef:
+                name: {{ default (printf "%s-storage-secret" .Release.Name) .Values.storage.storageSecretName }}
+                key: STORAGE_ACCESS_KEY_ID
+                optional: false
           - name: CRYOSTAT_SECRET_KEY
             valueFrom:
               secretKeyRef:
@@ -117,3 +122,4 @@ spec:
       - name: {{ .Chart.Name }}-storage
         emptyDir: {}
       {{- end }}
+{{- end }}

charts/cryostat/templates/storage_pvc.yaml

@@ -1,4 +1,4 @@
-{{- if ((.Values.storage.pvc).enabled) }}
+{{- if and (empty (.Values.storage.provider).url) ((.Values.storage.pvc).enabled) }}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:

charts/cryostat/templates/storage_secret.yaml

@@ -7,5 +7,6 @@ metadata:
     {{- include "cryostat.labels" $ | nindent 4 }}
 type: Opaque
 data:
+  STORAGE_ACCESS_KEY_ID: {{ b64enc "cryostat" }}
   STORAGE_ACCESS_KEY: {{ include "cryostat.objectStorageSecretKey" . }}
 {{- end -}}

charts/cryostat/templates/storage_service.yaml

@@ -1,5 +1,6 @@
 {{- $fullName := include "cryostat.fullname" . -}}
 ---
+{{- if (empty (.Values.storage.provider).url) }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -18,3 +19,4 @@ spec:
   selector:
     {{- include "cryostat.selectorLabels" $ | nindent 4 }}
     app.kubernetes.io/component: storage
+{{- end }}

charts/cryostat/tests/cryostat_deployment_test.yaml

@@ -103,33 +103,44 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_ARCHIVES_NAME')].value
           value: "archivedrecordings"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_EVENT_TEMPLATES_NAME')].value
+          value: "eventtemplates"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_PROBE_TEMPLATES_NAME')].value
+          value: "probes"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_METADATA_NAME')].value
+          value: "metadata"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_METADATA_STORAGE_MODE')].value
+          value: "tagging"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='CRYOSTAT_SERVICES_REPORTS_STORAGE_CACHE_NAME')].value
+          value: "archivedreports"
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_ENDPOINT_OVERRIDE')].value
           value: "http://RELEASE-NAME-cryostat-storage:8333"
+      - notExists:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_SYNC_CLIENT_TLS_TRUST_MANAGERS_PROVIDER_TYPE')]
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_PATH_STYLE_ACCESS')].value
           value: "true"
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_AWS_REGION')].value
           value: "us-east-1"
       - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_AWS_CREDENTIALS_TYPE')].value
-          value: "static"
-      - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_ACCESS_KEY_ID')].value
-          value: "cryostat"
-      - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='AWS_ACCESS_KEY_ID')].value
-          value: $(QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_ACCESS_KEY_ID)
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='AWS_ACCESS_KEY_ID')].valueFrom.secretKeyRef
+          value:
+              key: "STORAGE_ACCESS_KEY_ID"
+              name: "RELEASE-NAME-storage-secret"
+              optional: false
       - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY')].valueFrom.secretKeyRef
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='AWS_SECRET_ACCESS_KEY')].valueFrom.secretKeyRef
           value:
               key: "STORAGE_ACCESS_KEY"
               name: "RELEASE-NAME-storage-secret"
               optional: false
-      - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='AWS_SECRET_ACCESS_KEY')].value
-          value: $(QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY)
       - notExists:
           path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_REST_CLIENT_REPORTS_URL')].value
       - equal:
@@ -183,6 +194,50 @@ tests:
       - notExists:
           path: spec.template.spec.contains[?(@.name=='cryostat')].volumeMounts
 
+  - it: should allow overriding S3 bucket names
+    set:
+      storage:
+        provider:
+          metadata:
+            storageMode: bucket
+        buckets:
+          names:
+            archivedRecordings: a
+            archivedReports: b
+            eventTemplates: c
+            jmcAgentProbeTemplates: d
+            metadata: e
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_METADATA_STORAGE_MODE')].value
+          value: "bucket"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_ARCHIVES_NAME')].value
+          value: "a"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='CRYOSTAT_SERVICES_REPORTS_STORAGE_CACHE_NAME')].value
+          value: "b"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_EVENT_TEMPLATES_NAME')].value
+          value: "c"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_PROBE_TEMPLATES_NAME')].value
+          value: "d"
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='STORAGE_BUCKETS_METADATA_NAME')].value
+          value: "e"
+
+  - it: should allow disabling S3 TLS validation
+    set:
+      storage:
+        provider:
+          tls:
+            trustAll: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat')].env[?(@.name=='QUARKUS_S3_SYNC_CLIENT_TLS_TRUST_MANAGERS_PROVIDER_TYPE')].value
+          value: trust-all
+
   - it: should set log level
     set:
       core:

charts/cryostat/tests/storage_deployment_test.yaml

@@ -69,16 +69,19 @@ tests:
           value: "Always"
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='CRYOSTAT_BUCKETS')].value
-          value: "archivedrecordings,archivedreports,eventtemplates,probes"
+          value: "archivedrecordings,archivedreports,eventtemplates,metadata,probes"
       - equal:
-          path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='CRYOSTAT_ACCESS_KEY')].value
-          value: "cryostat"
+          path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='CRYOSTAT_ACCESS_KEY')].valueFrom.secretKeyRef
+          value:
+            name: "RELEASE-NAME-storage-secret"
+            key: "STORAGE_ACCESS_KEY_ID"
+            optional: false
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='CRYOSTAT_SECRET_KEY')].valueFrom.secretKeyRef
           value:
-              name: "RELEASE-NAME-storage-secret"
-              key: "STORAGE_ACCESS_KEY"
-              optional: false
+            name: "RELEASE-NAME-storage-secret"
+            key: "STORAGE_ACCESS_KEY"
+            optional: false
       - equal:
           path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='DATA_DIR')].value
           value: "/data"
@@ -134,6 +137,21 @@ tests:
               cpu: 50m
               memory: 256Mi
 
+  - it: should precreate buckets based on supplied bucket names
+    set:
+      storage:
+        buckets:
+          names:
+            metadata: a
+            archivedRecordings: b
+            archivedReports: c
+            eventTemplates: d
+            jmcAgentProbeTemplates: e
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=='cryostat-storage')].env[?(@.name=='CRYOSTAT_BUCKETS')].value
+          value: "a,b,c,d,e"
+
   - it: should allow disabling at-rest encryption
     set:
       storage:

charts/cryostat/tests/storage_secret_test.yaml

@@ -18,6 +18,8 @@ tests:
           value: Opaque
       - exists:
           path: data.STORAGE_ACCESS_KEY
+      - exists:
+          path: data.STORAGE_ACCESS_KEY_ID
       - equal:
           path: metadata.labels
           value: