use data folder with S3

This will mount the data folder even when using S3 storage. This data folder in that case will be used to host the temp files. If not using persistence, make sure tempfolder points to a writeable place.

Author: robkooper

.gitignore

@@ -12,3 +12,6 @@ clowder-*.tgz
 secrets-*.yaml
 values-*.yaml
 *.values.yaml
+
+# backup files
+*~

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Change Log
 
+## 0.16.6 
+
+**Release date:** 2023-04-10
+
+![AppVersion: 1.21.0](https://img.shields.io/static/v1?label=AppVersion&message=1.21.0&color=success&logo=)
+![Helm: v3](https://img.shields.io/static/v1?label=Helm&message=v3&color=informational&logo=helm)
+
+
+* use data folder with S3 
+
+### Default value changes
+
+```diff
+diff --git a/values.yaml b/values.yaml
+index 4b084c8..82f1c41 100644
+--- a/values.yaml
++++ b/values.yaml
+@@ -70,6 +70,8 @@ groupid: 0
+ 
+ ## tempfolder is the place where clowder will safe the files when they are uploaded. Best
+ ## is for this to be on the same volume as the final storage, or on a fast storage.
++## If persistence is not enabled, make sure this points to a folder  writeable by the
++## clowder pod.
+ tempfolder: /home/clowder/data/tmp/
+ 
+ ## helpmenu is a list of entries that are added to the help menu in clowder, each entry has
+@@ -188,6 +190,8 @@ minio:
+ ## Enable persistence using Persistent Volume Claims
+ ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+ ##
++## This is used for data and temp space. In case of S3 storage, this should
++## be at least as big as the biggest uploaded file.
+ persistence:
+   enabled: true
+   ## A manually managed Persistent Volume and Claim
+```
+
 ## 0.16.5 
 
 **Release date:** 2023-04-10

Chart.yaml

@@ -13,7 +13,7 @@ description: >
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.16.5
+version: 0.16.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -55,4 +55,4 @@ annotations:
     - name: Helm Chart
       url: https://github.com/clowder-framework/clowder-helm
   artifacthub.io/changes: |
-    - "fix tmp folder with s3 again"
+    - "use data folder with S3"

changelog.sh

@@ -7,3 +7,5 @@ changelog="$(sed -e "1,/^## ${version}/d" -e "/^###/,\$d" -e '/^$/d' -e 's/^* /-
 sed -i~ -e '/^  artifacthub.io\/changes: |/,$d' Chart.yaml
 echo "  artifacthub.io/changes: |" >> Chart.yaml
 echo "${changelog}" >> Chart.yaml
+
+git commit --amend Chart.yaml CHANGELOG.md

templates/deployment.yaml

@@ -35,6 +35,7 @@ spec:
                 secretKeyRef:
                   name: {{ include "clowder.fullname" . }}
                   key: mongodb-uri
+{{- if .Values.persistence.enabled }}
         - name: setup-data-folder
           image: busybox:latest
           command: ["/bin/sh"]
@@ -47,13 +48,12 @@ spec:
             runAsUser: 0
             runAsGroup: 0
           volumeMounts:
-{{- if not (or .Values.storage.s3 .Values.minio.enabled) }}
           - name: data
             mountPath: /home/clowder/data
-{{- end }}
           {{- with .Values.extraVolumeMounts }}
           {{- . | toYaml | nindent 10 }}
           {{- end }}
+{{- end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -116,7 +116,7 @@ spec:
           volumeMounts:
             - name: custom-volume
               mountPath: /home/clowder/custom
-{{- if not (or .Values.storage.s3 .Values.minio.enabled) }}
+{{- if .Values.persistence.enabled }}
             - name: data
               mountPath: /home/clowder/data
 {{- end }}
@@ -144,15 +144,15 @@ spec:
                 path: custom.conf
               - key: play.plugins
                 path: play.plugins
-        {{- if not (or .Values.storage.s3 .Values.minio.enabled) }}
+{{- if .Values.persistence.enabled }}
         - name: data
           persistentVolumeClaim:
             {{- if $.Values.persistence.existingClaim }}
             claimName: {{ $.Values.persistence.existingClaim }}
             {{- else }}
             claimName: {{ include "clowder.fullname" . }}-data
             {{- end }}
-        {{- end }}
+{{- end }}
         {{- with .Values.extraVolumes }}
           {{- . | toYaml | nindent 8 }}
         {{- end }}

templates/persistence.yaml

@@ -1,4 +1,3 @@
-{{- if not (or .Values.storage.s3 .Values.minio.enabled) }}
 {{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
 kind: PersistentVolumeClaim
 apiVersion: v1
@@ -16,4 +15,3 @@ spec:
   storageClassName: {{ .Values.persistence.storageClass }}
   {{- end -}}
 {{- end }}
-{{- end }}

values.yaml

@@ -70,6 +70,8 @@ groupid: 0
 
 ## tempfolder is the place where clowder will safe the files when they are uploaded. Best
 ## is for this to be on the same volume as the final storage, or on a fast storage.
+## If persistence is not enabled, make sure this points to a folder  writeable by the
+## clowder pod.
 tempfolder: /home/clowder/data/tmp/
 
 ## helpmenu is a list of entries that are added to the help menu in clowder, each entry has
@@ -188,6 +190,8 @@ minio:
 ## Enable persistence using Persistent Volume Claims
 ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
 ##
+## This is used for data and temp space. In case of S3 storage, this should
+## be at least as big as the biggest uploaded file.
 persistence:
   enabled: true
   ## A manually managed Persistent Volume and Claim