Merge branch 'release/v2.0-beta-3' into 978-migrate-users

Author: tcnichol

.gitignore

@@ -59,6 +59,7 @@ scripts/keycloak/data/*
 # ignore clowder chart deps
 deployments/kubernetes/charts/clowder2/charts
 deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml
+*secret*.yaml
 
 # Environments
 .env

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v2.0.0-beta.3] - 2024-07-29
+
+### Added
+
+- License management
+- Release dataset with versions
+- Enable and disable user account through Keycloak
+- Jupyterhub integration
+- Interface for creating and editing matching criteria for triggering extractors
+- Interface for editing metadata definitions
+- My dataset tab listing all the datasets created by the user
+- Drag and drop upload multiple files
+- Footer with links to documentation, source code, and contact information
+- Documentation through MKDocs
+
+### Changed
+
+- Allow public datasets and files to be searchable
+- List all the extractors with the ability to enable/disable the extractors
+- Filter listeners based on their support for file or dataset
+- Helm chart updated to support custom existing secret
+
+### Fixed
+
+- Clowder registration link on the top bar
+- Case-insensitive search
+- Download count immediately increments after download
+
 ## [v2.0.0-beta.2] - 2024-02-16
 
 ### Added

backend/app/config.py

@@ -9,7 +9,7 @@ class Settings(BaseSettings):
     API_V2_STR: str = "/api/v2"
     admin_email: str = "[email protected]"
     frontend_url: str = "http://localhost:3000"
-    version: str = "2.0.0-beta.2"
+    version: str = "2.0.0-beta.3"
 
     # Unique secret for hashing API keys. Generate with `openssl rand -hex 32`
     local_auth_secret = "clowder_secret_key"

backend/app/main.py

@@ -86,7 +86,7 @@
     description="A cloud native data management framework to support any research domain. Clowder was "
     "developed to help researchers and scientists in data intensive domains manage raw data, complex "
     "metadata, and automatic data pipelines. ",
-    version="2.0.0-beta.2",
+    version="2.0.0-beta.3",
     contact={"name": "Clowder", "url": "https://clowderframework.org/"},
     license_info={
         "name": "Apache 2.0",

deployments/kubernetes/charts/clowder2/Chart.yaml

@@ -13,13 +13,13 @@ description: >
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.9.1
+version: 1.9.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.0.0-beta.2"
+appVersion: "2.0.0-beta.3"
 
 # List of people that maintain this helm chart.
 maintainers:

deployments/kubernetes/charts/clowder2/README.md

@@ -17,13 +17,13 @@ hostname: { your hostname. E.g. clowder2.software-dev.ncsa.illinois.edu }
 frontend:
   image:
     repository: clowder/clowder2-frontend
-    tag: 2.0.0-beta.2
+    tag: 2.0.0-beta.3
     pullPolicy: Always
 
 backend:
   image:
     repository: clowder/clowder2-backend
-    tag: 2.0.0-beta.2
+    tag: 2.0.0-beta.3
     pullPolicy: Always
 
 geoserver:

deployments/kubernetes/charts/clowder2/templates/NOTES.txt

@@ -15,7 +15,7 @@
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "clowder2.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
   echo http://$SERVICE_IP:{{ .Values.frontend.service.port }}
 {{- else if contains "ClusterIP" .Values.frontend.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "clowder2.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "clowder2.fullname" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT

deployments/kubernetes/charts/clowder2/templates/_helpers.tpl

@@ -46,7 +46,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Selector labels
 */}}
 {{- define "clowder2.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "clowder2.name" . }}
+app.kubernetes.io/name: {{ include "clowder2.fullname" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 

deployments/kubernetes/charts/clowder2/templates/backend/deployment.yaml

@@ -10,7 +10,7 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: {{ include "clowder2.name" . }}-backend
+      app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-backend
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
@@ -19,7 +19,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        app.kubernetes.io/name: {{ include "clowder2.name" . }}-backend
+        app.kubernetes.io/name: {{ include "clowder2.fullname" . }}-backend
         app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
       {{- with .Values.imagePullSecrets }}
@@ -39,25 +39,31 @@ spec:
             - name: WEB_CONCURRENCY
               value: "1"
             - name: MINIO_SERVER_URL
-              value: {{ include "clowder2.name" . }}-minio:9000
-#              value: {{ include "clowder2.name" . }}-minio-headless:9000
+              value: {{ .Values.backend.env.MINIO_SERVER_URL }}
             - name: MINIO_EXTERNAL_SERVER_URL
               value: minio-api.{{ .Values.hostname }}
             - name: MINIO_SECURE
-              value: "true"
+              value: {{ .Values.backend.env.MINIO_SECURE | quote }}
             - name: MINIO_BUCKET_NAME
-              value: clowder
+              value: {{ .Values.backend.env.MINIO_BUCKET_NAME }}
             - name: MINIO_ACCESS_KEY
               value: {{ .Values.minio.auth.rootUser }}
             - name: MINIO_SECRET_KEY
+              {{- if .Values.backend.existingSecret }}
               valueFrom:
                 secretKeyRef:
-                  name: clowder2-minio
+                  name: {{ .Values.backend.existingSecret }}
+                  key: {{ .Values.backend.existingMinioSecretKey | default "root-password" }}
+              {{- else }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clowder2.fullname" . }}-minio
                   key: root-password
+              {{- end }}
             - name: MINIO_UPLOAD_CHUNK_SIZE
-              value: "10485760"
+              value: {{ .Values.backend.env.MINIO_UPLOAD_CHUNK_SIZE | quote}}
             - name: MONGODB_URL
-              value: mongodb://{{ include "clowder2.name" . }}-mongodb:27017
+              value: {{ .Values.backend.env.MONGODB_URL }}
             - name: MONGO_DATABASE
               value: {{ .Values.mongodb.database }}
             - name: CLOWDER2_URL
@@ -73,37 +79,43 @@ spec:
             - name: auth_url
               value: $(CLOWDER2_URL)/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code
             - name: oauth2_scheme_auth_url
-              value: http://{{ include "clowder2.name" .}}-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/auth?client_id=clowder2-backend&response_type=code
+              value: {{ .Values.backend.env.oauth2_scheme_auth_url }}
             - name: auth_register_url
-              value: $(CLOWDER2_URL)/keycloak/realms/clowder/protocol/openid-connect/registrations?client_id=clowder2-backend&response_type=code&redirect_uri=$(auth_redirect_uri)&scope=openid%20email
+              value: $(CLOWDER2_URL){{ .Values.backend.env.auth_register_url }}
             - name: auth_token_url
-              value: http://{{ include "clowder2.name" .}}-keycloak-headless:8080/keycloak/realms/clowder/protocol/openid-connect/token
+              value: {{ .Values.backend.env.auth_token_url }}
             - name: auth_server_url
               value: $(CLOWDER2_URL)/keycloak/
             - name: keycloak_base
               value: $(CLOWDER2_URL)/api
             - name: frontend_url
               value: $(CLOWDER2_URL)
             - name: elasticsearch_url
-              value: http://{{ include "clowder2.name" . }}-elasticsearch:9200
+              value: {{ .Values.backend.env.elasticsearch_url }}
             - name: elasticsearch_no_of_shards
               value: "5"
             - name: elasticsearch_no_of_replicas
               value: "5"
             - name: RABBITMQ_USER
               value: {{ .Values.rabbitmq.auth.username }}
             - name: RABBITMQ_PASS
+              {{- if .Values.backend.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.backend.existingSecret }}
+                  key: {{ .Values.backend.existingRabbitMQSecretKey | default "rabbitmq-password" }}
+              {{- else }}
               valueFrom:
                 secretKeyRef:
-                  name: clowder2-rabbitmq
+                  name: {{ include "clowder2.fullname" . }}-rabbitmq
                   key: rabbitmq-password
+              {{- end }}
             - name: RABBITMQ_HOST
-              value: {{ include "clowder2.name" .  }}-rabbitmq
+              value: {{ .Values.backend.env.RABBITMQ_HOST }}
             - name: HEARTBEAT_EXCHANGE
               value: "extractors"
             - name: API_HOST
-#              value: $(CLOWDER2_URL)
-              value: http://{{ include "clowder2.name" . }}-backend:{{ .Values.backend.service.port }}
+              value: {{ .Values.backend.env.API_HOST }}
           ports:
             - name: http
               containerPort: 80

deployments/kubernetes/charts/clowder2/templates/backend/ingress.yaml

@@ -21,7 +21,6 @@ spec:
   {{- if .Values.ingress.tls }}
   tls:
     - hosts:
-        - {{ .Values.hostname }}
     {{- range .Values.ingress.tls }}
         {{- range .hosts }}
         - {{ . | quote }}