Bitnami switchoff

.github/workflows/ci.yaml

@@ -45,6 +45,8 @@ jobs:
         run: helm repo add bitnami https://charts.bitnami.com/bitnami
       - name: Add elastic repo
         run: helm repo add elastic https://helm.elastic.co
+      - name: Add mongodb repo
+        run: helm repo add mongodb-kubernetes https://mongodb.github.io/helm-charts
       - name: Add prometheus repo
         run: helm repo add prometheus https://prometheus-community.github.io/helm-charts
       - name: Add grafana repo

.github/workflows/release.yaml

@@ -15,6 +15,8 @@ jobs:
         uses: actions/checkout@v1
       - name: Add bitnami repo
         run: helm repo add bitnami https://charts.bitnami.com/bitnami
+      - name: Add mongodb repo
+        run: helm repo add mongodb-kubernetes https://mongodb.github.io/helm-charts
       - name: Add elastic repo
         run: helm repo add elastic https://helm.elastic.co
       - name: Add prometheus repo

charts/clearml/Chart.lock

@@ -1,12 +1,15 @@
 dependencies:
-- name: redis
-  repository: https://charts.bitnami.com/bitnami
-  version: 17.8.3
+- name: dragonfly
+  repository: oci://ghcr.io/dragonflydb/dragonfly/helm
+  version: v1.36.0
 - name: mongodb
   repository: https://charts.bitnami.com/bitnami
   version: 13.18.5
+- name: mongodb-kubernetes
+  repository: https://mongodb.github.io/helm-charts
+  version: 1.6.1
 - name: elasticsearch
   repository: https://helm.elastic.co
   version: 7.17.3
-digest: sha256:d5864444fa8c5cb66a83ec02cdbe4b0776f9f6538dfe67132be7eeb2781b50e4
-generated: "2025-01-07T15:18:46.845769+01:00"
+digest: sha256:75c08f9445fe7e5de2212036a63042cc95e65c59398c940974e468b234a7a08b
+generated: "2026-01-27T13:05:14.554201+01:00"

charts/clearml/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: clearml
 description: MLOps platform
 type: application
-version: "7.14.7"
+version: "7.15.0"
 appVersion: "2.0"
 kubeVersion: ">= 1.21.0-0"
 home: https://clear.ml
@@ -13,19 +13,26 @@ sources:
 maintainers:
   - name: filippo-clearml
     url: https://github.com/filippo-clearml
+  - name: vm-clearml
+    url: https://github.com/vm-clearml
 keywords:
   - clearml
   - "machine learning"
   - mlops
 dependencies:
-  - name: redis
-    version: "17.8.3"
-    repository: "https://charts.bitnami.com/bitnami"
-    condition: redis.enabled
+  - name: dragonfly
+    version: "v1.36.0"
+    repository: "oci://ghcr.io/dragonflydb/dragonfly/helm"
+    condition: dragonfly.enabled
   - name: mongodb
     version: "13.18.5"
     repository: "https://charts.bitnami.com/bitnami"
     condition: mongodb.enabled
+  - name: mongodb-kubernetes
+    alias: mckMongodb
+    version: "1.6.1"
+    repository: "https://mongodb.github.io/helm-charts"
+    condition: mckMongodb.enabled
   - name: elasticsearch
     version: "7.17.3"
     repository: "https://helm.elastic.co"

charts/clearml/README.md

@@ -1,6 +1,6 @@
 # ClearML Ecosystem for Kubernetes
 
-![Version: 7.14.7](https://img.shields.io/badge/Version-7.14.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.0](https://img.shields.io/badge/AppVersion-2.0-informational?style=flat-square)
+![Version: 7.15.0](https://img.shields.io/badge/Version-7.15.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.0](https://img.shields.io/badge/AppVersion-2.0-informational?style=flat-square)
 
 MLOps platform
 
@@ -11,6 +11,7 @@ MLOps platform
 | Name | Email | Url |
 | ---- | ------ | --- |
 | filippo-clearml |  | <https://github.com/filippo-clearml> |
+| vm-clearml |  | <https://github.com/vm-clearml> |
 
 ## Introduction
 
@@ -33,6 +34,17 @@ To add this chart to your local Helm repository:
 helm repo add clearml https://clearml.github.io/clearml-helm-charts
 ```
 
+## Kubernetes CRD requirements
+
+During the initial Helm installation, Custom Resource Definitions (CRDs) are automatically installed.
+However, due to Helm's update strategy, these CRDs will not be updated automatically during subsequent `helm upgrade` operations.
+
+To ensure you have the latest CRD versions when performing an upgrade, please apply the CRDs manually before running `helm upgrade`:
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml
+```
+
 ## Local environment
 
 For development/evaluation it's possible to use [kind](https://kind.sigs.k8s.io).
@@ -99,6 +111,148 @@ A production ready cluster should also have some different configuration like th
 helm install clearml clearml/clearml -f values-production.yaml
 ```
 
+## Dependencies migration
+
+Starting chart version `7.15.0`, a migration path for database dependencies is available to help users transition to new bundled dependencies.
+
+**IMPORTANT**: This procedure is required to prepare for future Helm chart updates that may not be possible if the migration is not completed. The migration ensures data consistency and compatibility with upcoming chart versions.
+
+For detailed migration instructions, see the [Kubernetes Dependencies Migration Guide](k8s_dependencies_migration.md).
+
+### Procedure
+
+#### Phase 1: Enable MCK MongoDB Deployment
+
+### Prerequisites
+
+Export your `clearml namespace` (the default is `clearml` but verify your actual namespace as it may have been customized during installation):
+
+```bash
+CLEARML_NAMESPACE=clearml
+```
+
+Add CRDs following Helm chart README:
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml
+```
+
+Update your Helm chart configuration with the following values to deploy the MCK MongoDB instance, staying on Helm chart version `7.15.0`:
+
+```yaml
+mckMongodb:
+  enabled: true
+```
+
+Upgrade chart accordingly with helm upgrade command.
+
+**Note**: This step creates the new MongoDB instance in parallel with the existing Bitnami deployment, allowing for data migration without service interruption.
+
+#### Phase 2: Data Migration Process
+
+**2.1 Scale down the ClearML API server**
+
+Scale down ClearML services:
+
+```bash
+kubectl -n $CLEARML_NAMESPACE scale deployment -l app.kubernetes.io/name=clearml --replicas=0
+```
+
+**2.2 Deploy Migration Pod**
+
+Create a Kubernetes pod equipped with MongoDB tools for the migration process. Create a file called `mongodb-migrate.yaml`:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mongodb-migrate
+spec:
+  restartPolicy: Never
+  containers:
+    - name: mongo-tools
+      image: mongo:7
+      command: ["sleep", "infinity"]
+      env:
+        - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING
+          valueFrom:
+            secretKeyRef:
+              name: mongodb-root
+              key: connectionString
+```
+
+Deploy and access the migration pod:
+
+```bash
+kubectl -n $CLEARML_NAMESPACE apply -f mongodb-migrate.yaml
+```
+
+**2.3 Export Data from Source Database**
+
+Dump data from Bitnami MongoDB:
+
+```bash
+MONGO_CONNECTION_STRING=$(kubectl get deploy -n $CLEARML_NAMESPACE -o jsonpath='{.items[*].spec.template.spec.containers[?(@.name=="clearml-apiserver")].env[?(@.name=="CLEARML_MONGODB_SERVICE_CONNECTION_STRING")].value}{"\n"}')
+kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- mongodump --uri="$MONGO_CONNECTION_STRING" --archive=/dump.archive --gzip
+```
+
+Copy the dump to a local system so there's a further local copy just in case:
+
+```bash
+kubectl -n $CLEARML_NAMESPACE cp mongodb-migrate:/dump.archive /tmp/dump.archive
+```
+
+**2.4 Import Data to Target Database**
+
+Restore data to MCK MongoDB:
+
+```bash
+kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- sh -c 'mongorestore \
+  --uri="$CLEARML_MONGODB_SERVICE_CONNECTION_STRING" \
+  --archive=/dump.archive \
+  --gzip \
+  --drop'
+```
+
+#### Phase 3: Configuration Updates
+
+**3.1 Mark Migration as Complete**
+
+Update your Helm configuration to indicate successful data migration, staying on Helm chart version 7.15.0:
+
+```yaml
+mckMongodb:
+  enabled: true
+  migrated: true
+```
+
+Upgrade chart accordingly with helm upgrade command.
+
+**3.2 Validate System Functionality**
+
+Thoroughly test ClearML Enterprise functionality to ensure all components are working correctly with the new MongoDB deployment.
+
+**3.3 Decommission Bitnami MongoDB**
+
+Once validation is complete, disable the legacy Bitnami MongoDB deployment, staying on Helm chart version 7.15.0:
+
+```yaml
+mongodb:
+  enabled: false
+mckMongodb:
+  enabled: true
+  migrated: true
+```
+
+Upgrade chart accordingly with helm upgrade command.
+
+**3.4 Remove migration pod**
+
+Remove the migration pod:
+
+```bash
+kubectl -n $CLEARML_NAMESPACE delete -f mongodb-migrate.yaml
+
 ## Upgrades/ Values upgrades
 
 Updating to latest version of this chart can be done in two steps:
@@ -146,8 +300,9 @@ Kubernetes: `>= 1.21.0-0`
 | Repository | Name | Version |
 |------------|------|---------|
 | https://charts.bitnami.com/bitnami | mongodb | 13.18.5 |
-| https://charts.bitnami.com/bitnami | redis | 17.8.3 |
 | https://helm.elastic.co | elasticsearch | 7.17.3 |
+| https://mongodb.github.io/helm-charts | mckMongodb(mongodb-kubernetes) | 1.6.1 |
+| oci://ghcr.io/dragonflydb/dragonfly/helm | dragonfly | v1.36.0 |
 
 ## Values
 
@@ -208,13 +363,14 @@ Kubernetes: `>= 1.21.0-0`
 | clearml.secureAuthTokenSecret | string | `"ymLh1ok5k5xNUQfS944Xdx9xjf0wueokqKM2dMZfHuH9ayItG2"` | Secure Auth secret |
 | clearml.testUserKey | string | `"ENP39EQM4SLACGD5FXB7"` | Test Server basic auth key |
 | clearml.testUserSecret | string | `"lPcm0imbcBZ8mwgO7tpadutiS3gnJD05x9j7afwXPS35IKbpiQ"` | Test File Server basic auth secret |
+| dragonfly | object | `{"enabled":true,"replicaCount":1,"storage":{"enabled":true,"requests":"5Gi"}}` | Configuration from https://github.com/dragonflydb/dragonfly/blob/main/contrib/charts/dragonfly/values.yaml |
 | elasticsearch | object | `{"clusterHealthCheckParams":"wait_for_status=yellow&timeout=1s","clusterName":"clearml-elastic","enabled":true,"esConfig":{"elasticsearch.yml":"xpack.security.enabled: false\n"},"esJavaOpts":"-Xmx2g -Xms2g","extraEnvs":[{"name":"bootstrap.memory_lock","value":"false"},{"name":"cluster.routing.allocation.node_initial_primaries_recoveries","value":"500"},{"name":"cluster.routing.allocation.disk.watermark.low","value":"500mb"},{"name":"cluster.routing.allocation.disk.watermark.high","value":"500mb"},{"name":"cluster.routing.allocation.disk.watermark.flood_stage","value":"500mb"},{"name":"http.compression_level","value":"7"},{"name":"reindex.remote.whitelist","value":"*.*"},{"name":"xpack.monitoring.enabled","value":"false"},{"name":"xpack.security.enabled","value":"false"}],"httpPort":9200,"minimumMasterNodes":1,"persistence":{"enabled":true},"rbac":{"create":true},"replicas":1,"resources":{"limits":{"cpu":"2000m","memory":"4Gi"},"requests":{"cpu":"100m","memory":"2Gi"}},"roles":{"data":"true","ingest":"true","master":"true","remote_cluster_client":"true"},"volumeClaimTemplate":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"50Gi"}},"storageClassName":null}}` | Configuration from https://github.com/elastic/helm-charts/blob/7.16/elasticsearch/values.yaml |
-| externalServices | object | `{"elasticsearchConnectionString":"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]","mongodbConnectionStringAuth":"mongodb://mongodb_hostname:27017/auth","mongodbConnectionStringBackend":"mongodb://mongodb_hostnamehostname:27017/backend","redisHost":"redis_hostname","redisPort":6379}` | Definition of external services to use if not enabled as dependency charts here |
+| externalServices | object | `{"dragonflyHost":"dragonfly_hostname","dragonflyPort":6379,"elasticsearchConnectionString":"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]","mongodbConnectionStringAuth":"mongodb://mongodb_hostname:27017/auth","mongodbConnectionStringBackend":"mongodb://mongodb_hostnamehostname:27017/backend"}` | Definition of external services to use if not enabled as dependency charts here |
+| externalServices.dragonflyHost | string | `"dragonfly_hostname"` | Existing Redis Hostname to use if dragonfly.enabled is false (example in values.yaml) |
+| externalServices.dragonflyPort | int | `6379` | Existing Redis Port to use if dragonfly.enabled is false |
 | externalServices.elasticsearchConnectionString | string | `"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]"` | Existing ElasticSearch connectionstring if elasticsearch.enabled is false (example in values.yaml) |
 | externalServices.mongodbConnectionStringAuth | string | `"mongodb://mongodb_hostname:27017/auth"` | Existing MongoDB connection string for BACKEND to use if mongodb.enabled is false (example in values.yaml) |
 | externalServices.mongodbConnectionStringBackend | string | `"mongodb://mongodb_hostnamehostname:27017/backend"` | Existing MongoDB connection string for AUTH to use if mongodb.enabled is false (example in values.yaml) |
-| externalServices.redisHost | string | `"redis_hostname"` | Existing Redis Hostname to use if redis.enabled is false (example in values.yaml) |
-| externalServices.redisPort | int | `6379` | Existing Redis Port to use if redis.enabled is false |
 | fileserver | object | `{"additionalVolumeMounts":{},"additionalVolumes":{},"affinity":{},"containerSecurityContext":{},"deploymentAnnotations":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"2.0.0-613"},"ingress":{"annotations":{},"enabled":false,"hostName":"files.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30081,"port":8081,"type":"NodePort"},"serviceAccountAnnotations":{},"serviceAccountName":"clearml","storage":{"data":{"accessMode":"ReadWriteOnce","class":"","existingPVC":"","size":"50Gi"},"enabled":true},"tolerations":[]}` | File Server configurations |
 | fileserver.additionalVolumeMounts | object | `{}` | Specifies where and how the volumes defined in additionalVolumes. |
 | fileserver.additionalVolumes | object | `{}` | # Defines extra Kubernetes volumes to be attached to the pod. |
@@ -257,8 +413,8 @@ Kubernetes: `>= 1.21.0-0`
 | imageCredentials.password | string | `"pwd"` | Registry password |
 | imageCredentials.registry | string | `"docker.io"` | Registry name |
 | imageCredentials.username | string | `"someone"` | Registry username |
+| mckMongodb | object | `{"community":{"createResource":true,"resource":{"members":1,"name":"mongodb-replica-set","users":[{"db":"admin","name":"root","passwordSecretRef":{"name":"mongodb-root"},"roles":[{"db":"admin","name":"clusterAdmin"},{"db":"admin","name":"userAdminAnyDatabase"},{"db":"admin","name":"readWriteAnyDatabase"},{"db":"admin","name":"dbAdminAnyDatabase"}],"scramCredentialsSecretName":"clearml-scram"}],"version":"7.0.28"}},"enabled":false,"migrated":false}` | Configuration from https://github.com/mongodb/mongodb-kubernetes/blob/1.6.1/helm_chart/values.yaml |
 | mongodb | object | `{"architecture":"standalone","auth":{"enabled":false},"enabled":true,"image":{"repository":"bitnamilegacy/mongodb"},"persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"50Gi","storageClass":null},"replicaCount":1,"updateStrategy":{"rollingUpdate":{"maxSurge":0,"maxUnavailable":1},"type":"RollingUpdate"}}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/mongodb/values.yaml |
-| redis | object | `{"architecture":"standalone","auth":{"enabled":false},"databaseNumber":0,"enabled":true,"image":{"repository":"bitnamilegacy/redis"},"master":{"name":"{{ .Release.Name }}-redis-master","persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"5Gi","storageClass":null},"port":6379}}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml |
 | webserver | object | `{"additionalConfigs":{},"additionalVolumeMounts":{},"additionalVolumes":{},"affinity":{},"containerSecurityContext":{},"deploymentAnnotations":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"2.0.0-613"},"ingress":{"annotations":{},"enabled":false,"hostName":"app.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30080,"port":8080,"type":"NodePort"},"serviceAccountAnnotations":{},"serviceAccountName":"clearml","tolerations":[]}` | Web Server configurations |
 | webserver.additionalConfigs | object | `{}` | Additional specific webserver configurations |
 | webserver.additionalVolumeMounts | object | `{}` | Specifies where and how the volumes defined in additionalVolumes. |