diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0d2e6f3d..8980c38c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,8 @@ jobs: run: helm repo add bitnami https://charts.bitnami.com/bitnami - name: Add elastic repo run: helm repo add elastic https://helm.elastic.co + - name: Add mongodb repo + run: helm repo add mongodb-kubernetes https://mongodb.github.io/helm-charts - name: Add prometheus repo run: helm repo add prometheus https://prometheus-community.github.io/helm-charts - name: Add grafana repo diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c31f828f..c062a9d4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,6 +15,8 @@ jobs: uses: actions/checkout@v1 - name: Add bitnami repo run: helm repo add bitnami https://charts.bitnami.com/bitnami + - name: Add mongodb repo + run: helm repo add mongodb-kubernetes https://mongodb.github.io/helm-charts - name: Add elastic repo run: helm repo add elastic https://helm.elastic.co - name: Add prometheus repo diff --git a/charts/clearml/Chart.lock b/charts/clearml/Chart.lock index 818ee1ad..f55d05c9 100644 --- a/charts/clearml/Chart.lock +++ b/charts/clearml/Chart.lock @@ -1,12 +1,15 @@ dependencies: -- name: redis - repository: https://charts.bitnami.com/bitnami - version: 17.8.3 +- name: dragonfly + repository: oci://ghcr.io/dragonflydb/dragonfly/helm + version: v1.36.0 - name: mongodb repository: https://charts.bitnami.com/bitnami version: 13.18.5 +- name: mongodb-kubernetes + repository: https://mongodb.github.io/helm-charts + version: 1.6.1 - name: elasticsearch repository: https://helm.elastic.co version: 7.17.3 -digest: sha256:d5864444fa8c5cb66a83ec02cdbe4b0776f9f6538dfe67132be7eeb2781b50e4 -generated: "2025-01-07T15:18:46.845769+01:00" +digest: sha256:75c08f9445fe7e5de2212036a63042cc95e65c59398c940974e468b234a7a08b +generated: "2026-01-27T13:05:14.554201+01:00" diff --git a/charts/clearml/Chart.yaml b/charts/clearml/Chart.yaml index d77960b8..65ae7f7c 100644 --- a/charts/clearml/Chart.yaml +++ b/charts/clearml/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: clearml description: MLOps platform type: application -version: "7.14.7" +version: "7.15.0" appVersion: "2.0" kubeVersion: ">= 1.21.0-0" home: https://clear.ml @@ -13,19 +13,26 @@ sources: maintainers: - name: filippo-clearml url: https://github.com/filippo-clearml + - name: vm-clearml + url: https://github.com/vm-clearml keywords: - clearml - "machine learning" - mlops dependencies: - - name: redis - version: "17.8.3" - repository: "https://charts.bitnami.com/bitnami" - condition: redis.enabled + - name: dragonfly + version: "v1.36.0" + repository: "oci://ghcr.io/dragonflydb/dragonfly/helm" + condition: dragonfly.enabled - name: mongodb version: "13.18.5" repository: "https://charts.bitnami.com/bitnami" condition: mongodb.enabled + - name: mongodb-kubernetes + alias: mckMongodb + version: "1.6.1" + repository: "https://mongodb.github.io/helm-charts" + condition: mckMongodb.enabled - name: elasticsearch version: "7.17.3" repository: "https://helm.elastic.co" diff --git a/charts/clearml/README.md b/charts/clearml/README.md index cfcfb68d..9d1e0303 100644 --- a/charts/clearml/README.md +++ b/charts/clearml/README.md @@ -1,6 +1,6 @@ # ClearML Ecosystem for Kubernetes -![Version: 7.14.7](https://img.shields.io/badge/Version-7.14.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.0](https://img.shields.io/badge/AppVersion-2.0-informational?style=flat-square) +![Version: 7.15.0](https://img.shields.io/badge/Version-7.15.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.0](https://img.shields.io/badge/AppVersion-2.0-informational?style=flat-square) MLOps platform @@ -11,6 +11,7 @@ MLOps platform | Name | Email | Url | | ---- | ------ | --- | | filippo-clearml | | | +| vm-clearml | | | ## Introduction @@ -33,6 +34,17 @@ To add this chart to your local Helm repository: helm repo add clearml https://clearml.github.io/clearml-helm-charts ``` +## Kubernetes CRD requirements + +During the initial Helm installation, Custom Resource Definitions (CRDs) are automatically installed. +However, due to Helm's update strategy, these CRDs will not be updated automatically during subsequent `helm upgrade` operations. + +To ensure you have the latest CRD versions when performing an upgrade, please apply the CRDs manually before running `helm upgrade`: + +```bash +kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml +``` + ## Local environment For development/evaluation it's possible to use [kind](https://kind.sigs.k8s.io). @@ -99,6 +111,148 @@ A production ready cluster should also have some different configuration like th helm install clearml clearml/clearml -f values-production.yaml ``` +## Dependencies migration + +Starting chart version `7.15.0`, a migration path for database dependencies is available to help users transition to new bundled dependencies. + +**IMPORTANT**: This procedure is required to prepare for future Helm chart updates that may not be possible if the migration is not completed. The migration ensures data consistency and compatibility with upcoming chart versions. + +For detailed migration instructions, see the [Kubernetes Dependencies Migration Guide](k8s_dependencies_migration.md). + +### Procedure + +#### Phase 1: Enable MCK MongoDB Deployment + +### Prerequisites + +Export your `clearml namespace` (the default is `clearml` but verify your actual namespace as it may have been customized during installation): + +```bash +CLEARML_NAMESPACE=clearml +``` + +Add CRDs following Helm chart README: + +```bash +kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml +``` + +Update your Helm chart configuration with the following values to deploy the MCK MongoDB instance, staying on Helm chart version `7.15.0`: + +```yaml +mckMongodb: + enabled: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**Note**: This step creates the new MongoDB instance in parallel with the existing Bitnami deployment, allowing for data migration without service interruption. + +#### Phase 2: Data Migration Process + +**2.1 Scale down the ClearML API server** + +Scale down ClearML services: + +```bash +kubectl -n $CLEARML_NAMESPACE scale deployment -l app.kubernetes.io/name=clearml --replicas=0 +``` + +**2.2 Deploy Migration Pod** + +Create a Kubernetes pod equipped with MongoDB tools for the migration process. Create a file called `mongodb-migrate.yaml`: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: mongodb-migrate +spec: + restartPolicy: Never + containers: + - name: mongo-tools + image: mongo:7 + command: ["sleep", "infinity"] + env: + - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: mongodb-root + key: connectionString +``` + +Deploy and access the migration pod: + +```bash +kubectl -n $CLEARML_NAMESPACE apply -f mongodb-migrate.yaml +``` + +**2.3 Export Data from Source Database** + +Dump data from Bitnami MongoDB: + +```bash +MONGO_CONNECTION_STRING=$(kubectl get deploy -n $CLEARML_NAMESPACE -o jsonpath='{.items[*].spec.template.spec.containers[?(@.name=="clearml-apiserver")].env[?(@.name=="CLEARML_MONGODB_SERVICE_CONNECTION_STRING")].value}{"\n"}') +kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- mongodump --uri="$MONGO_CONNECTION_STRING" --archive=/dump.archive --gzip +``` + +Copy the dump to a local system so there's a further local copy just in case: + +```bash +kubectl -n $CLEARML_NAMESPACE cp mongodb-migrate:/dump.archive /tmp/dump.archive +``` + +**2.4 Import Data to Target Database** + +Restore data to MCK MongoDB: + +```bash +kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- sh -c 'mongorestore \ + --uri="$CLEARML_MONGODB_SERVICE_CONNECTION_STRING" \ + --archive=/dump.archive \ + --gzip \ + --drop' +``` + +#### Phase 3: Configuration Updates + +**3.1 Mark Migration as Complete** + +Update your Helm configuration to indicate successful data migration, staying on Helm chart version 7.15.0: + +```yaml +mckMongodb: + enabled: true + migrated: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**3.2 Validate System Functionality** + +Thoroughly test ClearML Enterprise functionality to ensure all components are working correctly with the new MongoDB deployment. + +**3.3 Decommission Bitnami MongoDB** + +Once validation is complete, disable the legacy Bitnami MongoDB deployment, staying on Helm chart version 7.15.0: + +```yaml +mongodb: + enabled: false +mckMongodb: + enabled: true + migrated: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**3.4 Remove migration pod** + +Remove the migration pod: + +```bash +kubectl -n $CLEARML_NAMESPACE delete -f mongodb-migrate.yaml + ## Upgrades/ Values upgrades Updating to latest version of this chart can be done in two steps: @@ -146,8 +300,9 @@ Kubernetes: `>= 1.21.0-0` | Repository | Name | Version | |------------|------|---------| | https://charts.bitnami.com/bitnami | mongodb | 13.18.5 | -| https://charts.bitnami.com/bitnami | redis | 17.8.3 | | https://helm.elastic.co | elasticsearch | 7.17.3 | +| https://mongodb.github.io/helm-charts | mckMongodb(mongodb-kubernetes) | 1.6.1 | +| oci://ghcr.io/dragonflydb/dragonfly/helm | dragonfly | v1.36.0 | ## Values @@ -208,13 +363,14 @@ Kubernetes: `>= 1.21.0-0` | clearml.secureAuthTokenSecret | string | `"ymLh1ok5k5xNUQfS944Xdx9xjf0wueokqKM2dMZfHuH9ayItG2"` | Secure Auth secret | | clearml.testUserKey | string | `"ENP39EQM4SLACGD5FXB7"` | Test Server basic auth key | | clearml.testUserSecret | string | `"lPcm0imbcBZ8mwgO7tpadutiS3gnJD05x9j7afwXPS35IKbpiQ"` | Test File Server basic auth secret | +| dragonfly | object | `{"enabled":true,"replicaCount":1,"storage":{"enabled":true,"requests":"5Gi"}}` | Configuration from https://github.com/dragonflydb/dragonfly/blob/main/contrib/charts/dragonfly/values.yaml | | elasticsearch | object | `{"clusterHealthCheckParams":"wait_for_status=yellow&timeout=1s","clusterName":"clearml-elastic","enabled":true,"esConfig":{"elasticsearch.yml":"xpack.security.enabled: false\n"},"esJavaOpts":"-Xmx2g -Xms2g","extraEnvs":[{"name":"bootstrap.memory_lock","value":"false"},{"name":"cluster.routing.allocation.node_initial_primaries_recoveries","value":"500"},{"name":"cluster.routing.allocation.disk.watermark.low","value":"500mb"},{"name":"cluster.routing.allocation.disk.watermark.high","value":"500mb"},{"name":"cluster.routing.allocation.disk.watermark.flood_stage","value":"500mb"},{"name":"http.compression_level","value":"7"},{"name":"reindex.remote.whitelist","value":"*.*"},{"name":"xpack.monitoring.enabled","value":"false"},{"name":"xpack.security.enabled","value":"false"}],"httpPort":9200,"minimumMasterNodes":1,"persistence":{"enabled":true},"rbac":{"create":true},"replicas":1,"resources":{"limits":{"cpu":"2000m","memory":"4Gi"},"requests":{"cpu":"100m","memory":"2Gi"}},"roles":{"data":"true","ingest":"true","master":"true","remote_cluster_client":"true"},"volumeClaimTemplate":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"50Gi"}},"storageClassName":null}}` | Configuration from https://github.com/elastic/helm-charts/blob/7.16/elasticsearch/values.yaml | -| externalServices | object | `{"elasticsearchConnectionString":"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]","mongodbConnectionStringAuth":"mongodb://mongodb_hostname:27017/auth","mongodbConnectionStringBackend":"mongodb://mongodb_hostnamehostname:27017/backend","redisHost":"redis_hostname","redisPort":6379}` | Definition of external services to use if not enabled as dependency charts here | +| externalServices | object | `{"dragonflyHost":"dragonfly_hostname","dragonflyPort":6379,"elasticsearchConnectionString":"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]","mongodbConnectionStringAuth":"mongodb://mongodb_hostname:27017/auth","mongodbConnectionStringBackend":"mongodb://mongodb_hostnamehostname:27017/backend"}` | Definition of external services to use if not enabled as dependency charts here | +| externalServices.dragonflyHost | string | `"dragonfly_hostname"` | Existing Redis Hostname to use if dragonfly.enabled is false (example in values.yaml) | +| externalServices.dragonflyPort | int | `6379` | Existing Redis Port to use if dragonfly.enabled is false | | externalServices.elasticsearchConnectionString | string | `"[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]"` | Existing ElasticSearch connectionstring if elasticsearch.enabled is false (example in values.yaml) | | externalServices.mongodbConnectionStringAuth | string | `"mongodb://mongodb_hostname:27017/auth"` | Existing MongoDB connection string for BACKEND to use if mongodb.enabled is false (example in values.yaml) | | externalServices.mongodbConnectionStringBackend | string | `"mongodb://mongodb_hostnamehostname:27017/backend"` | Existing MongoDB connection string for AUTH to use if mongodb.enabled is false (example in values.yaml) | -| externalServices.redisHost | string | `"redis_hostname"` | Existing Redis Hostname to use if redis.enabled is false (example in values.yaml) | -| externalServices.redisPort | int | `6379` | Existing Redis Port to use if redis.enabled is false | | fileserver | object | `{"additionalVolumeMounts":{},"additionalVolumes":{},"affinity":{},"containerSecurityContext":{},"deploymentAnnotations":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"2.0.0-613"},"ingress":{"annotations":{},"enabled":false,"hostName":"files.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30081,"port":8081,"type":"NodePort"},"serviceAccountAnnotations":{},"serviceAccountName":"clearml","storage":{"data":{"accessMode":"ReadWriteOnce","class":"","existingPVC":"","size":"50Gi"},"enabled":true},"tolerations":[]}` | File Server configurations | | fileserver.additionalVolumeMounts | object | `{}` | Specifies where and how the volumes defined in additionalVolumes. | | fileserver.additionalVolumes | object | `{}` | # Defines extra Kubernetes volumes to be attached to the pod. | @@ -257,8 +413,8 @@ Kubernetes: `>= 1.21.0-0` | imageCredentials.password | string | `"pwd"` | Registry password | | imageCredentials.registry | string | `"docker.io"` | Registry name | | imageCredentials.username | string | `"someone"` | Registry username | +| mckMongodb | object | `{"community":{"createResource":true,"resource":{"members":1,"name":"mongodb-replica-set","users":[{"db":"admin","name":"root","passwordSecretRef":{"name":"mongodb-root"},"roles":[{"db":"admin","name":"clusterAdmin"},{"db":"admin","name":"userAdminAnyDatabase"},{"db":"admin","name":"readWriteAnyDatabase"},{"db":"admin","name":"dbAdminAnyDatabase"}],"scramCredentialsSecretName":"clearml-scram"}],"version":"7.0.28"}},"enabled":false,"migrated":false}` | Configuration from https://github.com/mongodb/mongodb-kubernetes/blob/1.6.1/helm_chart/values.yaml | | mongodb | object | `{"architecture":"standalone","auth":{"enabled":false},"enabled":true,"image":{"repository":"bitnamilegacy/mongodb"},"persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"50Gi","storageClass":null},"replicaCount":1,"updateStrategy":{"rollingUpdate":{"maxSurge":0,"maxUnavailable":1},"type":"RollingUpdate"}}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/mongodb/values.yaml | -| redis | object | `{"architecture":"standalone","auth":{"enabled":false},"databaseNumber":0,"enabled":true,"image":{"repository":"bitnamilegacy/redis"},"master":{"name":"{{ .Release.Name }}-redis-master","persistence":{"accessModes":["ReadWriteOnce"],"enabled":true,"size":"5Gi","storageClass":null},"port":6379}}` | Configuration from https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml | | webserver | object | `{"additionalConfigs":{},"additionalVolumeMounts":{},"additionalVolumes":{},"affinity":{},"containerSecurityContext":{},"deploymentAnnotations":{},"enabled":true,"extraEnvs":[],"image":{"pullPolicy":"IfNotPresent","registry":"","repository":"allegroai/clearml","tag":"2.0.0-613"},"ingress":{"annotations":{},"enabled":false,"hostName":"app.clearml.127-0-0-1.nip.io","ingressClassName":"","path":"/","tlsSecretName":""},"initContainers":{"resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}},"nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"replicaCount":1,"resources":{"limits":{"cpu":"2000m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"service":{"annotations":{},"nodePort":30080,"port":8080,"type":"NodePort"},"serviceAccountAnnotations":{},"serviceAccountName":"clearml","tolerations":[]}` | Web Server configurations | | webserver.additionalConfigs | object | `{}` | Additional specific webserver configurations | | webserver.additionalVolumeMounts | object | `{}` | Specifies where and how the volumes defined in additionalVolumes. | diff --git a/charts/clearml/README.md.gotmpl b/charts/clearml/README.md.gotmpl index e2cd5f00..7e0e11b3 100644 --- a/charts/clearml/README.md.gotmpl +++ b/charts/clearml/README.md.gotmpl @@ -30,6 +30,17 @@ To add this chart to your local Helm repository: helm repo add clearml https://clearml.github.io/clearml-helm-charts ``` +## Kubernetes CRD requirements + +During the initial Helm installation, Custom Resource Definitions (CRDs) are automatically installed. +However, due to Helm's update strategy, these CRDs will not be updated automatically during subsequent `helm upgrade` operations. + +To ensure you have the latest CRD versions when performing an upgrade, please apply the CRDs manually before running `helm upgrade`: + +```bash +kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml +``` + ## Local environment For development/evaluation it's possible to use [kind](https://kind.sigs.k8s.io). @@ -96,6 +107,148 @@ A production ready cluster should also have some different configuration like th helm install clearml clearml/clearml -f values-production.yaml ``` +## Dependencies migration + +Starting chart version `7.15.0`, a migration path for database dependencies is available to help users transition to new bundled dependencies. + +**IMPORTANT**: This procedure is required to prepare for future Helm chart updates that may not be possible if the migration is not completed. The migration ensures data consistency and compatibility with upcoming chart versions. + +For detailed migration instructions, see the [Kubernetes Dependencies Migration Guide](k8s_dependencies_migration.md). + +### Procedure + +#### Phase 1: Enable MCK MongoDB Deployment + +### Prerequisites + +Export your `clearml namespace` (the default is `clearml` but verify your actual namespace as it may have been customized during installation): + +```bash +CLEARML_NAMESPACE=clearml +``` + +Add CRDs following Helm chart README: + +```bash +kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-kubernetes/1.6.1/public/crds.yaml +``` + +Update your Helm chart configuration with the following values to deploy the MCK MongoDB instance, staying on Helm chart version `7.15.0`: + +```yaml +mckMongodb: + enabled: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**Note**: This step creates the new MongoDB instance in parallel with the existing Bitnami deployment, allowing for data migration without service interruption. + +#### Phase 2: Data Migration Process + +**2.1 Scale down the ClearML API server** + +Scale down ClearML services: + +```bash +kubectl -n $CLEARML_NAMESPACE scale deployment -l app.kubernetes.io/name=clearml --replicas=0 +``` + +**2.2 Deploy Migration Pod** + +Create a Kubernetes pod equipped with MongoDB tools for the migration process. Create a file called `mongodb-migrate.yaml`: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: mongodb-migrate +spec: + restartPolicy: Never + containers: + - name: mongo-tools + image: mongo:7 + command: ["sleep", "infinity"] + env: + - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: mongodb-root + key: connectionString +``` + +Deploy and access the migration pod: + +```bash +kubectl -n $CLEARML_NAMESPACE apply -f mongodb-migrate.yaml +``` + +**2.3 Export Data from Source Database** + +Dump data from Bitnami MongoDB: + +```bash +MONGO_CONNECTION_STRING=$(kubectl get deploy -n $CLEARML_NAMESPACE -o jsonpath='{.items[*].spec.template.spec.containers[?(@.name=="clearml-apiserver")].env[?(@.name=="CLEARML_MONGODB_SERVICE_CONNECTION_STRING")].value}{"\n"}') +kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- mongodump --uri="$MONGO_CONNECTION_STRING" --archive=/dump.archive --gzip +``` + +Copy the dump to a local system so there's a further local copy just in case: + +```bash +kubectl -n $CLEARML_NAMESPACE cp mongodb-migrate:/dump.archive /tmp/dump.archive +``` + +**2.4 Import Data to Target Database** + +Restore data to MCK MongoDB: + +```bash +kubectl -n $CLEARML_NAMESPACE exec -it mongodb-migrate -- sh -c 'mongorestore \ + --uri="$CLEARML_MONGODB_SERVICE_CONNECTION_STRING" \ + --archive=/dump.archive \ + --gzip \ + --drop' +``` + +#### Phase 3: Configuration Updates + +**3.1 Mark Migration as Complete** + +Update your Helm configuration to indicate successful data migration, staying on Helm chart version 7.15.0: + +```yaml +mckMongodb: + enabled: true + migrated: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**3.2 Validate System Functionality** + +Thoroughly test ClearML Enterprise functionality to ensure all components are working correctly with the new MongoDB deployment. + +**3.3 Decommission Bitnami MongoDB** + +Once validation is complete, disable the legacy Bitnami MongoDB deployment, staying on Helm chart version 7.15.0: + +```yaml +mongodb: + enabled: false +mckMongodb: + enabled: true + migrated: true +``` + +Upgrade chart accordingly with helm upgrade command. + +**3.4 Remove migration pod** + +Remove the migration pod: + +```bash +kubectl -n $CLEARML_NAMESPACE delete -f mongodb-migrate.yaml + ## Upgrades/ Values upgrades Updating to latest version of this chart can be done in two steps: @@ -111,7 +264,7 @@ Changing values on existing installation can be done with: helm upgrade clearml clearml/clearml --version -f custom_values.yaml ``` -Please note: updating values only should always be done setting explicit chart version to avoid a possible chart update. +Please note: updating values only should always be done setting explicit chart version to avoid a possible chart update. Keeping separate updates procedures between version and values can be a good practice to seprate potential concerns. ### Major upgrade from 5.* to 6.* diff --git a/charts/clearml/charts/dragonfly-v1.36.0.tgz b/charts/clearml/charts/dragonfly-v1.36.0.tgz new file mode 100644 index 00000000..23e949b8 Binary files /dev/null and b/charts/clearml/charts/dragonfly-v1.36.0.tgz differ diff --git a/charts/clearml/charts/mongodb-kubernetes-1.6.1.tgz b/charts/clearml/charts/mongodb-kubernetes-1.6.1.tgz new file mode 100644 index 00000000..b0305b95 Binary files /dev/null and b/charts/clearml/charts/mongodb-kubernetes-1.6.1.tgz differ diff --git a/charts/clearml/charts/redis-17.8.3.tgz b/charts/clearml/charts/redis-17.8.3.tgz deleted file mode 100644 index f5339ee5..00000000 Binary files a/charts/clearml/charts/redis-17.8.3.tgz and /dev/null differ diff --git a/charts/clearml/templates/_helpers.tpl b/charts/clearml/templates/_helpers.tpl index 7afb371d..c4406e5d 100644 --- a/charts/clearml/templates/_helpers.tpl +++ b/charts/clearml/templates/_helpers.tpl @@ -199,7 +199,7 @@ Elasticsearch Service schema {{- end }} {{/* -Elasticsearch Comnnection string +Elasticsearch Connection string */}} {{- define "elasticsearch.connectionstring" -}} {{- if .Values.elasticsearch.enabled }} @@ -210,50 +210,59 @@ Elasticsearch Comnnection string {{- end }} {{/* -MongoDB Comnnection string +MongoDB Connection string */}} {{- define "mongodb.connectionstring" -}} -{{- if eq .Values.mongodb.architecture "standalone" }} -{{- printf "%s%s%s" "mongodb://" .Release.Name "-mongodb:27017" }} -{{- else }} -{{- $connectionString := "mongodb://" }} -{{- range $i,$e := until (.Values.mongodb.replicaCount | int) }} -{{- $connectionString = printf "%s%s%s%s%s%s%s%s%s" $connectionString $.Release.Name "-mongodb-" ( $i | toString ) "." $.Release.Name "-mongodb-headless." $.Release.Namespace ".svc.cluster.local," }} -{{- end }} -{{- printf "%s" ( trimSuffix "," $connectionString ) }} -{{- end }} -{{- end }} +{{- $authPrefix := "" -}} +{{- if .Values.mongodb.auth.enabled -}} + {{- $authPrefix = printf "%s:%s@" .Values.mongodb.auth.rootUser .Values.mongodb.auth.rootPassword -}} +{{- end -}} +{{- if eq .Values.mongodb.architecture "standalone" -}} + {{- printf "mongodb://%s%s-mongodb:27017" $authPrefix .Release.Name -}} +{{- else -}} + {{- $connectionString := printf "mongodb://%s" $authPrefix -}} + {{- range $i, $e := until (.Values.mongodb.replicaCount | int) -}} + {{- $connectionString = printf "%s%s-mongodb-%d.%s-mongodb-headless.%s.svc.cluster.local," $connectionString $.Release.Name $i $.Release.Name $.Release.Namespace -}} + {{- end -}} + {{- $connectionString = printf "%s" (trimSuffix "," $connectionString) -}} + {{- printf "%s/?replicaSet=%s" $connectionString .Values.mongodb.replicaSetName -}} +{{- end -}} +{{- end -}} {{/* MongoDB hostname */}} {{- define "mongodb.hostname" -}} -{{- if eq .Values.mongodb.architecture "standalone" }} -{{- printf "%s" "mongodb" }} -{{- else }} -{{- printf "%s" "mongodb-headless" }} +{{- if .Values.mckMongodb.migrated }} + {{- printf "%s" "mongodb-replica-set-svc" }} +{{- else -}} + {{- if eq .Values.mongodb.architecture "standalone" }} + {{- printf "%s" "mongodb" }} + {{- else }} + {{- printf "%s" "mongodb-headless" }} + {{- end }} {{- end }} {{- end }} {{/* -Redis Service name +Dragonfly Service name */}} -{{- define "redis.servicename" -}} -{{- if .Values.redis.enabled }} -{{- tpl .Values.redis.master.name . }} +{{- define "dragonfly.servicename" -}} +{{- if .Values.dragonfly.enabled }} +{{- include "dragonfly.fullname" .Subcharts.dragonfly }} {{- else }} -{{- .Values.externalServices.redisHost }} +{{- .Values.externalServices.dragonflyHost }} {{- end }} {{- end }} {{/* -Redis Service port +Dragonfly Service port */}} -{{- define "redis.serviceport" -}} -{{- if .Values.redis.enabled }} -{{- .Values.redis.master.port }} +{{- define "dragonfly.serviceport" -}} +{{- if .Values.dragonfly.enabled }} +{{- .Values.dragonfly.service.port }} {{- else }} -{{- .Values.externalServices.redisPort }} +{{- .Values.externalServices.dragonflyPort }} {{- end }} {{- end }} diff --git a/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml b/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml index 0085e3a7..1bc8e9a4 100644 --- a/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml +++ b/charts/clearml/templates/apiserver-asyncdelete-deployment.yaml @@ -59,15 +59,20 @@ spec: sleep 5 ; done ; {{- end }} - {{- if .Values.mongodb.enabled }} + {{- if .Values.mckMongodb.migrated }} + while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://mongodb-replica-set-svc:27017" -o /dev/null; echo $?) -ne 49 ] ; do + echo "waiting for MCK mongodb" ; + sleep 5 ; + done ; + {{- else if .Values.mongodb.enabled }} while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ .Release.Name }}-{{ include "mongodb.hostname" . }}:27017" -o /dev/null; echo $?) -ne 49 ] ; do echo "waiting for mongodb" ; sleep 5 ; done ; {{- end }} - {{- if .Values.redis.enabled }} - while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ include "redis.servicename" . }}:{{ include "redis.serviceport" . }}" -o /dev/null; echo $?) -ne 49 ] ; do - echo "waiting for redis" ; + {{- if .Values.dragonfly.enabled }} + while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ include "dragonfly.servicename" . }}:{{ include "dragonfly.serviceport" . }}" -o /dev/null; echo $?) -ne 49 ] ; do + echo "waiting for dragonfly" ; sleep 5 ; done ; {{- end }} @@ -86,10 +91,16 @@ spec: python3 -m jobs.async_urls_delete --fileserver-host http://{{ include "fileserver.referenceName" . }}:{{ .Values.fileserver.service.port }} env: - name: CLEARML_REDIS_SERVICE_HOST - value: {{ include "redis.servicename" . }} + value: {{ include "dragonfly.servicename" . }} - name: CLEARML_REDIS_SERVICE_PORT - value: "{{ include "redis.serviceport" . }}" - {{- if .Values.mongodb.enabled }} + value: "{{ include "dragonfly.serviceport" . }}" + {{- if .Values.mckMongodb.migrated }} + - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ (index .Values.mckMongodb.community.resource.users 0).passwordSecretRef.name }} + key: connectionString + {{- else if .Values.mongodb.enabled }} - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING value: {{ include "mongodb.connectionstring" . | quote }} {{- else }} diff --git a/charts/clearml/templates/apiserver-deployment.yaml b/charts/clearml/templates/apiserver-deployment.yaml index 29bc0ae1..cc6e03d4 100644 --- a/charts/clearml/templates/apiserver-deployment.yaml +++ b/charts/clearml/templates/apiserver-deployment.yaml @@ -67,15 +67,20 @@ spec: sleep 5 ; done ; {{- end }} - {{- if .Values.mongodb.enabled }} + {{- if .Values.mckMongodb.migrated }} + while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://mongodb-replica-set-svc:27017" -o /dev/null; echo $?) -ne 49 ] ; do + echo "waiting for MCK mongodb" ; + sleep 5 ; + done ; + {{- else if .Values.mongodb.enabled }} while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ .Release.Name }}-{{ include "mongodb.hostname" . }}:27017" -o /dev/null; echo $?) -ne 49 ] ; do echo "waiting for mongodb" ; sleep 5 ; done ; {{- end }} - {{- if .Values.redis.enabled }} - while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ include "redis.servicename" . }}:{{ include "redis.serviceport" . }}" -o /dev/null; echo $?) -ne 49 ] ; do - echo "waiting for redis" ; + {{- if .Values.dragonfly.enabled }} + while [ $(curl --telnet-option BOGUS --connect-timeout 2 -s "telnet://{{ include "dragonfly.servicename" . }}:{{ include "dragonfly.serviceport" . }}" -o /dev/null; echo $?) -ne 49 ] ; do + echo "waiting for dragonfly" ; sleep 5 ; done ; {{- end }} @@ -100,7 +105,13 @@ spec: value: {{ include "elasticsearch.connectionstring" . }} - name: CLEARML__HOSTS__ELASTIC__LOGS__HOSTS value: {{ include "elasticsearch.connectionstring" . }} - {{- if .Values.mongodb.enabled }} + {{- if .Values.mckMongodb.migrated }} + - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ (index .Values.mckMongodb.community.resource.users 0).passwordSecretRef.name }} + key: connectionString + {{- else if .Values.mongodb.enabled }} - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING value: {{ include "mongodb.connectionstring" . | quote }} {{- else }} @@ -110,9 +121,9 @@ spec: value: {{ .Values.externalServices.mongodbConnectionStringAuth | quote }} {{- end }} - name: CLEARML_REDIS_SERVICE_HOST - value: {{ include "redis.servicename" . }} + value: {{ include "dragonfly.servicename" . }} - name: CLEARML_REDIS_SERVICE_PORT - value: "{{ include "redis.serviceport" . }}" + value: "{{ include "dragonfly.serviceport" . }}" - name: CLEARML_CONFIG_PATH value: /opt/clearml/config - name: CLEARML__apiserver__default_company_name diff --git a/charts/clearml/templates/fileserver-deployment.yaml b/charts/clearml/templates/fileserver-deployment.yaml index 68870c49..813f6efb 100644 --- a/charts/clearml/templates/fileserver-deployment.yaml +++ b/charts/clearml/templates/fileserver-deployment.yaml @@ -80,9 +80,9 @@ spec: - name: CLEARML__HOSTS__API_SERVER value: "http://{{ include "apiserver.referenceName" . }}:{{ .Values.apiserver.service.port }}" - name: CLEARML_REDIS_SERVICE_HOST - value: {{ include "redis.servicename" . }} + value: {{ include "dragonfly.servicename" . }} - name: CLEARML_REDIS_SERVICE_PORT - value: "{{ include "redis.serviceport" . }}" + value: "{{ include "dragonfly.serviceport" . }}" {{- if not (eq .Values.clearml.cookieDomain "") }} - name: CLEARML__FILESERVER__AUTH__COOKIE_NAMES value: "[ {{ .Values.clearml.cookieName }} ]" diff --git a/charts/clearml/templates/mongodb-secret.yaml b/charts/clearml/templates/mongodb-secret.yaml new file mode 100644 index 00000000..4d4f0577 --- /dev/null +++ b/charts/clearml/templates/mongodb-secret.yaml @@ -0,0 +1,19 @@ +{{- if and (.Values.mckMongodb.enabled) (.Values.mckMongodb.community.createResource) }} + +{{- $secretName := (index .Values.mckMongodb.community.resource.users 0).passwordSecretRef.name -}} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace $secretName) -}} +{{- $password := "" -}} +{{- if and $existing $existing.data.password -}} +{{- $password = ($existing.data.password | b64dec) -}} +{{- else -}} +{{- $password = (randAlphaNum 16) -}} +{{- end -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ (index .Values.mckMongodb.community.resource.users 0).passwordSecretRef.name }} +type: Opaque +stringData: + password: {{ $password | quote }} + connectionString: {{ printf "%s%s%s%s%s" "mongodb://" (index .Values.mckMongodb.community.resource.users 0).name ":" $password "@mongodb-replica-set-svc" }} +{{- end }} diff --git a/charts/clearml/values-production.yaml b/charts/clearml/values-production.yaml index 0f9d9ae8..28ab3b79 100755 --- a/charts/clearml/values-production.yaml +++ b/charts/clearml/values-production.yaml @@ -16,18 +16,8 @@ webserver: ingress: enabled: true hostName: "app.clearml.127-0-0-1.nip.io" -redis: - architecture: replication - master: - persistence: - enabled: true - accessModes: - - ReadWriteOnce - size: 5Gi - ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner - storageClass: null - replica: - replicaCount: 2 +dragonfly: + replicaCount: 2 mongodb: enabled: true architecture: replicaset diff --git a/charts/clearml/values.yaml b/charts/clearml/values.yaml index 219c528c..335579f5 100644 --- a/charts/clearml/values.yaml +++ b/charts/clearml/values.yaml @@ -382,30 +382,18 @@ externalServices: mongodbConnectionStringAuth: "mongodb://mongodb_hostname:27017/auth" # -- Existing MongoDB connection string for AUTH to use if mongodb.enabled is false (example in values.yaml) mongodbConnectionStringBackend: "mongodb://mongodb_hostnamehostname:27017/backend" - # -- Existing Redis Hostname to use if redis.enabled is false (example in values.yaml) - redisHost: "redis_hostname" - # -- Existing Redis Port to use if redis.enabled is false - redisPort: 6379 + # -- Existing Redis Hostname to use if dragonfly.enabled is false (example in values.yaml) + dragonflyHost: "dragonfly_hostname" + # -- Existing Redis Port to use if dragonfly.enabled is false + dragonflyPort: 6379 -# -- Configuration from https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml -redis: +# -- Configuration from https://github.com/dragonflydb/dragonfly/blob/main/contrib/charts/dragonfly/values.yaml +dragonfly: enabled: true - image: - repository: bitnamilegacy/redis - auth: - enabled: false - databaseNumber: 0 - master: - name: "{{ .Release.Name }}-redis-master" - port: 6379 - persistence: - enabled: true - accessModes: - - ReadWriteOnce - size: 5Gi - ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner - storageClass: null - architecture: standalone + replicaCount: 1 + storage: + enabled: true + requests: 5Gi # -- Configuration from https://github.com/bitnami/charts/blob/master/bitnami/mongodb/values.yaml mongodb: @@ -429,6 +417,50 @@ mongodb: ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner storageClass: null +# -- Configuration from https://github.com/mongodb/mongodb-kubernetes/blob/1.6.1/helm_chart/values.yaml +mckMongodb: + enabled: false + ## set to true ONLY when data is migrated from mongodb to mckMongoDB + migrated: false + community: + createResource: true + resource: + name: mongodb-replica-set + version: 7.0.28 + members: 1 + users: + # At least ONE user MUST be defined. + # This user is the one ClearML will use to connect to MongoDB. + - name: root + db: admin + passwordSecretRef: + # Name of an existing Kubernetes Secret containing the credentials. + # + # IMPORTANT: + # - This secret MUST be created BEFORE enabling mckMongodb. + # - Use this when you need a custom or pre-generated password. + # + # The Secret MUST have the following structure: + # + # stringData: + # connectionString: "mongodb://:@mongodb-replica-set-svc" + # password: "" + # + # Notes: + # - must match the value of `name` above. + # - must be identical in both fields. + name: mongodb-root + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + - name: readWriteAnyDatabase + db: admin + - name: dbAdminAnyDatabase + db: admin + scramCredentialsSecretName: clearml-scram + # -- Configuration from https://github.com/elastic/helm-charts/blob/7.16/elasticsearch/values.yaml elasticsearch: enabled: true