Merge pull request opendatahub-io#22 from rimolive/rmartine

Parametrize ML Pipelines

Author: openshift-merge-robot

manifests/opendatahub/README.md

@@ -46,6 +46,15 @@ This directory contains artifacts for deploying all backend components of ML Pip
 
 This directory contains the service monitor definition for ML Pipelines. It is always deployed by base, so this will eventually be moved into the base directory itself.
 
+## Parameters
+
+You can customize the ML Pipelines deployment by injecting custom parameters to change the default deployment. The following parameters can be used:
+
+* **pipeline_install_configuration**: The ConfigMap name that contains the values to install the ML Pipelines environment. This parameter defaults to `pipeline-install-config` and you can find an example in the [repository](./base/configmaps/pipeline-install-config.yaml).
+* **ml_pipelines_configuration**: The ConfigMap name that contains the values to integrate ML Pipelines with the underlying components (Database and Object Store). This parameter defaults to `kfp-tekton-config` and you can find an example in the [repository](./base/configmaps/kfp-tekton-config.yaml).
+* **database_secret**: The secret that contains the credentials for the ML Pipelines Databse. It defaults to `mysql-secret` if using the `metadata-store-mysql` overlay or `postgresql-secret` if using the `metadata-store-postgresql` overlay.
+* **ml_pipelines_ui_configuration**: The ConfigMap that contains the values to customize UI. It defaults to `ml-pipeline-ui-configmap`.
+
 ## Configuration
 
 * It is possible to configure what S3 storage is being used by Pipeline Runs. Detailed instructions on how to configure this will be added once Minio is moved to an overlay.

manifests/opendatahub/base/deployments/metadata-grpc-deployment.yaml

@@ -33,27 +33,27 @@ spec:
               valueFrom:
                 secretKeyRef:
                   key: username
-                  name: mysql-secret
+                  name: $(database_secret)
             - name: DBCONFIG_PASSWORD
               valueFrom:
                 secretKeyRef:
                   key: password
-                  name: mysql-secret
+                  name: $(database_secret)
             - name: MYSQL_DATABASE
               valueFrom:
                 configMapKeyRef:
                   key: mlmdDb
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: MYSQL_HOST
               valueFrom:
                 configMapKeyRef:
                   key: dbHost
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: MYSQL_PORT
               valueFrom:
                 configMapKeyRef:
                   key: dbPort
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
           image: metadata-grpc
           livenessProbe:
             initialDelaySeconds: 3

manifests/opendatahub/base/deployments/metadata-writer.yaml

@@ -29,7 +29,7 @@ spec:
               valueFrom:
                 configMapKeyRef:
                   key: archive_logs
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
           image: metadata-writer
           name: main
       serviceAccountName: kubeflow-pipelines-metadata-writer

manifests/opendatahub/base/deployments/ml-pipeline-scheduledworkflow.yaml

@@ -28,7 +28,7 @@ spec:
               valueFrom:
                 configMapKeyRef:
                   key: cronScheduleTimezone
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
           image: scheduledworkflow
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-scheduledworkflow

manifests/opendatahub/base/deployments/ml-pipeline.yaml

@@ -31,94 +31,94 @@ spec:
               valueFrom:
                 secretKeyRef:
                   key: username
-                  name: mysql-secret
+                  name: $(database_secret)
             - name: DBCONFIG_PASSWORD
               valueFrom:
                 secretKeyRef:
                   key: password
-                  name: mysql-secret
+                  name: $(database_secret)
             - name: DBCONFIG_DBNAME
               valueFrom:
                 configMapKeyRef:
                   key: pipelineDb
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: DBCONFIG_HOST
               valueFrom:
                 configMapKeyRef:
                   key: dbHost
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: DBCONFIG_PORT
               valueFrom:
                 configMapKeyRef:
                   key: dbPort
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: PIPELINE_RUNTIME
               value: tekton
             - name: ARTIFACT_BUCKET
               valueFrom:
                 configMapKeyRef:
                   key: artifact_bucket
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: ARTIFACT_ENDPOINT
               valueFrom:
                 configMapKeyRef:
                   key: artifact_endpoint
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: ARTIFACT_ENDPOINT_SCHEME
               valueFrom:
                 configMapKeyRef:
                   key: artifact_endpoint_scheme
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: ARCHIVE_LOGS
               valueFrom:
                 configMapKeyRef:
                   key: archive_logs
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: TRACK_ARTIFACTS
               valueFrom:
                 configMapKeyRef:
                   key: track_artifacts
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: STRIP_EOF
               valueFrom:
                 configMapKeyRef:
                   key: strip_eof
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: ARTIFACT_SCRIPT
               valueFrom:
                 configMapKeyRef:
                   key: artifact_script
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: ARTIFACT_IMAGE
               valueFrom:
                 configMapKeyRef:
                   key: artifact_image
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: INJECT_DEFAULT_SCRIPT
               valueFrom:
                 configMapKeyRef:
                   key: inject_default_script
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: APPLY_TEKTON_CUSTOM_RESOURCE
               valueFrom:
                 configMapKeyRef:
                   key: apply_tekton_custom_resource
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: TERMINATE_STATUS
               valueFrom:
                 configMapKeyRef:
                   key: terminate_status
-                  name: kfp-tekton-config
+                  name: $(ml_pipelines_configuration)
             - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
               valueFrom:
                 configMapKeyRef:
                   key: autoUpdatePipelineDefaultVersion
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
             - name: DBCONFIG_CONMAXLIFETIMESEC
               valueFrom:
                 configMapKeyRef:
                   key: ConMaxLifeTimeSec
-                  name: pipeline-install-config
+                  name: $(pipeline_install_configuration)
           image: api-server
           imagePullPolicy: Always
           livenessProbe:

manifests/opendatahub/base/kustomization.yaml

@@ -73,6 +73,34 @@ vars:
       apiVersion: v1
     fieldref:
       fieldpath: data.artifact_secret_name
+  - name: pipeline_install_configuration
+    objref:
+      name: kfp-tekton-params-config
+      kind: ConfigMap
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.pipeline_install_configuration
+  - name: ml_pipelines_configuration
+    objref:
+      name: kfp-tekton-params-config
+      kind: ConfigMap
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.ml_pipelines_configuration
+  - name: database_secret
+    objref:
+      name: kfp-tekton-params-config
+      kind: ConfigMap
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.database_secret
+  - name: ml_pipelines_ui_configuration
+    objref:
+      name: kfp-tekton-params-config
+      kind: ConfigMap
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.ml_pipelines_ui_configuration
 configurations:
   - params.yaml
 

manifests/opendatahub/base/params.env

@@ -1 +1,5 @@
 artifact_secret_name=mlpipeline-minio-artifact
+pipeline_install_configuration=pipeline-install-config
+ml_pipelines_configuration=kfp-tekton-config
+database_secret=mysql-secret
+ml_pipelines_ui_configuration=ml-pipeline-ui-configmap

manifests/opendatahub/base/params.yaml

@@ -4,3 +4,7 @@ varReference:
     kind: Deployment
   - path: spec/template/spec/containers[]/envFrom/secretRef/name
     kind: Deployment
+  - path: spec/template/spec/containers[]/env[]/valueFrom/configMapKeyRef/name
+    kind: Deployment
+  - path: spec/template/spec/volumes[]/configMap/name
+    kind: Deployment

manifests/opendatahub/overlays/metadata-store-mysql/kustomization.yaml

@@ -2,6 +2,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+bases:
+  - ../../base
+
 resources:
   # ServiceAccounts
   - ./serviceaccounts/mysql.yaml

manifests/opendatahub/overlays/metadata-store-postgresql/deployments/postgresql.yaml

@@ -26,12 +26,12 @@ spec:
               valueFrom:
                 secretKeyRef:
                   key: username
-                  name: postgresql-secret
+                  name: $(database_secret)
             - name: POSTGRESQL_PASSWORD
               valueFrom:
                 secretKeyRef:
                   key: password
-                  name: postgresql-secret
+                  name: $(database_secret)
             - name: POSTGRESQL_DATABASE
               value: kfp-tekton
           imagePullPolicy: Always