update to support kfp v1.0.0

Author: lzuwei

pipelines/base/aws-configurations-patch.yaml

@@ -5,7 +5,7 @@ metadata:
 data:
   config: |
     {
-    namespace: $(NAMESPACE),
+    namespace: $(namespace),
     artifactRepository:
     {
         archiveLogs: true,
@@ -62,10 +62,17 @@ spec:
       containers:
         - name: ml-pipeline-api-server
           env:
-            - name: OBJECTSTORECONFIG_SECURE
-              value: "true"
-            - name: OBJECTSTORECONFIG_BUCKETNAME
-              value: $(pipelineBucket)
+            # we do not use minio related access key options, delete these
+            # env vars as it interferes with config.json settings due to viper config
+            # management taking precedence of env vars over config.json
+            # currently, minio client will init a static client if access keys and secret
+            # are not empty.
+            - $patch: delete
+              name: OBJECTSTORECONFIG_ACCESSKEY
+            - $patch: delete
+              name: OBJECTSTORECONFIG_SECRETACCESSKEY
+            - $patch: delete
+              name: OBJECTSTORECONFIG_SECURE
             - name: DBCONFIG_USER
               valueFrom:
                 secretKeyRef:

pipelines/base/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 bases:
-  - github.com/kubeflow/pipelines//manifests/kustomize/base?ref=0.5.1
+  - github.com/kubeflow/pipelines/manifests/kustomize/base?ref=1.0.0
 
 resources:
   # use a custom podTemplate definition for tensorboard viewer
@@ -20,7 +20,7 @@ patchesStrategicMerge:
 
 configMapGenerator:
   # Note: pipeline-install-config is used by official kubeflow for installation parameters.
-  # See: https://github.com/kubeflow/pipelines/blob/0.5.1/manifests/kustomize/base/kustomization.yaml
+  # See: https://github.com/kubeflow/pipelines/blob/1.0.0/manifests/kustomize/base/kustomization.yaml
   # Create your own if you want to overwrite those parameters
   - name: pipeline-aws-parameters
     env: params.env    # All parameters can be set inside this file
@@ -111,6 +111,41 @@ vars:
       apiVersion: v1
     fieldref:
       fieldpath: data.groupConcatMaxLen
+  - name: dbHost
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.dbHost
+  - name: dbPort
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.dbPort
+  - name: mlmdDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.mlmdDb
+  - name: cacheDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.cacheDb
+  - name: pipelineDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.pipelineDb
 
 configurations:
   - params.yaml

pipelines/base/manifest/ml-pipeline-apiserver-configmap.yaml

@@ -8,20 +8,22 @@ data:
   config.json: |
     {
       "DBConfig": {
-        "Host": "$(DBSERVICE_HOST)",
+        "Host": "$(dbHost)",
         "Port": "3306",
         "DriverName": "mysql",
         "DataSourceName": "",
-        "DBName": "$(DBNAME_PIPELINE)",
+        "DBName": "$(pipelineDb)",
         "GroupConcatMaxLen": "$(groupConcatMaxLen)"
       },
       "ObjectStoreConfig": {
         "Host": "s3.amazonaws.com",
         "Region": "$(awsRegion)",
         "Secure": true,
-        "BucketName": "$(pipelineBucket)",
+        "BucketName": "$(artifactRepositoryBucket)",
         "PipelineFolder": "$(pipelinePath)",
-        "PipelinePath": "$(pipelinePath)"
+        "PipelinePath": "$(pipelinePath)",
+        "AccessKey": "",
+        "SecretAccessKey": ""
       },
       "InitConnectionTimeout": "6m",
       "DefaultPipelineRunnerServiceAccount": "pipeline-runner"

pipelines/base/manifest/mysql.yaml

@@ -21,7 +21,7 @@ spec:
         - name: mysql
           env:
             - name: MYSQL_DATABASES
-              value: "$(DBNAME_PIPELINE),$(DBNAME_MLMD),$(DBNAME_CACHE)"
+              value: "$(pipelineDb),$(mlmdDb),$(cacheDb)"
             - name: MYSQL_ROOT_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -64,7 +64,7 @@ kind: Service
 metadata:
   labels:
     app: mysql
-  name: $(DBSERVICE_HOST)
+  name: $(dbHost)
 spec:
   selector:
     app: mysql

pipelines/base/params.env

@@ -5,6 +5,5 @@ mysqlStorageSize=20Gi
 awsRegion=ap-southeast-1
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines
 groupConcatMaxLen=4194304

pipelines/base/params.yaml

@@ -27,3 +27,5 @@ varReference:
   kind: StorageClass
 - path: metadata/name
   kind: Service
+- path: spec/ports/port
+  kind: Service

pipelines/cluster-scoped-resources/kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 bases:
-  - github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=0.5.1
+  - github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=1.0.0

pipelines/overlay/accesskey/params.env

@@ -8,5 +8,4 @@ awsAccessKeySecretKey=accesskey
 awsSecretKeySecretKey=secretkey
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines

pipelines/overlay/iam/params.env

@@ -7,5 +7,4 @@ awsIAMAnnotationKey=iam.amazonaws.com/role
 awsIAMRole=kube2iam-role
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines