Merge pull request opendatahub-io#11 from kubeflow/master

[pull] master from kubeflow:master

Author: rimolive

Makefile

@@ -16,7 +16,7 @@
 #  - The help target was derived from https://stackoverflow.com/a/35730328/5601796
 
 VENV ?= .venv
-KFP_TEKTON_RELEASE ?= v1.2.1
+KFP_TEKTON_RELEASE ?= v1.3.0
 export VIRTUAL_ENV := $(abspath ${VENV})
 export PATH := ${VIRTUAL_ENV}/bin:${PATH}
 DOCKER_REGISTRY ?= aipipeline

VERSION

@@ -1 +1 @@
-1.2.1
+1.3.0

backend/src/apiserver/client/minio.go

@@ -57,7 +57,7 @@ func CreateMinioClient(minioServiceHost string, minioServicePort string,
 	return minioClient, nil
 }
 
-func CreateMinioClientOrFatal(minioServiceHost string, minioServicePort string,
+func CreateObjectStoreClientOrFatal(minioServiceHost string, minioServicePort string,
 	accessKey string, secretKey string, secure bool, region string, initConnectionTimeout time.Duration) *minio.Client {
 	var minioClient *minio.Client
 	var err error

backend/src/apiserver/client_manager.go

@@ -35,22 +35,22 @@ import (
 )
 
 const (
-	minioServiceHost       = "MINIO_SERVICE_SERVICE_HOST"
-	minioServicePort       = "MINIO_SERVICE_SERVICE_PORT"
-	minioServiceRegion     = "MINIO_SERVICE_REGION"
-	minioServiceSecure     = "MINIO_SERVICE_SECURE"
-	pipelineBucketName     = "MINIO_PIPELINE_BUCKET_NAME"
-	pipelinePath           = "MINIO_PIPELINE_PATH"
-	mysqlServiceHost       = "DBConfig.Host"
-	mysqlServicePort       = "DBConfig.Port"
-	mysqlUser              = "DBConfig.User"
-	mysqlPassword          = "DBConfig.Password"
-	mysqlDBName            = "DBConfig.DBName"
-	mysqlGroupConcatMaxLen = "DBConfig.GroupConcatMaxLen"
-	mysqlExtraParams       = "DBConfig.ExtraParams"
-	archiveLogFileName     = "ARCHIVE_LOG_FILE_NAME"
-	archiveLogPathPrefix   = "ARCHIVE_LOG_PATH_PREFIX"
-	dbConMaxLifeTime       = "DBConfig.ConMaxLifeTime"
+	objectStoreServiceHost   = "MINIO_SERVICE_SERVICE_HOST"
+	objectStoreServicePort   = "MINIO_SERVICE_SERVICE_PORT"
+	objectStoreServiceRegion = "MINIO_SERVICE_REGION"
+	objectStoreServiceSecure = "MINIO_SERVICE_SECURE"
+	pipelineBucketName       = "MINIO_PIPELINE_BUCKET_NAME"
+	pipelinePath             = "MINIO_PIPELINE_PATH"
+	mysqlServiceHost         = "DBConfig.Host"
+	mysqlServicePort         = "DBConfig.Port"
+	mysqlUser                = "DBConfig.User"
+	mysqlPassword            = "DBConfig.Password"
+	mysqlDBName              = "DBConfig.DBName"
+	mysqlGroupConcatMaxLen   = "DBConfig.GroupConcatMaxLen"
+	mysqlExtraParams         = "DBConfig.ExtraParams"
+	archiveLogFileName       = "ARCHIVE_LOG_FILE_NAME"
+	archiveLogPathPrefix     = "ARCHIVE_LOG_PATH_PREFIX"
+	dbConMaxLifeTime         = "DBConfig.ConMaxLifeTime"
 
 	visualizationServiceHost = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST"
 	visualizationServicePort = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT"
@@ -175,7 +175,7 @@ func (c *ClientManager) init() {
 	c.resourceReferenceStore = storage.NewResourceReferenceStore(db)
 	c.dBStatusStore = storage.NewDBStatusStore(db)
 	c.defaultExperimentStore = storage.NewDefaultExperimentStore(db)
-	c.objectStore = initMinioClient(common.GetDurationConfig(initConnectionTimeout))
+	c.objectStore = initObjectStoreClient(common.GetDurationConfig(initConnectionTimeout))
 
 	// Use default value of client QPS (5) & burst (10) defined in
 	// k8s.io/client-go/rest/config.go#RESTClientFor
@@ -381,32 +381,32 @@ func initMysql(driverName string, initConnectionTimeout time.Duration) string {
 	return mysqlConfig.FormatDSN()
 }
 
-func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInterface {
-	// Create minio client.
-	minioServiceHost := common.GetStringConfigWithDefault(
-		"ObjectStoreConfig.Host", os.Getenv(minioServiceHost))
-	minioServicePort := common.GetStringConfigWithDefault(
-		"ObjectStoreConfig.Port", os.Getenv(minioServicePort))
-	minioServiceRegion := common.GetStringConfigWithDefault(
-		"ObjectStoreConfig.Region", os.Getenv(minioServiceRegion))
-	minioServiceSecure := common.GetBoolConfigWithDefault(
-		"ObjectStoreConfig.Secure", common.GetBoolFromStringWithDefault(os.Getenv(minioServiceSecure), false))
+func initObjectStoreClient(initConnectionTimeout time.Duration) storage.ObjectStoreInterface {
+	// Create client.
+	objectStoreServiceHost := common.GetStringConfigWithDefault(
+		"ObjectStoreConfig.Host", os.Getenv(objectStoreServiceHost))
+	objectStoreServicePort := common.GetStringConfigWithDefault(
+		"ObjectStoreConfig.Port", os.Getenv(objectStoreServicePort))
+	objectStoreServiceRegion := common.GetStringConfigWithDefault(
+		"ObjectStoreConfig.Region", os.Getenv(objectStoreServiceRegion))
+	objectStoreServiceSecure := common.GetBoolConfigWithDefault(
+		"ObjectStoreConfig.Secure", common.GetBoolFromStringWithDefault(os.Getenv(objectStoreServiceSecure), false))
 	accessKey := common.GetStringConfigWithDefault("ObjectStoreConfig.AccessKey", "")
 	secretKey := common.GetStringConfigWithDefault("ObjectStoreConfig.SecretAccessKey", "")
 	bucketName := common.GetStringConfigWithDefault("ObjectStoreConfig.BucketName", os.Getenv(pipelineBucketName))
 	pipelinePath := common.GetStringConfigWithDefault("ObjectStoreConfig.PipelinePath", os.Getenv(pipelinePath))
 	disableMultipart := common.GetBoolConfigWithDefault("ObjectStoreConfig.Multipart.Disable", true)
 
-	minioClient := client.CreateMinioClientOrFatal(minioServiceHost, minioServicePort, accessKey,
-		secretKey, minioServiceSecure, minioServiceRegion, initConnectionTimeout)
-	createMinioBucket(minioClient, bucketName, minioServiceRegion)
+	client := client.CreateObjectStoreClientOrFatal(objectStoreServiceHost, objectStoreServicePort, accessKey,
+		secretKey, objectStoreServiceSecure, objectStoreServiceRegion, initConnectionTimeout)
+	createBucket(client, bucketName, objectStoreServiceRegion)
 
-	return storage.NewMinioObjectStore(&storage.MinioClient{Client: minioClient}, bucketName, pipelinePath, disableMultipart)
+	return storage.NewMinioObjectStore(&storage.MinioClient{Client: client}, bucketName, pipelinePath, disableMultipart)
 }
 
-func createMinioBucket(minioClient *minio.Client, bucketName, region string) {
+func createBucket(client *minio.Client, bucketName, region string) {
 	// Check to see if we already own this bucket.
-	exists, err := minioClient.BucketExists(bucketName)
+	exists, err := client.BucketExists(bucketName)
 	if err != nil {
 		glog.Fatalf("Failed to check if Minio bucket exists. Error: %v", err)
 	}
@@ -415,7 +415,7 @@ func createMinioBucket(minioClient *minio.Client, bucketName, region string) {
 		return
 	}
 	// Create bucket if it does not exist
-	err = minioClient.MakeBucket(bucketName, region)
+	err = client.MakeBucket(bucketName, region)
 	if err != nil {
 		glog.Fatalf("Failed to create Minio bucket. Error: %v", err)
 	}

backend/src/apiserver/common/config.go

@@ -46,6 +46,8 @@ const (
 	ApplyTektonCustomResource               string = "APPLY_TEKTON_CUSTOM_RESOURCE"
 	TerminateStatus                         string = "TERMINATE_STATUS"
 	Path4InternalResults                    string = "PATH_FOR_INTERNAL_RESULTS"
+	ObjectStoreAccessKey                    string = "OBJECTSTORECONFIG_ACCESSKEY"
+	ObjectStoreSecretKey                    string = "OBJECTSTORECONFIG_SECRETKEY"
 )
 
 func IsPipelineVersionUpdatedByDefault() bool {

backend/src/cache/server/mutation.go

@@ -297,7 +297,7 @@ func mutateContainer(results map[string][]*tektonv1beta1.TaskRunResult, original
 	args := []string{}
 	args = append(args, "printf 'This step output is taken from cache.\n\n'")
 	for _, result := range outputs {
-		arg := fmt.Sprintf("printf '%s: %s\n'; printf '%s' > /tekton/results/%s", result.Name, result.Value, result.Value, result.Name)
+		arg := fmt.Sprintf("printf '%s: %s\n'; printf '%s' > /tekton/results/%s", result.Name, result.Value.StringVal, result.Value.StringVal, result.Name)
 		args = append(args, arg)
 	}
 
@@ -335,7 +335,6 @@ func unmarshalResult(taskResult string) (map[string][]*tektonv1beta1.TaskRunResu
 	if err != nil {
 		return nil, err
 	}
-
 	return results, nil
 }
 

guides/kfp_tekton_install.md

@@ -53,15 +53,15 @@ To install the standalone Kubeflow Pipelines with Tekton, run the following step
          -p '{"data":{"enable-custom-tasks": "true"}}'
    ```
 
-3. Install Kubeflow Pipelines with Tekton backend (`kfp-tekton`) `v1.2.1` [custom resource definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)(CRDs).
+3. Install Kubeflow Pipelines with Tekton backend (`kfp-tekton`) `v1.3.0` [custom resource definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)(CRDs).
    > Note: You can ignore the error `no matches for kind "Application" in version "app.k8s.io/v1beta1"` since it's a warning saying `application` CRD is not yet ready.
     ```shell
-    kubectl apply --selector kubeflow/crd-install=true -f install/v1.2.1/kfp-tekton.yaml
+    kubectl apply --selector kubeflow/crd-install=true -f install/v1.3.0/kfp-tekton.yaml
     ```
 
-4. Install Kubeflow Pipelines with Tekton backend (`kfp-tekton`) `v1.2.1` deployment
+4. Install Kubeflow Pipelines with Tekton backend (`kfp-tekton`) `v1.3.0` deployment
     ```shell
-    kubectl apply -f install/v1.2.1/kfp-tekton.yaml
+    kubectl apply -f install/v1.3.0/kfp-tekton.yaml
     ```
 
 5. Then, if you want to expose the Kubeflow Pipelines endpoint outside the cluster, run the following commands:
@@ -91,7 +91,7 @@ To install the standalone Kubeflow Pipelines with Tekton, run the following step
 
 1. Follow the [Kubeflow install instructions](https://www.kubeflow.org/docs/ibm/deploy/install-kubeflow-on-iks/#kubeflow-installation)
    to install the entire Kubeflow stack with `kfp-tekton`.
-   Kubeflow `v1.5.0` uses Tekton `v0.30.0` and `kfp-tekton` `v1.1.1`. <!-- TODO update-->
+   Kubeflow `v1.6.0` uses Tekton `v0.31.4` and `kfp-tekton` `v1.2.1`. <!-- TODO update-->
 
 2. Visit [KFP Tekton User Guide](/guides/kfp-user-guide) and start learning how to use Kubeflow pipeline.