Make object store port/secure scheme optional.

For cloud storage like aws s3, the secure port is 443, but this does not
work with the minio module that is used by dsp api server, so instead in
such a case we make port optional, and do not require it. When an empty
port is provided, then it is simply not used by dsp api server.

Test cases are provided to reflect this change.

Docs have also been provided on how to utilize this external object
storage with an aws s3 specific example usecase.

Signed-off-by: Humair Khan <[email protected]>

Author: HumairAK

README.md

@@ -9,8 +9,10 @@ Data Science Pipeline stacks onto individual OCP namespaces.
    1. [Pre-requisites](#pre-requisites)
    2. [Deploy the Operator via ODH](#deploy-the-operator-via-odh)
    3. [Deploy the Operator standalone](#deploy-the-operator-standalone)
-   4. [Deploy DSP instance](#deploy-dsp-instance)
-      1. [Deploy another DSP instance](#deploy-another-dsp-instance)
+   4. [Deploy DSPA instance](#deploy-dsp-instance)
+      1. [Deploy another DSPA instance](#deploy-another-dsp-instance)
+      2. [Deploy a DSPA with custom credentials](#deploy-a-dsp-with-custom-credentials)
+      3. [Deploy a DSPA with External Object Storage](#deploy-a-dsp-with-external-object-storage)
 2. [DataSciencePipelinesApplication Component Overview](#datasciencepipelinesapplication-component-overview)
 3. [Using a DataSciencePipelinesApplication](#using-a-datasciencepipelinesapplication)
    1. [Using the Graphical UI](#using-the-graphical-ui)
@@ -171,6 +173,22 @@ Notice the introduction of 2 `secrets` `testdbsecret`, `teststoragesecret` and 2
 
 These can be configured by the end user as needed.
 
+### Deploy a DSP with external Object Storage
+
+To specify a custom Object Storage (example an AWS s3 bucket) you will need to provide DSPO with your S3 credentials in 
+the form of a k8s `Secret`, see an example of such a secret here `config/samples/external-object-storage/storage-creds.yaml`.
+
+DSPO can deploy a DSPA instance and use this S3 bucket for storing its metadata and pipeline artifacts. A sample 
+configuration for a DSPA that does this is found in `config/samples/external-object-storage`, you can update this as 
+needed, and deploy this DSPA by running the following:
+
+```bash
+DSP_Namespace_3=test-ds-project-4
+oc new-project ${DSP_Namespace_4}
+cd ${WORKING_DIR}/config/samples/external-object-storage
+kustomize build . | oc -n ${DSP_Namespace_3} apply -f -
+```
+
 # DataSciencePipelinesApplication Component Overview
 
 When a `DataSciencePipelinesApplication` is deployed, the following components are deployed in the target namespace: 

api/v1alpha1/dspipeline_types.go

@@ -194,8 +194,12 @@ type ExternalStorage struct {
 	Host                string `json:"host"`
 	Bucket              string `json:"bucket"`
 	Scheme              string `json:"scheme"`
-	Port                string `json:"port"`
 	*S3CredentialSecret `json:"s3CredentialsSecret"`
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	Secure bool `json:"secure"`
+	// +kubebuilder:validation:Optional
+	Port string `json:"port"`
 }
 
 type S3CredentialSecret struct {

config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml

@@ -311,10 +311,12 @@ spec:
                         type: object
                       scheme:
                         type: string
+                      secure:
+                        default: true
+                        type: boolean
                     required:
                     - bucket
                     - host
-                    - port
                     - s3CredentialsSecret
                     - scheme
                     type: object

config/samples/external-object-storage/dspa_simple.yaml

@@ -0,0 +1,18 @@
+apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1
+kind: DataSciencePipelinesApplication
+metadata:
+  name: sample
+spec:
+  objectStorage:
+    externalStorage:
+      bucket: rhods-dsp-dev
+      host: s3.amazonaws.com
+      s3CredentialsSecret:
+        accessKey: k8saccesskey
+        secretKey: k8ssecretkey
+        secretName: aws-bucket-creds
+      scheme: https
+  # Optional
+  mlpipelineUI:
+    # Image field is required
+    image: 'quay.io/opendatahub/odh-ml-pipelines-frontend-container:beta-ui'

config/samples/external-object-storage/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - dspa.yaml
+  - storage-creds.yaml

config/samples/external-object-storage/storage-creds.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-bucket-creds
+  labels:
+    opendatahub.io/dashboard: 'true'
+    opendatahub.io/managed: 'true'
+  annotations:
+    opendatahub.io/connection-type: s3
+    openshift.io/display-name: AWS S3 Connection
+stringData:
+  k8saccesskey: someaccesskey
+  k8ssecretkey: somesecretkey
+type: Opaque

controllers/config/defaults.go

@@ -45,10 +45,9 @@ const (
 	MinioDefaultBucket = "mlpipeline"
 	MinioPVCSize       = "10Gi"
 
-	ObjectStoreConnectionSecure = false
-	ObjectStorageSecretName     = "mlpipeline-minio-artifact" // hardcoded in kfp-tekton
-	ObjectStorageAccessKey      = "accesskey"
-	ObjectStorageSecretKey      = "secretkey"
+	ObjectStorageSecretName = "mlpipeline-minio-artifact" // hardcoded in kfp-tekton
+	ObjectStorageAccessKey  = "accesskey"
+	ObjectStorageSecretKey  = "secretkey"
 )
 
 // DSPO Config File Paths

controllers/dspipeline_params.go

@@ -221,14 +221,15 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 		AccessKey:  config.ObjectStorageAccessKey,
 		SecretKey:  config.ObjectStorageSecretKey,
 	}
-	p.ObjectStorageConnection.Secure = config.ObjectStoreConnectionSecure
 
 	if usingExternalObjectStorage {
 		// Assume validation for CR ensures these values exist
 		p.ObjectStorageConnection.Bucket = dsp.Spec.ObjectStorage.ExternalStorage.Bucket
 		p.ObjectStorageConnection.Host = dsp.Spec.ObjectStorage.ExternalStorage.Host
-		p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port
 		p.ObjectStorageConnection.Scheme = dsp.Spec.ObjectStorage.ExternalStorage.Scheme
+		p.ObjectStorageConnection.Secure = dsp.Spec.ObjectStorage.ExternalStorage.Secure
+		// Port can be empty, which is fine.
+		p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port
 		customCreds = dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret
 	} else {
 		if p.Minio == nil {
@@ -262,15 +263,22 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 	}
 
 	endpoint := fmt.Sprintf(
-		"%s://%s:%s",
+		"%s://%s",
 		p.ObjectStorageConnection.Scheme,
 		p.ObjectStorageConnection.Host,
-		p.ObjectStorageConnection.Port,
 	)
 
+	if p.ObjectStorageConnection.Port != "" {
+		endpoint = fmt.Sprintf(
+			"%s:%s",
+			endpoint,
+			p.ObjectStorageConnection.Port,
+		)
+	}
+
 	p.ObjectStorageConnection.Endpoint = endpoint
 
-	// Secret where DB credentials reside on cluster
+	// Secret where credentials reside on cluster
 	var credsSecretName string
 	var credsAccessKey string
 	var credsSecretKey string

controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml

@@ -129,7 +129,7 @@ spec:
                   key: "secretkey"
                   name: "mlpipeline-minio-artifact"
             - name: OBJECTSTORECONFIG_SECURE
-              value: "false"
+              value: "true"
             - name: MINIO_SERVICE_SERVICE_HOST
               value: "teststoragehost3"
             - name: MINIO_SERVICE_SERVICE_PORT

controllers/testdata/declarative/case_3/expected/created/storage_secret.yaml

@@ -11,5 +11,5 @@ data:
   host: dGVzdHN0b3JhZ2Vob3N0Mw==
   port: ODA=
   secretkey: dGVzdHNlY3JldGtleXZhbHVlMw==
-  secure: ZmFsc2U=
+  secure: dHJ1ZQ==
 type: Opaque