Merge pull request opendatahub-io#108 from HumairAK/optional-port

Make object store port/secure scheme optional.

Author: openshift-merge-robot

README.md

@@ -9,8 +9,10 @@ Data Science Pipeline stacks onto individual OCP namespaces.
    1. [Pre-requisites](#pre-requisites)
    2. [Deploy the Operator via ODH](#deploy-the-operator-via-odh)
    3. [Deploy the Operator standalone](#deploy-the-operator-standalone)
-   4. [Deploy DSP instance](#deploy-dsp-instance)
-      1. [Deploy another DSP instance](#deploy-another-dsp-instance)
+   4. [Deploy DSPA instance](#deploy-dsp-instance)
+      1. [Deploy another DSPA instance](#deploy-another-dsp-instance)
+      2. [Deploy a DSPA with custom credentials](#deploy-a-dsp-with-custom-credentials)
+      3. [Deploy a DSPA with External Object Storage](#deploy-a-dsp-with-external-object-storage)
 2. [DataSciencePipelinesApplication Component Overview](#datasciencepipelinesapplication-component-overview)
 3. [Using a DataSciencePipelinesApplication](#using-a-datasciencepipelinesapplication)
    1. [Using the Graphical UI](#using-the-graphical-ui)
@@ -171,6 +173,22 @@ Notice the introduction of 2 `secrets` `testdbsecret`, `teststoragesecret` and 2
 
 These can be configured by the end user as needed.
 
+### Deploy a DSP with external Object Storage
+
+To specify a custom Object Storage (example an AWS s3 bucket) you will need to provide DSPO with your S3 credentials in 
+the form of a k8s `Secret`, see an example of such a secret here `config/samples/external-object-storage/storage-creds.yaml`.
+
+DSPO can deploy a DSPA instance and use this S3 bucket for storing its metadata and pipeline artifacts. A sample 
+configuration for a DSPA that does this is found in `config/samples/external-object-storage`, you can update this as 
+needed, and deploy this DSPA by running the following:
+
+```bash
+DSP_Namespace_3=test-ds-project-4
+oc new-project ${DSP_Namespace_4}
+cd ${WORKING_DIR}/config/samples/external-object-storage
+kustomize build . | oc -n ${DSP_Namespace_3} apply -f -
+```
+
 # DataSciencePipelinesApplication Component Overview
 
 When a `DataSciencePipelinesApplication` is deployed, the following components are deployed in the target namespace: 

api/v1alpha1/dspipeline_types.go

@@ -194,8 +194,12 @@ type ExternalStorage struct {
 	Host                string `json:"host"`
 	Bucket              string `json:"bucket"`
 	Scheme              string `json:"scheme"`
-	Port                string `json:"port"`
 	*S3CredentialSecret `json:"s3CredentialsSecret"`
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	Secure bool `json:"secure"`
+	// +kubebuilder:validation:Optional
+	Port string `json:"port"`
 }
 
 type S3CredentialSecret struct {

config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml

@@ -311,10 +311,12 @@ spec:
                         type: object
                       scheme:
                         type: string
+                      secure:
+                        default: true
+                        type: boolean
                     required:
                     - bucket
                     - host
-                    - port
                     - s3CredentialsSecret
                     - scheme
                     type: object

config/samples/external-object-storage/dspa_simple.yaml

@@ -0,0 +1,18 @@
+apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1
+kind: DataSciencePipelinesApplication
+metadata:
+  name: sample
+spec:
+  objectStorage:
+    externalStorage:
+      bucket: rhods-dsp-dev
+      host: s3.amazonaws.com
+      s3CredentialsSecret:
+        accessKey: k8saccesskey
+        secretKey: k8ssecretkey
+        secretName: aws-bucket-creds
+      scheme: https
+  # Optional
+  mlpipelineUI:
+    # Image field is required
+    image: 'quay.io/opendatahub/odh-ml-pipelines-frontend-container:beta-ui'

config/samples/external-object-storage/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - dspa.yaml
+  - storage-creds.yaml

config/samples/external-object-storage/storage-creds.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-bucket-creds
+  labels:
+    opendatahub.io/dashboard: 'true'
+    opendatahub.io/managed: 'true'
+  annotations:
+    opendatahub.io/connection-type: s3
+    openshift.io/display-name: AWS S3 Connection
+stringData:
+  k8saccesskey: someaccesskey
+  k8ssecretkey: somesecretkey
+type: Opaque

controllers/config/defaults.go

@@ -45,10 +45,9 @@ const (
 	MinioDefaultBucket = "mlpipeline"
 	MinioPVCSize       = "10Gi"
 
-	ObjectStoreConnectionSecure = false
-	ObjectStorageSecretName     = "mlpipeline-minio-artifact" // hardcoded in kfp-tekton
-	ObjectStorageAccessKey      = "accesskey"
-	ObjectStorageSecretKey      = "secretkey"
+	ObjectStorageSecretName = "mlpipeline-minio-artifact" // hardcoded in kfp-tekton
+	ObjectStorageAccessKey  = "accesskey"
+	ObjectStorageSecretKey  = "secretkey"
 )
 
 // DSPO Config File Paths

controllers/dspipeline_params.go

@@ -221,14 +221,15 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 		AccessKey:  config.ObjectStorageAccessKey,
 		SecretKey:  config.ObjectStorageSecretKey,
 	}
-	p.ObjectStorageConnection.Secure = config.ObjectStoreConnectionSecure
 
 	if usingExternalObjectStorage {
 		// Assume validation for CR ensures these values exist
 		p.ObjectStorageConnection.Bucket = dsp.Spec.ObjectStorage.ExternalStorage.Bucket
 		p.ObjectStorageConnection.Host = dsp.Spec.ObjectStorage.ExternalStorage.Host
-		p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port
 		p.ObjectStorageConnection.Scheme = dsp.Spec.ObjectStorage.ExternalStorage.Scheme
+		p.ObjectStorageConnection.Secure = dsp.Spec.ObjectStorage.ExternalStorage.Secure
+		// Port can be empty, which is fine.
+		p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port
 		customCreds = dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret
 	} else {
 		if p.Minio == nil {
@@ -262,15 +263,22 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 	}
 
 	endpoint := fmt.Sprintf(
-		"%s://%s:%s",
+		"%s://%s",
 		p.ObjectStorageConnection.Scheme,
 		p.ObjectStorageConnection.Host,
-		p.ObjectStorageConnection.Port,
 	)
 
+	if p.ObjectStorageConnection.Port != "" {
+		endpoint = fmt.Sprintf(
+			"%s:%s",
+			endpoint,
+			p.ObjectStorageConnection.Port,
+		)
+	}
+
 	p.ObjectStorageConnection.Endpoint = endpoint
 
-	// Secret where DB credentials reside on cluster
+	// Secret where credentials reside on cluster
 	var credsSecretName string
 	var credsAccessKey string
 	var credsSecretKey string

controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml

@@ -129,7 +129,7 @@ spec:
                   key: "secretkey"
                   name: "mlpipeline-minio-artifact"
             - name: OBJECTSTORECONFIG_SECURE
-              value: "false"
+              value: "true"
             - name: MINIO_SERVICE_SERVICE_HOST
               value: "teststoragehost3"
             - name: MINIO_SERVICE_SERVICE_PORT

controllers/testdata/declarative/case_3/expected/created/storage_secret.yaml

@@ -11,5 +11,5 @@ data:
   host: dGVzdHN0b3JhZ2Vob3N0Mw==
   port: ODA=
   secretkey: dGVzdHNlY3JldGtleXZhbHVlMw==
-  secure: ZmFsc2U=
+  secure: dHJ1ZQ==
 type: Opaque