Merge pull request opendatahub-io#33 from DharmitD/probes

Adding Probes to ML Pipelines' pods

Author: DharmitD

manifests/opendatahub/base/deployments/ml-pipeline-persistenceagent.yaml

@@ -31,6 +31,22 @@ spec:
           image: persistenceagent
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-persistenceagent
+          livenessProbe:
+            exec:
+              command:
+                - pidof
+                - persistence_agent
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            exec:
+              command:
+                - pidof
+                - persistence_agent
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
           resources:
             requests:
               cpu: 120m

manifests/opendatahub/base/deployments/ml-pipeline-scheduledworkflow.yaml

@@ -32,4 +32,20 @@ spec:
           image: scheduledworkflow
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-scheduledworkflow
+          livenessProbe:
+            exec:
+              command:
+                - pidof
+                - controller
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            exec:
+              command:
+                - pidof
+                - controller
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: ml-pipeline-scheduledworkflow

manifests/opendatahub/base/deployments/ml-pipeline-viewer-crd.yaml

@@ -33,4 +33,20 @@ spec:
           image: viewer-crd-controller
           imagePullPolicy: Always
           name: ml-pipeline-viewer-crd
+          livenessProbe:
+            exec:
+              command:
+                - pidof
+                - controller
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            exec:
+              command:
+                - pidof
+                - controller
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: ml-pipeline-viewer-crd-service-account

manifests/opendatahub/base/deployments/ml-pipeline-visualizationserver.yaml

@@ -21,6 +21,10 @@ spec:
       containers:
         - image: visualization-server
           imagePullPolicy: IfNotPresent
+          name: ml-pipeline-visualizationserver
+          ports:
+            - containerPort: 8888
+              name: http
           livenessProbe:
             exec:
               command:
@@ -30,13 +34,9 @@ spec:
                 - -O
                 - '-'
                 - http://localhost:8888/
-            initialDelaySeconds: 3
+            initialDelaySeconds: 30
             periodSeconds: 5
             timeoutSeconds: 2
-          name: ml-pipeline-visualizationserver
-          ports:
-            - containerPort: 8888
-              name: http
           readinessProbe:
             exec:
               command:

manifests/opendatahub/base/deployments/ml-pipeline.yaml

@@ -121,6 +121,12 @@ spec:
                   name: $(pipeline_install_configuration)
           image: api-server
           imagePullPolicy: Always
+          name: ml-pipeline-api-server
+          ports:
+            - containerPort: 8888
+              name: http
+            - containerPort: 8887
+              name: grpc
           livenessProbe:
             exec:
               command:
@@ -133,12 +139,6 @@ spec:
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
-          name: ml-pipeline-api-server
-          ports:
-            - containerPort: 8888
-              name: http
-            - containerPort: 8887
-              name: grpc
           readinessProbe:
             exec:
               command:

manifests/opendatahub/overlays/component-mlmd/deployments/metadata-envoy-deployment.yaml

@@ -27,3 +27,15 @@ spec:
               name: md-envoy
             - containerPort: 9901
               name: envoy-admin
+          livenessProbe:
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            tcpSocket:
+              port: md-envoy
+            timeoutSeconds: 2
+          readinessProbe:
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            tcpSocket:
+              port: md-envoy
+            timeoutSeconds: 2

manifests/opendatahub/overlays/component-mlmd/deployments/metadata-grpc-deployment.yaml

@@ -55,16 +55,16 @@ spec:
                   key: dbPort
                   name: $(pipeline_install_configuration)
           image: metadata-grpc
+          name: container
+          ports:
+            - containerPort: 8080
+              name: grpc-api
           livenessProbe:
-            initialDelaySeconds: 3
+            initialDelaySeconds: 30
             periodSeconds: 5
             tcpSocket:
               port: grpc-api
             timeoutSeconds: 2
-          name: container
-          ports:
-            - containerPort: 8080
-              name: grpc-api
           readinessProbe:
             initialDelaySeconds: 3
             periodSeconds: 5

manifests/opendatahub/overlays/component-mlmd/deployments/metadata-writer.yaml

@@ -32,4 +32,20 @@ spec:
                   name: $(ml_pipelines_configuration)
           image: metadata-writer
           name: main
+          livenessProbe:
+            exec:
+              command:
+                - pidof
+                - python3
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            exec:
+              command:
+                - pidof
+                - python3
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: kubeflow-pipelines-metadata-writer

manifests/opendatahub/overlays/metadata-store-mysql/deployments/mysql.yaml

@@ -31,6 +31,22 @@ spec:
           ports:
             - containerPort: 3306
               name: mysql
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3306
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3306
+            timeoutSeconds: 1
           resources:
             requests:
               cpu: 100m

manifests/opendatahub/overlays/metadata-store-postgresql/deployments/postgresql.yaml

@@ -35,15 +35,15 @@ spec:
             - name: POSTGRESQL_DATABASE
               value: kfp-tekton
           imagePullPolicy: Always
+          ports:
+            - containerPort: 5432
           livenessProbe:
             exec:
               command:
                 - /usr/libexec/check-container
                 - --live
             initialDelaySeconds: 120
             timeoutSeconds: 10
-          ports:
-            - containerPort: 5432
           readinessProbe:
             exec:
               command: