Merge remote-tracking branch 'kubeflow/master' into odh-master

Author: HumairAK

.github/workflows/kfp-tekton-unittests.yml

@@ -30,6 +30,10 @@ jobs:
   validate-testdata:
     runs-on: ubuntu-latest
     steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.x
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
         uses: actions/setup-python@v2
@@ -91,7 +95,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.17.x
+        go-version: 1.19.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: "run go unit tests"
@@ -104,7 +108,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17.x
+          go-version: 1.19.x
       - name: Checkout code
         uses: actions/checkout@v2
       - name: "run go pipelineLoop unit tests"
@@ -133,6 +137,9 @@ jobs:
         uses: container-tools/kind-action@v1
         with:
           cluster_name: kfp-tekton
+          kubectl_version: v1.24.7
+          version: v0.17.0
+          node_image: kindest/node:v1.24.7
       - name: build images
         if: ${{ steps.backend-changes.outputs.all }}
         run: ./scripts/deploy/github/build-images.sh

.tekton/pipeline.yaml

@@ -653,6 +653,29 @@ spec:
       workspaces:
       - name: task-pvc
         workspace: pipeline-pvc
+    - name: e2e-test-many-edges
+      retries: 1
+      taskRef:
+        name: e2e-test
+      runAfter:
+        - deploy
+        - deploy-pipeline-loops-e2e
+      params:
+        - name: apikey
+          value: $(params.apikey)
+        - name: kubernetes-cluster
+          value: $(params.kubernetes-cluster)
+        - name: kubeflow-ns
+          value: $(params.kubeflow-ns)
+        - name: slack-webhook
+          value: $(params.slack-webhook)
+        - name: slack-channel
+          value: $(params.slack-channel)
+        - name: test-script
+          value: "scripts/deploy/iks/test-many-edges.sh"
+      workspaces:
+      - name: task-pvc
+        workspace: pipeline-pvc
     - name: e2e-test-trusted-ai
       retries: 1
       taskRef:

.tekton/task.yaml

@@ -35,7 +35,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: run-go-unittests
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: DOCKER_USERNAME
           value: $(params.docker-username)
@@ -104,7 +104,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: build-image
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: DOCKER_USERNAME
           value: $(params.docker-username)
@@ -170,7 +170,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: build-image
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: DOCKER_USERNAME
           value: $(params.docker-username)
@@ -240,7 +240,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: build-image
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: DOCKER_USERNAME
           value: $(params.docker-username)
@@ -350,7 +350,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: deploy-kfp-tekton
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: DOCKER_USERNAME
           value: $(params.docker-username)
@@ -400,7 +400,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: run-test
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: IBM_CLOUD_API_KEY
           value: $(params.apikey)
@@ -472,7 +472,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: publish-images-to-dockerhub
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: IBM_CLOUD_API_KEY
           value: $(params.apikey)
@@ -577,7 +577,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: undeploy-kfp-tekton
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: PIPELINE_KUBERNETES_CLUSTER_NAME
           value: $(params.kubernetes-cluster)
@@ -607,7 +607,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: build-binaries
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: BUILD_DIR
           value: $(params.build-dir)
@@ -629,7 +629,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: setup-deploy
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       command:
         - /bin/bash
         - '-c'
@@ -651,7 +651,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: setup-deploy
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       command:
         - /bin/bash
         - '-c'
@@ -694,7 +694,7 @@ spec:
     mountPath: /artifacts
   steps:
     - name: iks-script
-      image: docker.io/aipipeline/pipeline-base-image:1.4
+      image: docker.io/aipipeline/pipeline-base-image:1.5
       env:
         - name: IBM_CLOUD_API_KEY
           value: $(params.apikey)

Makefile

@@ -16,7 +16,7 @@
 #  - The help target was derived from https://stackoverflow.com/a/35730328/5601796
 
 VENV ?= .venv
-KFP_TEKTON_RELEASE ?= v1.3.1
+KFP_TEKTON_RELEASE ?= v1.4.0
 export VIRTUAL_ENV := $(abspath ${VENV})
 export PATH := ${VIRTUAL_ENV}/bin:${PATH}
 DOCKER_REGISTRY ?= aipipeline

README.md

@@ -15,7 +15,7 @@ For more details about the project please follow this detailed [blog post](https
 ## Architecture
 
 We are currently using [Kubeflow Pipelines 1.8.4](https://github.com/kubeflow/pipelines/releases/tag/1.8.4) and
-[Tekton >= 0.38.3](https://github.com/tektoncd/pipeline/releases/tag/v0.38.3)
+[Tekton >= 0.41.0](https://github.com/tektoncd/pipeline/releases/tag/v0.41.0)
 for this project.
 
 ![kfp-tekton](images/kfp-tekton.png)

VERSION

@@ -1 +1 @@
-1.3.1
+1.4.0

backend/Dockerfile

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 # 1. Build api server application
-FROM golang:1.17.6-stretch as builder
+FROM golang:1.19.3-buster as builder
 RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
 WORKDIR /go/src/github.com/kubeflow/pipelines
 COPY . .

backend/Dockerfile.cacheserver

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Dockerfile for building the source code of cache_server
-FROM golang:1.17.6-alpine3.15 as builder
+FROM golang:1.19.3-alpine3.15 as builder
 
 RUN apk update && apk upgrade && \
     apk add --no-cache bash git openssh gcc musl-dev

backend/Dockerfile.persistenceagent

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.17.6-alpine3.15 as builder
+FROM golang:1.19.3-alpine3.15 as builder
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
 COPY . .
@@ -48,4 +48,7 @@ ENV TTL_SECONDS_AFTER_WORKFLOW_FINISH 86400
 # NUM_WORKERS indicates now many worker goroutines
 ENV NUM_WORKERS 2
 
-CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS}
+#CHILDREFERENCESKIND kind of runs to search for the childReferences
+ENV CHILDREFERENCES_KINDS ""
+
+CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} --childReferencesKinds ${CHILDREFERENCES_KINDS}

backend/Dockerfile.scheduledworkflow

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.17.6-alpine3.15 as builder
+FROM golang:1.19.3-alpine3.15 as builder
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
 COPY . .