Skip to content

Commit 114bee7

Browse files
rimoliverimolive
authored
chore: Add kfp-tekton integration tests and manifests (kubeflow#10702)
Signed-off-by: Ricardo M. Oliveira <[email protected]>
1 parent 67d3cd6 commit 114bee7

File tree

80 files changed

+7630
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+7630
-0
lines changed

‎.github/workflows/backend.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: KFP Tekton backend unit tests
2+
3+
on:
4+
push:
5+
branches: [master]
6+
7+
# Run tests for any PRs which change the backend code
8+
pull_request:
9+
paths:
10+
- 'go.mod'
11+
- 'backend/**'
12+
- 'scripts/deploy/github/**'
13+
- 'manifests/kustomize/**'
14+
15+
env:
16+
GITHUB_ACTION: "true"
17+
SETUPTOOLS_USE_DISTUTILS: "stdlib"
18+
19+
jobs:
20+
run-go-unittests:
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Install Go
24+
uses: actions/setup-go@v4
25+
with:
26+
go-version: 1.20.x
27+
- name: Checkout code
28+
uses: actions/checkout@v4
29+
- name: "run go unit tests"
30+
run: go test -v -cover ./backend/...
31+
backend-integration:
32+
runs-on: ubuntu-latest
33+
steps:
34+
- name: Checkout code
35+
uses: actions/checkout@v4
36+
- name: Create k8s Kind Cluster
37+
uses: container-tools/kind-action@v2
38+
with:
39+
cluster_name: kfp-tekton
40+
kubectl_version: v1.29.2
41+
version: v0.22.0
42+
node_image: kindest/node:v1.29.2
43+
- name: build images
44+
run: ./scripts/deploy/github/build-images.sh
45+
- name: Set up Python 3.10
46+
uses: actions/setup-python@v4
47+
with:
48+
python-version: '3.10'
49+
- name: "deploy kfp-tekton"
50+
run: ./scripts/deploy/github/deploy-kfp.sh
51+
- name: Install sdk
52+
run: |
53+
python3 -m venv .venv
54+
. .venv/bin/activate
55+
pip install -e sdk/python
56+
- name: "flip coin test"
57+
run: |
58+
. .venv/bin/activate
59+
TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh
60+
- name: "static loop test"
61+
run: |
62+
. .venv/bin/activate
63+
TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh
64+
- name: "dynamic loop test"
65+
run: |
66+
. .venv/bin/activate
67+
TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh
68+
- name: "use env"
69+
run: |
70+
. .venv/bin/activate
71+
TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh
72+
- name: "use volume"
73+
run: |
74+
. .venv/bin/activate
75+
TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh
76+
- name: Collect test results
77+
if: always()
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: kfp-tekton-backend-artifacts
81+
path: /tmp/tmp.*/*

‎.github/workflows/codeql.yml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# For most projects, this workflow file will not need changing; you simply need
2+
# to commit it to your repository.
3+
#
4+
# You may wish to alter this file to override the set of languages analyzed,
5+
# or to provide custom queries or build logic.
6+
#
7+
# ******** NOTE ********
8+
# We have attempted to detect the languages in your repository. Please check
9+
# the `language` matrix defined below to confirm you have the correct set of
10+
# supported CodeQL languages.
11+
#
12+
name: "CodeQL"
13+
14+
on:
15+
schedule:
16+
# Every Friday at 19:39
17+
- cron: '39 19 * * 5'
18+
19+
jobs:
20+
analyze:
21+
name: Analyze
22+
runs-on: ubuntu-latest
23+
permissions:
24+
actions: read
25+
contents: read
26+
security-events: write
27+
28+
strategy:
29+
fail-fast: false
30+
matrix:
31+
language: [ 'go', 'javascript', 'python' ]
32+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
33+
# Use only 'java' to analyze code written in Java, Kotlin or both
34+
# Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
35+
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
36+
37+
steps:
38+
- name: Checkout repository
39+
uses: actions/checkout@v3
40+
41+
# Initializes the CodeQL tools for scanning.
42+
- name: Initialize CodeQL
43+
uses: github/codeql-action/init@v2
44+
with:
45+
languages: ${{ matrix.language }}
46+
# If you wish to specify custom queries, you can do so here or in a config file.
47+
# By default, queries listed here will override any specified in a config file.
48+
# Prefix the list here with "+" to use these queries and those in the config file.
49+
50+
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
51+
# queries: security-extended,security-and-quality
52+
53+
54+
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java).
55+
# If this step fails, then you should remove it and run the build manually (see below)
56+
- name: Autobuild
57+
uses: github/codeql-action/autobuild@v2
58+
59+
# ℹ️ Command-line programs to run using the OS shell.
60+
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
61+
62+
# If the Autobuild fails above, remove it and uncomment the following three lines.
63+
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
64+
65+
# - run: |
66+
# echo "Run, Build Application using script"
67+
# ./location_of_script_within_repo/buildscript.sh
68+
69+
- name: Perform CodeQL Analysis
70+
uses: github/codeql-action/analyze@v2
71+
with:
72+
category: "/language:${{matrix.language}}"

‎manifests/kustomize/base/cache/cache-role.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,20 @@ rules:
3131
- watch
3232
- update
3333
- patch
34+
- apiGroups:
35+
- tekton.dev
36+
resources:
37+
- pipelineruns
38+
- customruns
39+
- taskruns
40+
- conditions
41+
- runs
42+
- tasks
43+
verbs:
44+
- create
45+
- get
46+
- list
47+
- watch
48+
- update
49+
- patch
50+
- delete

‎manifests/kustomize/base/metadata/base/kustomization.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3+
namespace: kubeflow
34
resources:
45
- metadata-grpc-configmap.yaml
56
- metadata-grpc-deployment.yaml

‎manifests/kustomize/base/pipeline/ml-pipeline-apiserver-role.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,33 @@ rules:
4949
- tokenreviews
5050
verbs:
5151
- create
52+
- apiGroups:
53+
- tekton.dev
54+
resources:
55+
- pipelineruns
56+
- taskruns
57+
- conditions
58+
- runs
59+
- tasks
60+
- customruns
61+
verbs:
62+
- create
63+
- get
64+
- list
65+
- watch
66+
- update
67+
- patch
68+
- delete
69+
- apiGroups:
70+
- custom.tekton.dev
71+
resources:
72+
- pipelineloops
73+
verbs:
74+
- create
75+
- get
76+
- list
77+
- watch
78+
- update
79+
- patch
80+
- delete
81+

‎manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,32 @@ rules:
3333
verbs:
3434
- reportMetrics
3535
- readArtifact
36+
- apiGroups:
37+
- tekton.dev
38+
resources:
39+
- pipelineruns
40+
- taskruns
41+
- conditions
42+
- runs
43+
- tasks
44+
- customruns
45+
verbs:
46+
- create
47+
- get
48+
- list
49+
- watch
50+
- update
51+
- patch
52+
- delete
53+
- apiGroups:
54+
- custom.tekton.dev
55+
resources:
56+
- pipelineloops
57+
verbs:
58+
- create
59+
- get
60+
- list
61+
- watch
62+
- update
63+
- patch
64+
- delete

‎manifests/kustomize/base/pipeline/pipeline-runner-role.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,32 @@ rules:
7878
- seldondeployments
7979
verbs:
8080
- '*'
81+
- apiGroups:
82+
- tekton.dev
83+
resources:
84+
- pipelineruns
85+
- taskruns
86+
- conditions
87+
- runs
88+
- tasks
89+
- customruns
90+
verbs:
91+
- create
92+
- get
93+
- list
94+
- watch
95+
- update
96+
- patch
97+
- delete
98+
- apiGroups:
99+
- custom.tekton.dev
100+
resources:
101+
- pipelineloops
102+
verbs:
103+
- create
104+
- get
105+
- list
106+
- watch
107+
- update
108+
- patch
109+
- delete

‎manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
namespace: kubeflow
5+
6+
resources:
7+
- namespace.yaml
8+
bases:
9+
- ../third-party/application/cluster-scoped
10+
- ../base/pipeline/cluster-scoped
11+
- ../base/cache-deployer/cluster-scoped
12+
vars:
13+
# NOTE: var name must be unique globally to allow composition of multiple kustomize
14+
# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from
15+
# others.
16+
- name: kfp-cluster-scoped-namespace
17+
objref:
18+
# cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
19+
# so that we only need to change kustomization.yaml's namespace field for namespace customization.
20+
kind: ServiceAccount
21+
name: kubeflow-pipelines-cache-deployer-sa
22+
apiVersion: v1
23+
fieldref:
24+
fieldpath: metadata.namespace
25+
configurations:
26+
- params.yaml

‎manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: '$(kfp-cluster-scoped-namespace)'

‎manifests/kustomize/cluster-scoped-resources-tekton/params.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Allow Kustomize var to replace following fields.
2+
varReference:
3+
- path: metadata/name
4+
kind: Namespace

0 commit comments

Comments
 (0)