Skip to content

Commit 51e7623

Browse files
authored
optimize(sdk): Further moving common caching annotations to pipeline level (kubeflow#1038)
1 parent 5e6e4cf commit 51e7623

File tree

135 files changed

+1039
-1283
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

135 files changed

+1039
-1283
lines changed

sdk/python/kfp_tekton/compiler/compiler.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ def _set_pipeline_conf(self, tekton_pipeline_conf: TektonPipelineConf):
157157
self.pipeline_labels['pipelines.kubeflow.org/pipelinename'] = ''
158158
self.pipeline_labels['pipelines.kubeflow.org/generation'] = ''
159159
self.pipeline_annotations = tekton_pipeline_conf.pipeline_annotations
160+
self.pipeline_annotations['tekton.dev/template'] = ''
160161
self.tekton_inline_spec = tekton_pipeline_conf.tekton_inline_spec
161162
self.resource_in_separate_yaml = tekton_pipeline_conf.resource_in_separate_yaml
162163
self.security_context = tekton_pipeline_conf.security_context
@@ -1072,9 +1073,7 @@ def map_cel_vars(a):
10721073
task_labels = template['metadata'].get('labels', {})
10731074
cache_default = self.pipeline_labels.get('pipelines.kubeflow.org/cache_enabled', 'true')
10741075
task_labels['pipelines.kubeflow.org/cache_enabled'] = task_labels.get('pipelines.kubeflow.org/cache_enabled', cache_default)
1075-
10761076
task_annotations = template['metadata'].get('annotations', {})
1077-
task_annotations['tekton.dev/template'] = task_annotations.get('tekton.dev/template', '')
10781077

10791078
# Updata default metadata at the end.
10801079
task_ref['taskSpec']['metadata']['labels'] = task_labels

sdk/python/tests/compiler/testdata/affinity.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ metadata:
2424
tekton.dev/artifact_endpoint_scheme: http://
2525
tekton.dev/artifact_items: '{"echo": []}'
2626
sidecar.istio.io/inject: "false"
27+
tekton.dev/template: ''
2728
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
2829
pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with affinity",
2930
"name": "affinity"}'
@@ -51,7 +52,6 @@ spec:
5152
annotations:
5253
pipelines.kubeflow.org/component_spec_digest: '{"name": "echo", "outputs":
5354
[], "version": "echo@sha256=7a01fe307dc14789e461b2122f62bb8bc694f3f2eec35fa3239cf971034e2504"}'
54-
tekton.dev/template: ''
5555
timeout: 525600m
5656
taskRunSpecs:
5757
- pipelineTaskName: echo

sdk/python/tests/compiler/testdata/any_sequencer.yaml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ metadata:
2929
"flip-coin": [["Output", "$(results.Output.path)"]], "sleepcomponent": [], "sleepcomponent-2":
3030
[], "sleepcomponent-3": [], "sleepcomponent-4": []}'
3131
sidecar.istio.io/inject: "false"
32+
tekton.dev/template: ''
3233
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
3334
pipelines.kubeflow.org/pipeline_spec: '{"description": "Any Sequencer Component
3435
Demo", "name": "any-sequencer"}'
@@ -53,7 +54,6 @@ spec:
5354
annotations:
5455
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
5556
"outputs": [], "version": "sleepComponent@sha256=a885c0df4b9597faab0f119871ba6935f574cbd2b7232dcfa0d7602bffc9e223"}'
56-
tekton.dev/template: ''
5757
timeout: 525600m
5858
- name: sleepcomponent-2
5959
taskSpec:
@@ -70,7 +70,6 @@ spec:
7070
annotations:
7171
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
7272
"outputs": [], "version": "sleepComponent@sha256=a885c0df4b9597faab0f119871ba6935f574cbd2b7232dcfa0d7602bffc9e223"}'
73-
tekton.dev/template: ''
7473
timeout: 525600m
7574
- name: sleepcomponent-3
7675
taskSpec:
@@ -87,7 +86,6 @@ spec:
8786
annotations:
8887
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
8988
"outputs": [], "version": "sleepComponent@sha256=a885c0df4b9597faab0f119871ba6935f574cbd2b7232dcfa0d7602bffc9e223"}'
90-
tekton.dev/template: ''
9189
timeout: 525600m
9290
- name: flip-coin
9391
taskSpec:
@@ -151,7 +149,6 @@ spec:
151149
annotations:
152150
pipelines.kubeflow.org/component_spec_digest: '{"name": "Flip coin", "outputs":
153151
[{"name": "Output", "type": "String"}], "version": "Flip coin@sha256=83ef617eda2e04a688908716a02461237baee3cd04754ea99894e441b1f0679a"}'
154-
tekton.dev/template: ''
155152
runAfter:
156153
- sleepcomponent
157154
timeout: 525600m
@@ -170,7 +167,6 @@ spec:
170167
annotations:
171168
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
172169
"outputs": [], "version": "sleepComponent@sha256=a885c0df4b9597faab0f119871ba6935f574cbd2b7232dcfa0d7602bffc9e223"}'
173-
tekton.dev/template: ''
174170
runAfter:
175171
- any-test
176172
timeout: 525600m
@@ -203,7 +199,6 @@ spec:
203199
pipelines.kubeflow.org/component_spec_digest: '{"name": "any_test", "outputs":
204200
[{"description": "The output file to create the status", "name": "status"}],
205201
"version": "any_test@sha256=c4aaac826d09733534b96ec4a17c6646fa9d71e55d1e4bece1a5c0d20873e563"}'
206-
tekton.dev/template: ''
207202
params:
208203
- name: pipelineRun-name
209204
- name: pipelineRun-namespace

sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ metadata:
3737
"print-10": [["output_value", "$(results.output-value.path)"]], "print-11":
3838
[["output_value", "$(results.output-value.path)"]]}'
3939
sidecar.istio.io/inject: "false"
40+
tekton.dev/template: ''
4041
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
4142
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
4243
\"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
@@ -76,7 +77,6 @@ spec:
7677
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
7778
[{"description": "Represents an output paramter.", "name": "output_value",
7879
"type": "String"}], "version": "print-00@sha256=7c633686b0b637cfba6cba8c9b4e871cf172395432360d6aaa2d89623fb336b3"}'
79-
tekton.dev/template: ''
8080
timeout: 525600m
8181
- name: print-01
8282
taskSpec:
@@ -102,7 +102,6 @@ spec:
102102
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
103103
[{"description": "Represents an output paramter.", "name": "output_value",
104104
"type": "String"}], "version": "print-01@sha256=d511ac628d43cc5b393fbebd10be93662b30117f1413b84afd4e7b2e5ff5ed33"}'
105-
tekton.dev/template: ''
106105
timeout: 525600m
107106
- name: any-seq-0
108107
taskSpec:
@@ -131,7 +130,6 @@ spec:
131130
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
132131
[{"description": "The output file to create the status", "name": "status"}],
133132
"version": "any-seq-0@sha256=2a9bf729444af0ff906f5a6a77eff3eecfd419ebfbdb51e5f04cdfc11d58a7b8"}'
134-
tekton.dev/template: ''
135133
params:
136134
- name: pipelineRun-name
137135
- name: pipelineRun-namespace
@@ -178,7 +176,6 @@ spec:
178176
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-10",
179177
"outputs": [{"description": "Represents an output paramter.",
180178
"name": "output_value", "type": "String"}], "version": "print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1"}'
181-
tekton.dev/template: ''
182179
timeout: 525600m
183180
- name: print-11
184181
taskSpec:
@@ -204,7 +201,6 @@ spec:
204201
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-11",
205202
"outputs": [{"description": "Represents an output paramter.",
206203
"name": "output_value", "type": "String"}], "version": "print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd"}'
207-
tekton.dev/template: ''
208204
timeout: 525600m
209205
- name: any-seq-1
210206
taskSpec:
@@ -233,7 +229,6 @@ spec:
233229
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-1",
234230
"outputs": [{"description": "The output file to create the status",
235231
"name": "status"}], "version": "any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22"}'
236-
tekton.dev/template: ''
237232
params:
238233
- name: pipelineRun-name
239234
type: string

sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ metadata:
3737
"print-10": [["output_value", "$(results.output-value.path)"]], "print-11":
3838
[["output_value", "$(results.output-value.path)"]]}'
3939
sidecar.istio.io/inject: "false"
40+
tekton.dev/template: ''
4041
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
4142
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
4243
\"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
@@ -47,29 +48,28 @@ metadata:
4748
{"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
4849
"{\"name\": \"print-10\", \"outputs\": [{\"description\": \"Represents an output
4950
paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
50-
\"print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1\"}",
51-
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
52-
"true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
53-
"output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
54-
-e\necho $0 > $1\n", "print-10", "$(results.output-value.path)"], "image": "alpine:3.6",
55-
"name": "main"}]}, "timeout": "525600m"}, {"name": "print-11", "taskSpec": {"metadata":
56-
{"annotations": {"pipelines.kubeflow.org/component_spec_digest": "{\"name\":
57-
\"print-11\", \"outputs\": [{\"description\": \"Represents an output paramter.\",
58-
\"name\": \"output_value\", \"type\": \"String\"}], \"version\": \"print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd\"}",
59-
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
60-
"true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
61-
"output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
62-
-e\necho $0 > $1\n", "print-11", "$(results.output-value.path)"], "image": "alpine:3.6",
63-
"name": "main"}]}, "timeout": "525600m"}, {"name": "any-seq-1", "params": [{"name":
64-
"pipelineRun-name", "value": "$(context.pipelineRun.name)"}, {"name": "pipelineRun-namespace",
65-
"value": "$(context.pipelineRun.namespace)"}], "taskSpec": {"metadata": {"annotations":
66-
{"pipelines.kubeflow.org/component_spec_digest": "{\"name\": \"any-seq-1\",
67-
\"outputs\": [{\"description\": \"The output file to create the status\", \"name\":
68-
\"status\"}], \"version\": \"any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22\"}",
69-
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
70-
"true"}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name":
71-
"pipelineRun-namespace", "type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
72-
"name": "status", "type": "string"}], "steps": [{"args": ["--namespace", "$(params.pipelineRun-namespace)",
51+
\"print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1\"}"},
52+
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "results": [{"description":
53+
"/tmp/outputs/output_value/data", "name": "output-value", "type": "string"}],
54+
"steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "print-10", "$(results.output-value.path)"],
55+
"image": "alpine:3.6", "name": "main"}]}, "timeout": "525600m"}, {"name": "print-11",
56+
"taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
57+
"{\"name\": \"print-11\", \"outputs\": [{\"description\": \"Represents an output
58+
paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
59+
\"print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd\"}"},
60+
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "results": [{"description":
61+
"/tmp/outputs/output_value/data", "name": "output-value", "type": "string"}],
62+
"steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "print-11", "$(results.output-value.path)"],
63+
"image": "alpine:3.6", "name": "main"}]}, "timeout": "525600m"}, {"name": "any-seq-1",
64+
"params": [{"name": "pipelineRun-name", "value": "$(context.pipelineRun.name)"},
65+
{"name": "pipelineRun-namespace", "value": "$(context.pipelineRun.namespace)"}],
66+
"taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
67+
"{\"name\": \"any-seq-1\", \"outputs\": [{\"description\": \"The output file
68+
to create the status\", \"name\": \"status\"}], \"version\": \"any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22\"}"},
69+
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "params": [{"name":
70+
"pipelineRun-name", "type": "string"}, {"name": "pipelineRun-namespace", "type":
71+
"string"}], "results": [{"description": "/tmp/outputs/status/data", "name":
72+
"status", "type": "string"}], "steps": [{"args": ["--namespace", "$(params.pipelineRun-namespace)",
7373
"--prName", "$(params.pipelineRun-name)", "--taskList", "print-10,print-11",
7474
"--statusPath", "$(results.status.path)"], "command": ["any-task"], "image":
7575
"dspipelines/any-sequencer:latest", "name": "main"}]}, "timeout": "525600m"}]}}}]'
@@ -109,7 +109,6 @@ spec:
109109
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
110110
[{"description": "Represents an output paramter.", "name": "output_value",
111111
"type": "String"}], "version": "print-00@sha256=7c633686b0b637cfba6cba8c9b4e871cf172395432360d6aaa2d89623fb336b3"}'
112-
tekton.dev/template: ''
113112
timeout: 525600m
114113
- name: print-01
115114
taskSpec:
@@ -135,7 +134,6 @@ spec:
135134
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
136135
[{"description": "Represents an output paramter.", "name": "output_value",
137136
"type": "String"}], "version": "print-01@sha256=d511ac628d43cc5b393fbebd10be93662b30117f1413b84afd4e7b2e5ff5ed33"}'
138-
tekton.dev/template: ''
139137
timeout: 525600m
140138
- name: any-seq-0
141139
taskSpec:
@@ -164,7 +162,6 @@ spec:
164162
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
165163
[{"description": "The output file to create the status", "name": "status"}],
166164
"version": "any-seq-0@sha256=2a9bf729444af0ff906f5a6a77eff3eecfd419ebfbdb51e5f04cdfc11d58a7b8"}'
167-
tekton.dev/template: ''
168165
params:
169166
- name: pipelineRun-name
170167
- name: pipelineRun-namespace

sdk/python/tests/compiler/testdata/artifact_outputs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ metadata:
2727
tekton.dev/artifact_items: '{"gcs-download": [["data", "$(workspaces.gcs-download.path)/artifacts/$ORIG_PR_NAME/$(context.taskRun.name)/data"],
2828
["data2", "$(results.data2.path)"]]}'
2929
sidecar.istio.io/inject: "false"
30+
tekton.dev/template: ''
3031
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
3132
pipelines.kubeflow.org/pipeline_spec: '{"description": "Add labels to identify
3233
outputs as artifacts.", "inputs": [{"default": "gs://ml-pipeline-playground/shakespeare1.txt",
@@ -121,7 +122,6 @@ spec:
121122
pipelines.kubeflow.org/component_spec_digest: '{"name": "gcs-download",
122123
"outputs": [{"name": "data", "type": "String"}, {"name": "data2", "type":
123124
"String"}], "version": "gcs-download@sha256=e61aa54d889d66b15db1bd0a8959fda75d5aa799e468d8985610af4958612dfe"}'
124-
tekton.dev/template: ''
125125
workspaces:
126126
- name: gcs-download
127127
timeout: 525600m

sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ metadata:
3939
"processor": [["Output-1", "$(results.Output-1.path)"], ["Output-2", "$(workspaces.processor.path)/artifacts/$ORIG_PR_NAME/$(context.taskRun.name)/Output-2"]],
4040
"producer": [["Output-1", "$(results.Output-1.path)"], ["Output-2", "$(workspaces.producer.path)/artifacts/$ORIG_PR_NAME/$(context.taskRun.name)/Output-2"]]}'
4141
sidecar.istio.io/inject: "false"
42+
tekton.dev/template: ''
4243
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
4344
pipelines.kubeflow.org/pipeline_spec: '{"name": "Artifact passing pipeline"}'
4445
labels:
@@ -120,7 +121,6 @@ spec:
120121
annotations:
121122
pipelines.kubeflow.org/component_spec_digest: '{"name": "Producer", "outputs":
122123
[{"name": "Output 1"}, {"name": "Output 2"}], "version": "Producer@sha256=f8c0448df5bf22241b635044bce2463869c10f9b450e979b7da029a931486367"}'
123-
tekton.dev/template: ''
124124
workspaces:
125125
- name: producer
126126
timeout: 525600m
@@ -210,7 +210,6 @@ spec:
210210
annotations:
211211
pipelines.kubeflow.org/component_spec_digest: '{"name": "Processor", "outputs":
212212
[{"name": "Output 1"}, {"name": "Output 2"}], "version": "Processor@sha256=dc713ae50cd79f58567512103e2e889ad7505f87770c6681731f94319615f373"}'
213-
tekton.dev/template: ''
214213
workspaces:
215214
- name: processor
216215
timeout: 525600m
@@ -252,7 +251,6 @@ spec:
252251
annotations:
253252
pipelines.kubeflow.org/component_spec_digest: '{"name": "Consumer", "outputs":
254253
[], "version": "Consumer@sha256=ae9cf1975753ed1382e04db32bd2440d61a9cbfb893ec327ff15199589311701"}'
255-
tekton.dev/template: ''
256254
workspaces:
257255
- name: consumer
258256
timeout: 525600m
@@ -346,7 +344,6 @@ spec:
346344
metrics", "outputs": [{"name": "mlpipeline_ui_metadata", "type": "UI_metadata"},
347345
{"name": "mlpipeline_metrics", "type": "Metrics"}], "version": "Metadata
348346
and metrics@sha256=457911fc37b191922b25d050fefef6a3d87e0351f35d0838a053245bd4095e8c"}'
349-
tekton.dev/template: ''
350347
timeout: 525600m
351348
workspaces:
352349
- name: artifact-passing-pipeline

sdk/python/tests/compiler/testdata/basic_no_decorator.yaml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ spec:
7676
annotations:
7777
pipelines.kubeflow.org/component_spec_digest: '{"name": "frequent-word",
7878
"outputs": [{"name": "word", "type": "String"}], "version": "frequent-word@sha256=8ffbb183d49c639c5884fc04232d3589916ed74ab82dc713617d8896180f6a65"}'
79-
tekton.dev/template: ''
8079
timeout: 525600m
8180
- name: save-message
8281
params:
@@ -109,7 +108,6 @@ spec:
109108
annotations:
110109
pipelines.kubeflow.org/component_spec_digest: '{"name": "save-message",
111110
"outputs": [], "version": "save-message@sha256=496ed803b1043558dc383b302185854d3e343fd33fd3a4c7a9e5d11598d0afe3"}'
112-
tekton.dev/template: ''
113111
timeout: 525600m
114112
finally:
115113
- name: exit-handler
@@ -127,7 +125,6 @@ spec:
127125
annotations:
128126
pipelines.kubeflow.org/component_spec_digest: '{"name": "exit-handler",
129127
"outputs": [], "version": "exit-handler@sha256=f602919c39b5b69833987b005c4e28a099a234d24e004890803e3de1900525bf"}'
130-
tekton.dev/template: ''
131128
timeout: 525600m
132129
taskRunSpecs:
133130
- pipelineTaskName: save-message

0 commit comments

Comments
 (0)