Skip to content

Commit 1abffb6

Browse files
TomcliTomcli
authored
optimize(sdk): Move common task labels to pipeline labels to reduce yaml size (kubeflow#1020)
* move common task labels to pipeline labels * move common task labels to pipeline labels * update new python lint requirements
1 parent d0079ac commit 1abffb6

File tree

136 files changed

+727
-1181
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

136 files changed

+727
-1181
lines changed

‎sdk/python/kfp_tekton/compiler/_op_to_template.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -579,7 +579,7 @@ def _op_to_template(op: BaseOp,
579579
try:
580580
result_size_map = json.loads(result_size_map)
581581
except ValueError:
582-
raise("tekton-result-sizes annotation is not a valid JSON")
582+
raise ("tekton-result-sizes annotation is not a valid JSON")
583583
# Normalize estimated result size keys.
584584
result_size_map = {sanitize_k8s_name(key, allow_capital_underscore=True): value
585585
for key, value in result_size_map.items()}
@@ -596,7 +596,7 @@ def _op_to_template(op: BaseOp,
596596
try:
597597
value = int(value)
598598
except ValueError:
599-
raise("Estimated value for result %s is %s, but it needs to be an integer." % (key, value))
599+
raise ("Estimated value for result %s is %s, but it needs to be an integer." % (key, value))
600600
if key in op_result_names:
601601
packed_index = -1
602602
# Look for bin that can fit the result value

‎sdk/python/kfp_tekton/compiler/compiler.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,8 @@ def __init__(self, **kwargs):
150150

151151
def _set_pipeline_conf(self, tekton_pipeline_conf: TektonPipelineConf):
152152
self.pipeline_labels = tekton_pipeline_conf.pipeline_labels
153+
self.pipeline_labels['pipelines.kubeflow.org/pipelinename'] = ''
154+
self.pipeline_labels['pipelines.kubeflow.org/generation'] = ''
153155
self.pipeline_annotations = tekton_pipeline_conf.pipeline_annotations
154156
self.tekton_inline_spec = tekton_pipeline_conf.tekton_inline_spec
155157
self.resource_in_separate_yaml = tekton_pipeline_conf.resource_in_separate_yaml
@@ -1012,7 +1014,7 @@ def map_cel_vars(a):
10121014

10131015
# Only one of --taskRef and --taskSpec allowed.
10141016
if custom_task_args.get('taskRef', '') and custom_task_args.get('taskSpec', ''):
1015-
raise("Custom task invalid configuration %s, Only one of --taskRef and --taskSpec allowed." % custom_task_args)
1017+
raise ("Custom task invalid configuration %s, Only one of --taskRef and --taskSpec allowed." % custom_task_args)
10161018
if custom_task_args.get('taskRef', ''):
10171019
try:
10181020
custom_task_cr = {
@@ -1031,7 +1033,7 @@ def map_cel_vars(a):
10311033
if custom_task_cr:
10321034
self.custom_task_crs.append(custom_task_cr)
10331035
except ValueError:
1034-
raise("Custom task ref %s is not a valid Python Dictionary" % custom_task_args['taskRef'])
1036+
raise ("Custom task ref %s is not a valid Python Dictionary" % custom_task_args['taskRef'])
10351037
# Setting --taskRef flag indicates, that spec be inlined.
10361038
if custom_task_args.get('taskSpec', ''):
10371039
try:
@@ -1046,7 +1048,7 @@ def map_cel_vars(a):
10461048
}
10471049
}
10481050
except ValueError:
1049-
raise("Custom task spec %s is not a valid Python Dictionary" % custom_task_args['taskSpec'])
1051+
raise ("Custom task spec %s is not a valid Python Dictionary" % custom_task_args['taskSpec'])
10501052
# Pop custom task artifacts since we have no control of how
10511053
# custom task controller is handling the container/task execution.
10521054
self.artifact_items.pop(template['metadata']['name'], None)
@@ -1055,8 +1057,6 @@ def map_cel_vars(a):
10551057
if task_ref.get('taskSpec', ''):
10561058
task_ref['taskSpec']['metadata'] = task_ref['taskSpec'].get('metadata', {})
10571059
task_labels = template['metadata'].get('labels', {})
1058-
task_labels['pipelines.kubeflow.org/pipelinename'] = task_labels.get('pipelines.kubeflow.org/pipelinename', '')
1059-
task_labels['pipelines.kubeflow.org/generation'] = task_labels.get('pipelines.kubeflow.org/generation', '')
10601060
cache_default = self.pipeline_labels.get('pipelines.kubeflow.org/cache_enabled', 'true')
10611061
task_labels['pipelines.kubeflow.org/cache_enabled'] = task_labels.get('pipelines.kubeflow.org/cache_enabled', cache_default)
10621062

@@ -1855,8 +1855,6 @@ def _inline_tasks(self, tasks: List[Dict[Text, Any]], crs: List[Dict[Text, Any]]
18551855
if 'taskSpec' in workflow_tasks[j]:
18561856
workflow_tasks[j]['taskSpec']['metadata'] = workflow_tasks[j]['taskSpec'].get('metadata', {})
18571857
task_labels = workflow_tasks[j]['taskSpec']['metadata'].get('labels', {})
1858-
task_labels['pipelines.kubeflow.org/pipelinename'] = task_labels.get('pipelines.kubeflow.org/pipelinename', '')
1859-
task_labels['pipelines.kubeflow.org/generation'] = task_labels.get('pipelines.kubeflow.org/generation', '')
18601858
cache_default = self.pipeline_labels.get('pipelines.kubeflow.org/cache_enabled', 'true')
18611859
task_labels['pipelines.kubeflow.org/cache_enabled'] = task_labels.get('pipelines.kubeflow.org/cache_enabled', cache_default)
18621860
workflow_tasks[j]['taskSpec']['metadata']['labels'] = task_labels

‎sdk/python/tests/compiler/testdata/affinity.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ metadata:
2727
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
2828
pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with affinity",
2929
"name": "affinity"}'
30+
labels:
31+
pipelines.kubeflow.org/pipelinename: ''
32+
pipelines.kubeflow.org/generation: ''
3033
spec:
3134
pipelineSpec:
3235
tasks:
@@ -44,8 +47,6 @@ spec:
4447
runAsUser: 0
4548
metadata:
4649
labels:
47-
pipelines.kubeflow.org/pipelinename: ''
48-
pipelines.kubeflow.org/generation: ''
4950
pipelines.kubeflow.org/cache_enabled: "true"
5051
annotations:
5152
pipelines.kubeflow.org/component_spec_digest: '{"name": "echo", "outputs":

‎sdk/python/tests/compiler/testdata/any_sequencer.yaml

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ metadata:
3232
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
3333
pipelines.kubeflow.org/pipeline_spec: '{"description": "Any Sequencer Component
3434
Demo", "name": "any-sequencer"}'
35+
labels:
36+
pipelines.kubeflow.org/pipelinename: ''
37+
pipelines.kubeflow.org/generation: ''
3538
spec:
3639
pipelineSpec:
3740
tasks:
@@ -46,8 +49,6 @@ spec:
4649
image: alpine:latest
4750
metadata:
4851
labels:
49-
pipelines.kubeflow.org/pipelinename: ''
50-
pipelines.kubeflow.org/generation: ''
5152
pipelines.kubeflow.org/cache_enabled: "true"
5253
annotations:
5354
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -65,8 +66,6 @@ spec:
6566
image: alpine:latest
6667
metadata:
6768
labels:
68-
pipelines.kubeflow.org/pipelinename: ''
69-
pipelines.kubeflow.org/generation: ''
7069
pipelines.kubeflow.org/cache_enabled: "true"
7170
annotations:
7271
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -84,8 +83,6 @@ spec:
8483
image: alpine:latest
8584
metadata:
8685
labels:
87-
pipelines.kubeflow.org/pipelinename: ''
88-
pipelines.kubeflow.org/generation: ''
8986
pipelines.kubeflow.org/cache_enabled: "true"
9087
annotations:
9188
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -150,8 +147,6 @@ spec:
150147
description: /tmp/outputs/Output/data
151148
metadata:
152149
labels:
153-
pipelines.kubeflow.org/pipelinename: ''
154-
pipelines.kubeflow.org/generation: ''
155150
pipelines.kubeflow.org/cache_enabled: "true"
156151
annotations:
157152
pipelines.kubeflow.org/component_spec_digest: '{"name": "Flip coin", "outputs":
@@ -171,8 +166,6 @@ spec:
171166
image: alpine:latest
172167
metadata:
173168
labels:
174-
pipelines.kubeflow.org/pipelinename: ''
175-
pipelines.kubeflow.org/generation: ''
176169
pipelines.kubeflow.org/cache_enabled: "true"
177170
annotations:
178171
pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -205,8 +198,6 @@ spec:
205198
description: /tmp/outputs/status/data
206199
metadata:
207200
labels:
208-
pipelines.kubeflow.org/pipelinename: ''
209-
pipelines.kubeflow.org/generation: ''
210201
pipelines.kubeflow.org/cache_enabled: "true"
211202
annotations:
212203
pipelines.kubeflow.org/component_spec_digest: '{"name": "any_test", "outputs":

‎sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ metadata:
4040
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
4141
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
4242
\"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
43+
labels:
44+
pipelines.kubeflow.org/pipelinename: ''
45+
pipelines.kubeflow.org/generation: ''
4346
spec:
4447
params:
4548
- name: param
@@ -68,8 +71,6 @@ spec:
6871
description: /tmp/outputs/output_value/data
6972
metadata:
7073
labels:
71-
pipelines.kubeflow.org/pipelinename: ''
72-
pipelines.kubeflow.org/generation: ''
7374
pipelines.kubeflow.org/cache_enabled: "true"
7475
annotations:
7576
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
@@ -96,8 +97,6 @@ spec:
9697
description: /tmp/outputs/output_value/data
9798
metadata:
9899
labels:
99-
pipelines.kubeflow.org/pipelinename: ''
100-
pipelines.kubeflow.org/generation: ''
101100
pipelines.kubeflow.org/cache_enabled: "true"
102101
annotations:
103102
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
@@ -127,8 +126,6 @@ spec:
127126
description: /tmp/outputs/status/data
128127
metadata:
129128
labels:
130-
pipelines.kubeflow.org/pipelinename: ''
131-
pipelines.kubeflow.org/generation: ''
132129
pipelines.kubeflow.org/cache_enabled: "true"
133130
annotations:
134131
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
@@ -176,8 +173,6 @@ spec:
176173
description: /tmp/outputs/output_value/data
177174
metadata:
178175
labels:
179-
pipelines.kubeflow.org/pipelinename: ''
180-
pipelines.kubeflow.org/generation: ''
181176
pipelines.kubeflow.org/cache_enabled: "true"
182177
annotations:
183178
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-10",
@@ -204,8 +199,6 @@ spec:
204199
description: /tmp/outputs/output_value/data
205200
metadata:
206201
labels:
207-
pipelines.kubeflow.org/pipelinename: ''
208-
pipelines.kubeflow.org/generation: ''
209202
pipelines.kubeflow.org/cache_enabled: "true"
210203
annotations:
211204
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-11",
@@ -235,8 +228,6 @@ spec:
235228
description: /tmp/outputs/status/data
236229
metadata:
237230
labels:
238-
pipelines.kubeflow.org/pipelinename: ''
239-
pipelines.kubeflow.org/generation: ''
240231
pipelines.kubeflow.org/cache_enabled: "true"
241232
annotations:
242233
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-1",
@@ -257,7 +248,5 @@ spec:
257248
iterateParam: param-loop-item
258249
metadata:
259250
labels:
260-
pipelines.kubeflow.org/pipelinename: ''
261-
pipelines.kubeflow.org/generation: ''
262251
pipelines.kubeflow.org/cache_enabled: "true"
263252
timeout: 525600m

‎sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,15 @@ metadata:
4949
paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
5050
\"print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1\"}",
5151
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
52-
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
53-
""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
52+
"true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
5453
"output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
5554
-e\necho $0 > $1\n", "print-10", "$(results.output-value.path)"], "image": "alpine:3.6",
5655
"name": "main"}]}, "timeout": "525600m"}, {"name": "print-11", "taskSpec": {"metadata":
5756
{"annotations": {"pipelines.kubeflow.org/component_spec_digest": "{\"name\":
5857
\"print-11\", \"outputs\": [{\"description\": \"Represents an output paramter.\",
5958
\"name\": \"output_value\", \"type\": \"String\"}], \"version\": \"print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd\"}",
6059
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
61-
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
62-
""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
60+
"true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
6361
"output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
6462
-e\necho $0 > $1\n", "print-11", "$(results.output-value.path)"], "image": "alpine:3.6",
6563
"name": "main"}]}, "timeout": "525600m"}, {"name": "any-seq-1", "params": [{"name":
@@ -69,13 +67,15 @@ metadata:
6967
\"outputs\": [{\"description\": \"The output file to create the status\", \"name\":
7068
\"status\"}], \"version\": \"any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22\"}",
7169
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
72-
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
73-
""}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name": "pipelineRun-namespace",
74-
"type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
70+
"true"}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name":
71+
"pipelineRun-namespace", "type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
7572
"name": "status", "type": "string"}], "steps": [{"args": ["--namespace", "$(params.pipelineRun-namespace)",
7673
"--prName", "$(params.pipelineRun-name)", "--taskList", "print-10,print-11",
7774
"--statusPath", "$(results.status.path)"], "command": ["any-task"], "image":
7875
"dspipelines/any-sequencer:latest", "name": "main"}]}, "timeout": "525600m"}]}}}]'
76+
labels:
77+
pipelines.kubeflow.org/pipelinename: ''
78+
pipelines.kubeflow.org/generation: ''
7979
spec:
8080
params:
8181
- name: param
@@ -104,8 +104,6 @@ spec:
104104
description: /tmp/outputs/output_value/data
105105
metadata:
106106
labels:
107-
pipelines.kubeflow.org/pipelinename: ''
108-
pipelines.kubeflow.org/generation: ''
109107
pipelines.kubeflow.org/cache_enabled: "true"
110108
annotations:
111109
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
@@ -132,8 +130,6 @@ spec:
132130
description: /tmp/outputs/output_value/data
133131
metadata:
134132
labels:
135-
pipelines.kubeflow.org/pipelinename: ''
136-
pipelines.kubeflow.org/generation: ''
137133
pipelines.kubeflow.org/cache_enabled: "true"
138134
annotations:
139135
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
@@ -163,8 +159,6 @@ spec:
163159
description: /tmp/outputs/status/data
164160
metadata:
165161
labels:
166-
pipelines.kubeflow.org/pipelinename: ''
167-
pipelines.kubeflow.org/generation: ''
168162
pipelines.kubeflow.org/cache_enabled: "true"
169163
annotations:
170164
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":

‎sdk/python/tests/compiler/testdata/artifact_outputs.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ metadata:
3131
pipelines.kubeflow.org/pipeline_spec: '{"description": "Add labels to identify
3232
outputs as artifacts.", "inputs": [{"default": "gs://ml-pipeline-playground/shakespeare1.txt",
3333
"name": "url1", "optional": true, "type": "String"}], "name": "artifact-out-pipeline"}'
34+
labels:
35+
pipelines.kubeflow.org/pipelinename: ''
36+
pipelines.kubeflow.org/generation: ''
3437
spec:
3538
params:
3639
- name: url1
@@ -109,8 +112,6 @@ spec:
109112
type: string
110113
metadata:
111114
labels:
112-
pipelines.kubeflow.org/pipelinename: ''
113-
pipelines.kubeflow.org/generation: ''
114115
pipelines.kubeflow.org/cache_enabled: "true"
115116
annotations:
116117
artifact_outputs: '["data"]'

‎sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ metadata:
4141
sidecar.istio.io/inject: "false"
4242
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
4343
pipelines.kubeflow.org/pipeline_spec: '{"name": "Artifact passing pipeline"}'
44+
labels:
45+
pipelines.kubeflow.org/pipelinename: ''
46+
pipelines.kubeflow.org/generation: ''
4447
spec:
4548
pipelineSpec:
4649
tasks:
@@ -110,8 +113,6 @@ spec:
110113
type: string
111114
metadata:
112115
labels:
113-
pipelines.kubeflow.org/pipelinename: ''
114-
pipelines.kubeflow.org/generation: ''
115116
pipelines.kubeflow.org/cache_enabled: "true"
116117
annotations:
117118
pipelines.kubeflow.org/component_spec_digest: '{"name": "Producer", "outputs":
@@ -199,8 +200,6 @@ spec:
199200
type: string
200201
metadata:
201202
labels:
202-
pipelines.kubeflow.org/pipelinename: ''
203-
pipelines.kubeflow.org/generation: ''
204203
pipelines.kubeflow.org/cache_enabled: "true"
205204
annotations:
206205
pipelines.kubeflow.org/component_spec_digest: '{"name": "Processor", "outputs":
@@ -243,8 +242,6 @@ spec:
243242
- name: processor-trname
244243
metadata:
245244
labels:
246-
pipelines.kubeflow.org/pipelinename: ''
247-
pipelines.kubeflow.org/generation: ''
248245
pipelines.kubeflow.org/cache_enabled: "true"
249246
annotations:
250247
pipelines.kubeflow.org/component_spec_digest: '{"name": "Consumer", "outputs":
@@ -337,8 +334,6 @@ spec:
337334
emptyDir: {}
338335
metadata:
339336
labels:
340-
pipelines.kubeflow.org/pipelinename: ''
341-
pipelines.kubeflow.org/generation: ''
342337
pipelines.kubeflow.org/cache_enabled: "true"
343338
annotations:
344339
pipelines.kubeflow.org/component_spec_digest: '{"name": "Metadata and

‎sdk/python/tests/compiler/testdata/basic_no_decorator.yaml

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,6 @@ spec:
7272
description: /tmp/outputs/word/data
7373
metadata:
7474
labels:
75-
pipelines.kubeflow.org/pipelinename: ''
76-
pipelines.kubeflow.org/generation: ''
7775
pipelines.kubeflow.org/cache_enabled: "true"
7876
annotations:
7977
pipelines.kubeflow.org/component_spec_digest: '{"name": "frequent-word",
@@ -107,8 +105,6 @@ spec:
107105
value: default_output
108106
metadata:
109107
labels:
110-
pipelines.kubeflow.org/pipelinename: ''
111-
pipelines.kubeflow.org/generation: ''
112108
pipelines.kubeflow.org/cache_enabled: "true"
113109
annotations:
114110
pipelines.kubeflow.org/component_spec_digest: '{"name": "save-message",
@@ -127,8 +123,6 @@ spec:
127123
image: python:3.6-jessie
128124
metadata:
129125
labels:
130-
pipelines.kubeflow.org/pipelinename: ''
131-
pipelines.kubeflow.org/generation: ''
132126
pipelines.kubeflow.org/cache_enabled: "true"
133127
annotations:
134128
pipelines.kubeflow.org/component_spec_digest: '{"name": "exit-handler",

0 commit comments

Comments
 (0)