Skip to content

Commit b4e322a

Browse files
TomcliTomcli
authored
fix(sdk): fix nested loop underscore param (kubeflow#1076)
1 parent 880f37e commit b4e322a

File tree

5 files changed

+355
-1
lines changed

5 files changed

+355
-1
lines changed

‎sdk/python/kfp_tekton/compiler/_tekton_handler.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,9 +314,17 @@ def _handle_tekton_custom_task(custom_task: dict, workflow: dict, recursive_task
314314
json.dumps(task['params']).replace(task_param['value'],
315315
'$(params.%s)' % param_result[1]))
316316
else:
317+
new_param_ref = '%s-%s' % param_result
318+
# Nested task reference name must be the same as the one in the params
319+
# Otherwise, use the default param ref since it will be generated
320+
# by the custom task functions such as condition and loop
321+
for i in task.get('params', []):
322+
if sanitize_k8s_name(i['name']) == new_param_ref:
323+
new_param_ref = i['name']
324+
break
317325
task['params'] = json.loads(
318326
json.dumps(task['params']).replace(task_param['value'],
319-
'$(params.%s-%s)' % param_result))
327+
'$(params.%s)' % new_param_ref))
320328
custom_task_crs.append(custom_task_cr)
321329
custom_task[custom_task_key]['spec']['params'] = sorted(custom_task[custom_task_key]['spec']['params'],
322330
key=lambda k: k['name'])

‎sdk/python/tests/compiler/compiler_tests.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,13 @@ def test_nested_loop_same_arg_workflow(self):
197197
from .testdata.nested_loop_same_arg import loop_multi
198198
self._test_pipeline_workflow(loop_multi, 'nested_loop_same_arg.yaml')
199199

200+
def test_nested_loop_with_underscore_workflow(self):
201+
"""
202+
Test compiling a nested loop with underscore argument workflow.
203+
"""
204+
from .testdata.nested_loop_with_underscore import double_loop_with_underscore
205+
self._test_pipeline_workflow(double_loop_with_underscore, 'nested_loop_with_underscore.yaml')
206+
200207
def test_loop_with_numeric_workflow(self):
201208
"""
202209
Test compiling a loop with numeric inputs in workflow.

‎sdk/python/tests/compiler/testdata/nested_loop_with_underscore.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright 2022 kubeflow.org
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from kfp import dsl
16+
from kfp.components import load_component_from_text
17+
from kfp_tekton.compiler import TektonCompiler
18+
19+
20+
class Coder:
21+
def empty(self):
22+
return ""
23+
24+
25+
TektonCompiler._get_unique_id_code = Coder.empty
26+
27+
28+
def PrintOp(name: str, msg: str = None):
29+
if msg is None:
30+
msg = name
31+
print_op = load_component_from_text(
32+
"""
33+
name: %s
34+
inputs:
35+
- {name: input_text, type: String, description: 'Represents an input parameter.'}
36+
outputs:
37+
- {name: output_value, type: String, description: 'Represents an output paramter.'}
38+
implementation:
39+
container:
40+
image: alpine:3.6
41+
command:
42+
- sh
43+
- -c
44+
- |
45+
set -e
46+
echo $0 > $1
47+
- {inputValue: input_text}
48+
- {outputPath: output_value}
49+
""" % (name)
50+
)
51+
return print_op(msg)
52+
53+
54+
@dsl.pipeline("double_loop_with_underscore")
55+
def double_loop_with_underscore(param_a: list = [1, 2, 3], param_b: list = ["a", "b", "c"]):
56+
op0 = PrintOp('print-0', f"Hello!")
57+
with dsl.ParallelFor(param_a):
58+
with dsl.ParallelFor(param_b):
59+
op1 = PrintOp('print-1', f"print {op0.output}")
60+
61+
62+
if __name__ == '__main__':
63+
TektonCompiler().compile(double_loop_with_underscore, __file__.replace('.py', '.yaml'))

‎sdk/python/tests/compiler/testdata/nested_loop_with_underscore.yaml

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Copyright 2021 kubeflow.org
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: tekton.dev/v1beta1
16+
kind: PipelineRun
17+
metadata:
18+
name: double-loop-with-underscore
19+
annotations:
20+
tekton.dev/output_artifacts: '{"print-0": [{"key": "artifacts/$PIPELINERUN/print-0/output_value.tgz",
21+
"name": "print-0-output_value", "path": "/tmp/outputs/output_value/data"}],
22+
"print-1": [{"key": "artifacts/$PIPELINERUN/print-1/output_value.tgz", "name":
23+
"print-1-output_value", "path": "/tmp/outputs/output_value/data"}]}'
24+
tekton.dev/input_artifacts: '{"print-1": [{"name": "print-0-output_value", "parent_task":
25+
"print-0"}]}'
26+
tekton.dev/artifact_bucket: mlpipeline
27+
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
28+
tekton.dev/artifact_endpoint_scheme: http://
29+
tekton.dev/artifact_items: '{"print-0": [["output_value", "$(results.output-value.path)"]],
30+
"print-1": [["output_value", "$(results.output-value.path)"]]}'
31+
sidecar.istio.io/inject: "false"
32+
tekton.dev/template: ''
33+
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
34+
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[1, 2, 3]", "name":
35+
"param_a", "optional": true, "type": "JsonArray"}, {"default": "[\"a\", \"b\",
36+
\"c\"]", "name": "param_b", "optional": true, "type": "JsonArray"}], "name":
37+
"double_loop_with_underscore"}'
38+
labels:
39+
pipelines.kubeflow.org/pipelinename: ''
40+
pipelines.kubeflow.org/generation: ''
41+
spec:
42+
params:
43+
- name: param_a
44+
value: '[1, 2, 3]'
45+
- name: param_b
46+
value: '["a", "b", "c"]'
47+
pipelineSpec:
48+
params:
49+
- name: param_a
50+
default: '[1, 2, 3]'
51+
- name: param_b
52+
default: '["a", "b", "c"]'
53+
tasks:
54+
- name: print-0
55+
taskSpec:
56+
steps:
57+
- name: main
58+
command:
59+
- sh
60+
- -c
61+
- |
62+
set -e
63+
echo $0 > $1
64+
- Hello!
65+
- $(results.output-value.path)
66+
image: alpine:3.6
67+
results:
68+
- name: output-value
69+
type: string
70+
description: /tmp/outputs/output_value/data
71+
metadata:
72+
labels:
73+
pipelines.kubeflow.org/cache_enabled: "true"
74+
annotations:
75+
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-0", "outputs":
76+
[{"description": "Represents an output paramter.", "name": "output_value",
77+
"type": "String"}], "version": "print-0@sha256=5dd3c506ec54281b82008ca8ec5d8142834eae18d74ce8a110a31dd6e371b40d"}'
78+
- runAfter:
79+
- print-0
80+
name: double-loop-with-underscore-for-loop-1
81+
params:
82+
- name: param_a-loop-item
83+
value: $(params.param_a)
84+
- name: param_b
85+
value: $(params.param_b)
86+
- name: param_b-loop-item
87+
value: $(params.param_b)
88+
- name: print-0-output_value
89+
value: $(tasks.print-0.results.output-value)
90+
taskSpec:
91+
apiVersion: custom.tekton.dev/v1alpha1
92+
kind: PipelineLoop
93+
spec:
94+
pipelineSpec:
95+
params:
96+
- name: param_a-loop-item
97+
type: string
98+
- name: param_b
99+
type: string
100+
- name: param_b-loop-item
101+
type: string
102+
- name: print-0-output_value
103+
type: string
104+
tasks:
105+
- name: double-loop-with-underscore-for-loop-2
106+
params:
107+
- name: param_b-loop-item
108+
value: $(params.param_b-loop-item)
109+
- name: print-0-output_value
110+
value: $(params.print-0-output_value)
111+
taskSpec:
112+
apiVersion: custom.tekton.dev/v1alpha1
113+
kind: PipelineLoop
114+
spec:
115+
pipelineSpec:
116+
params:
117+
- name: param_b-loop-item
118+
type: string
119+
- name: print-0-output_value
120+
type: string
121+
tasks:
122+
- name: print-1
123+
params:
124+
- name: print-0-output_value
125+
value: $(params.print-0-output_value)
126+
taskSpec:
127+
steps:
128+
- name: main
129+
command:
130+
- sh
131+
- -c
132+
- |
133+
set -e
134+
echo $0 > $1
135+
- print $(inputs.params.print-0-output_value)
136+
- $(results.output-value.path)
137+
image: alpine:3.6
138+
params:
139+
- name: print-0-output_value
140+
type: string
141+
results:
142+
- name: output-value
143+
type: string
144+
description: /tmp/outputs/output_value/data
145+
metadata:
146+
labels:
147+
pipelines.kubeflow.org/cache_enabled: "true"
148+
annotations:
149+
pipelines.kubeflow.org/component_spec_digest: '{"name":
150+
"print-1", "outputs": [{"description": "Represents an
151+
output paramter.", "name": "output_value", "type": "String"}],
152+
"version": "print-1@sha256=3b81342bc143f625b58ebdb01e7c83b145880dee807be35c1e16fdb835d46580"}'
153+
iterateParam: param_b-loop-item
154+
metadata:
155+
labels:
156+
pipelines.kubeflow.org/cache_enabled: "true"
157+
iterateParam: param_a-loop-item
158+
metadata:
159+
labels:
160+
pipelines.kubeflow.org/cache_enabled: "true"

‎sdk/python/tests/compiler/testdata/nested_loop_with_underscore_noninlined.yaml

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Copyright 2021 kubeflow.org
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: tekton.dev/v1beta1
16+
kind: PipelineRun
17+
metadata:
18+
name: double-loop-with-underscore
19+
annotations:
20+
tekton.dev/output_artifacts: '{"print-0": [{"key": "artifacts/$PIPELINERUN/print-0/output_value.tgz",
21+
"name": "print-0-output_value", "path": "/tmp/outputs/output_value/data"}],
22+
"print-1": [{"key": "artifacts/$PIPELINERUN/print-1/output_value.tgz", "name":
23+
"print-1-output_value", "path": "/tmp/outputs/output_value/data"}]}'
24+
tekton.dev/input_artifacts: '{"print-1": [{"name": "print-0-output_value", "parent_task":
25+
"print-0"}]}'
26+
tekton.dev/artifact_bucket: mlpipeline
27+
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
28+
tekton.dev/artifact_endpoint_scheme: http://
29+
tekton.dev/artifact_items: '{"print-0": [["output_value", "$(results.output-value.path)"]],
30+
"print-1": [["output_value", "$(results.output-value.path)"]]}'
31+
sidecar.istio.io/inject: "false"
32+
tekton.dev/template: ''
33+
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
34+
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[1, 2, 3]", "name":
35+
"param_a", "optional": true, "type": "JsonArray"}, {"default": "[\"a\", \"b\",
36+
\"c\"]", "name": "param_b", "optional": true, "type": "JsonArray"}], "name":
37+
"double_loop_with_underscore"}'
38+
tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
39+
"kind": "PipelineLoop", "metadata": {"name": "double-loop-with-underscore-for-loop-1"},
40+
"spec": {"iterateParam": "param_a-loop-item", "pipelineSpec": {"params": [{"name":
41+
"param_a-loop-item", "type": "string"}, {"name": "param_b", "type": "string"},
42+
{"name": "param_b-loop-item", "type": "string"}, {"name": "print-0-output_value",
43+
"type": "string"}], "tasks": [{"name": "double-loop-with-underscore-for-loop-2",
44+
"params": [{"name": "param_b-loop-item", "value": "$(params.param_b-loop-item)"},
45+
{"name": "print-0-output_value", "value": "$(params.print-0-output_value)"}],
46+
"taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop",
47+
"name": "double-loop-with-underscore-for-loop-2"}}]}}}, {"apiVersion": "custom.tekton.dev/v1alpha1",
48+
"kind": "PipelineLoop", "metadata": {"name": "double-loop-with-underscore-for-loop-2"},
49+
"spec": {"iterateParam": "param_b-loop-item", "pipelineSpec": {"params": [{"name":
50+
"param_b-loop-item", "type": "string"}, {"name": "print-0-output_value", "type":
51+
"string"}], "tasks": [{"name": "print-1", "params": [{"name": "print-0-output_value",
52+
"value": "$(params.print-0-output_value)"}], "taskSpec": {"metadata": {"annotations":
53+
{"pipelines.kubeflow.org/component_spec_digest": "{\"name\": \"print-1\", \"outputs\":
54+
[{\"description\": \"Represents an output paramter.\", \"name\": \"output_value\",
55+
\"type\": \"String\"}], \"version\": \"print-1@sha256=3b81342bc143f625b58ebdb01e7c83b145880dee807be35c1e16fdb835d46580\"}"},
56+
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "params": [{"name":
57+
"print-0-output_value", "type": "string"}], "results": [{"description": "/tmp/outputs/output_value/data",
58+
"name": "output-value", "type": "string"}], "steps": [{"command": ["sh", "-c",
59+
"set -e\necho $0 > $1\n", "print $(inputs.params.print-0-output_value)", "$(results.output-value.path)"],
60+
"image": "alpine:3.6", "name": "main"}]}}]}}}]'
61+
labels:
62+
pipelines.kubeflow.org/pipelinename: ''
63+
pipelines.kubeflow.org/generation: ''
64+
spec:
65+
params:
66+
- name: param_a
67+
value: '[1, 2, 3]'
68+
- name: param_b
69+
value: '["a", "b", "c"]'
70+
pipelineSpec:
71+
params:
72+
- name: param_a
73+
default: '[1, 2, 3]'
74+
- name: param_b
75+
default: '["a", "b", "c"]'
76+
tasks:
77+
- name: print-0
78+
taskSpec:
79+
steps:
80+
- name: main
81+
command:
82+
- sh
83+
- -c
84+
- |
85+
set -e
86+
echo $0 > $1
87+
- Hello!
88+
- $(results.output-value.path)
89+
image: alpine:3.6
90+
results:
91+
- name: output-value
92+
type: string
93+
description: /tmp/outputs/output_value/data
94+
metadata:
95+
labels:
96+
pipelines.kubeflow.org/cache_enabled: "true"
97+
annotations:
98+
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-0", "outputs":
99+
[{"description": "Represents an output paramter.", "name": "output_value",
100+
"type": "String"}], "version": "print-0@sha256=5dd3c506ec54281b82008ca8ec5d8142834eae18d74ce8a110a31dd6e371b40d"}'
101+
- runAfter:
102+
- print-0
103+
name: double-loop-with-underscore-for-loop-1
104+
taskRef:
105+
apiVersion: custom.tekton.dev/v1alpha1
106+
kind: PipelineLoop
107+
name: double-loop-with-underscore-for-loop-1
108+
params:
109+
- name: param_a-loop-item
110+
value: $(params.param_a)
111+
- name: param_b-loop-item
112+
value: $(params.param_b)
113+
- name: print-0-output_value
114+
value: $(tasks.print-0.results.output-value)
115+
- name: param_b
116+
value: $(params.param_b)

0 commit comments

Comments
 (0)