Skip to content

Commit e6027dd

Browse files
TomcliTomcli
authored
fix(sdk): fix nested param logic for pl (kubeflow#1091)
1 parent 0ab85f9 commit e6027dd

15 files changed

+103
-69
lines changed

‎sdk/python/kfp_tekton/compiler/_tekton_handler.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -442,14 +442,15 @@ def process_inline_cr_field(field_name):
442442
for custom_param in custom_task_cr['spec']['pipelineSpec']['params']:
443443
all_params.append(''.join(['$(params.', custom_param['name'], ')']))
444444
for global_task_value in global_task_values:
445-
if global_task_value not in all_params:
445+
if global_task_value not in all_params and \
446+
re.findall('\$\(params.([^ \t\n.:,;\{\}]+)\)', global_task_value)[0] not in nested_loop_counter_params:
446447
all_params.append(global_task_value)
447448
custom_task_cr['spec']['pipelineSpec']['params'].append(
448449
{'name': re.findall('\$\(params.([^ \t\n.:,;\{\}]+)\)', global_task_value)[0],
449450
'type': 'string'}
450451
)
451452
for task in tasks:
452-
if task['name'] == nested_custom_task['father_ct'] and global_task_value not in global_task_values:
453+
if task['name'] == nested_custom_task['father_ct']:
453454
task['params'].append(
454455
{'name': re.findall('\$\(params.([^ \t\n.:,;\{\}]+)\)', global_task_value)[0],
455456
'value': global_task_value}

‎sdk/python/tests/compiler/testdata/loop_with_enumerate_withitem_multi_nested.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,12 @@ spec:
5050
params:
5151
- name: loop-item-param-2
5252
value: '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]'
53+
- name: my_pipe_param
54+
value: $(params.my_pipe_param)
5355
- name: my_pipe_param-loop-item
5456
value: $(params.my_pipe_param)
57+
- name: my_pipe_param3
58+
value: $(params.my_pipe_param3)
5559
- name: my_pipe_param3-loop-item
5660
value: $(params.my_pipe_param3)
5761
taskSpec:
@@ -140,6 +144,8 @@ spec:
140144
value: $(params.loop-item-param-2-subvar-b)
141145
- name: my_pipe_param-loop-item
142146
value: $(params.my_pipe_param-loop-item)
147+
- name: my_pipe_param3
148+
value: $(params.my_pipe_param3)
143149
- name: my_pipe_param3-loop-item
144150
value: $(params.my_pipe_param3-loop-item)
145151
taskSpec:
@@ -199,6 +205,8 @@ spec:
199205
value: $(params.loop-item-param-2-subvar-b)
200206
- name: loop-item-param-7
201207
value: '[4, 5]'
208+
- name: my_pipe_param3
209+
value: $(params.my_pipe_param3)
202210
- name: my_pipe_param3-loop-item
203211
value: $(params.my_pipe_param3-loop-item)
204212
taskSpec:

‎sdk/python/tests/compiler/testdata/loop_with_enumerate_withitem_multi_nested_noninlined.yaml

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,20 @@ metadata:
5858
{"name": "loop-item-param-2-subvar-a", "value": "$(params.loop-item-param-2-subvar-a)"},
5959
{"name": "loop-item-param-2-subvar-b", "value": "$(params.loop-item-param-2-subvar-b)"},
6060
{"name": "my_pipe_param-loop-item", "value": "$(params.my_pipe_param-loop-item)"},
61-
{"name": "my_pipe_param3-loop-item", "value": "$(params.my_pipe_param3-loop-item)"}],
62-
"taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop",
63-
"name": "withitem-multiple-nesting-pipeline-for-loop-5"}}]}}}, {"apiVersion":
64-
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata": {"name": "withitem-multiple-nesting-pipeline-for-loop-5"},
65-
"spec": {"iterateParam": "my_pipe_param-loop-item", "pipelineSpec": {"params":
66-
[{"name": "iteration-number-4", "type": "string"}, {"name": "loop-item-param-2-subvar-a",
67-
"type": "string"}, {"name": "loop-item-param-2-subvar-b", "type": "string"},
68-
{"name": "my_pipe_param-loop-item", "type": "string"}, {"name": "my_pipe_param3",
69-
"type": "string"}, {"name": "my_pipe_param3-loop-item", "type": "string"}],
70-
"tasks": [{"name": "my-1st-inner-coop", "params": [{"name": "loop-item-param-2-subvar-a",
71-
"value": "$(params.loop-item-param-2-subvar-a)"}, {"name": "my_pipe_param-loop-item",
72-
"value": "$(params.my_pipe_param-loop-item)"}], "taskSpec": {"metadata": {"annotations":
73-
{"pipelines.kubeflow.org/component_spec_digest": "{\"name\": \"my-1st-inner-coop\",
74-
\"outputs\": [], \"version\": \"my-1st-inner-coop@sha256=feaa8a218c3caaa76fa1154bcca9fc485286e72814986c35d08edcd28d2a50b9\"}"},
61+
{"name": "my_pipe_param3", "value": "$(params.my_pipe_param3)"}, {"name": "my_pipe_param3-loop-item",
62+
"value": "$(params.my_pipe_param3-loop-item)"}], "taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1",
63+
"kind": "PipelineLoop", "name": "withitem-multiple-nesting-pipeline-for-loop-5"}}]}}},
64+
{"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata":
65+
{"name": "withitem-multiple-nesting-pipeline-for-loop-5"}, "spec": {"iterateParam":
66+
"my_pipe_param-loop-item", "pipelineSpec": {"params": [{"name": "iteration-number-4",
67+
"type": "string"}, {"name": "loop-item-param-2-subvar-a", "type": "string"},
68+
{"name": "loop-item-param-2-subvar-b", "type": "string"}, {"name": "my_pipe_param-loop-item",
69+
"type": "string"}, {"name": "my_pipe_param3", "type": "string"}, {"name": "my_pipe_param3-loop-item",
70+
"type": "string"}], "tasks": [{"name": "my-1st-inner-coop", "params": [{"name":
71+
"loop-item-param-2-subvar-a", "value": "$(params.loop-item-param-2-subvar-a)"},
72+
{"name": "my_pipe_param-loop-item", "value": "$(params.my_pipe_param-loop-item)"}],
73+
"taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
74+
"{\"name\": \"my-1st-inner-coop\", \"outputs\": [], \"version\": \"my-1st-inner-coop@sha256=feaa8a218c3caaa76fa1154bcca9fc485286e72814986c35d08edcd28d2a50b9\"}"},
7575
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "params": [{"name":
7676
"loop-item-param-2-subvar-a", "type": "string"}, {"name": "my_pipe_param-loop-item",
7777
"type": "string"}], "steps": [{"args": ["set -e\necho \"op1-1\" \"$0\" \"$1\"\n",
@@ -80,10 +80,11 @@ metadata:
8080
{"name": "withitem-multiple-nesting-pipeline-for-loop-8", "params": [{"name":
8181
"iteration-number-4", "value": "$(params.iteration-number-4)"}, {"name": "loop-item-param-2-subvar-b",
8282
"value": "$(params.loop-item-param-2-subvar-b)"}, {"name": "loop-item-param-7",
83-
"value": "[4, 5]"}, {"name": "my_pipe_param3-loop-item", "value": "$(params.my_pipe_param3-loop-item)"}],
84-
"taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop",
85-
"name": "withitem-multiple-nesting-pipeline-for-loop-8"}}]}}}, {"apiVersion":
86-
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata": {"name": "withitem-multiple-nesting-pipeline-for-loop-8"},
83+
"value": "[4, 5]"}, {"name": "my_pipe_param3-loop-item", "value": "$(params.my_pipe_param3-loop-item)"},
84+
{"name": "my_pipe_param3", "value": "$(params.my_pipe_param3)"}], "taskRef":
85+
{"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "name":
86+
"withitem-multiple-nesting-pipeline-for-loop-8"}}]}}}, {"apiVersion": "custom.tekton.dev/v1alpha1",
87+
"kind": "PipelineLoop", "metadata": {"name": "withitem-multiple-nesting-pipeline-for-loop-8"},
8788
"spec": {"iterateParam": "loop-item-param-7", "iterationNumberParam": "iteration-number-9",
8889
"pipelineSpec": {"params": [{"name": "iteration-number-4", "type": "string"},
8990
{"name": "iteration-number-9", "type": "string"}, {"name": "loop-item-param-2-subvar-b",
@@ -144,7 +145,11 @@ spec:
144145
params:
145146
- name: loop-item-param-2
146147
value: '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]'
148+
- name: my_pipe_param
149+
value: $(params.my_pipe_param)
147150
- name: my_pipe_param-loop-item
148151
value: $(params.my_pipe_param)
152+
- name: my_pipe_param3
153+
value: $(params.my_pipe_param3)
149154
- name: my_pipe_param3-loop-item
150155
value: $(params.my_pipe_param3)

‎sdk/python/tests/compiler/testdata/nested_loop_counter.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ spec:
4848
params:
4949
- name: loop-item-param-1
5050
type: string
51-
- name: loop-item-param-3
52-
type: string
5351
tasks:
5452
- name: loop-3-range-for-loop-4
5553
params:

‎sdk/python/tests/compiler/testdata/nested_loop_counter_noninlined.yaml

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,17 @@ metadata:
3131
tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
3232
"kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-2"}, "spec":
3333
{"iterateNumeric": "loop-item-param-1", "pipelineSpec": {"params": [{"name":
34-
"loop-item-param-1", "type": "string"}, {"name": "loop-item-param-3", "type":
35-
"string"}], "tasks": [{"name": "loop-3-range-for-loop-4", "params": [{"name":
36-
"from", "value": "1"}, {"name": "loop-item-param-1", "value": "$(params.loop-item-param-1)"},
34+
"loop-item-param-1", "type": "string"}], "tasks": [{"name": "loop-3-range-for-loop-4",
35+
"params": [{"name": "from", "value": "1"}, {"name": "loop-item-param-1", "value":
36+
"$(params.loop-item-param-1)"}, {"name": "to", "value": "2"}], "taskRef": {"apiVersion":
37+
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "name": "loop-3-range-for-loop-4"}}]}}},
38+
{"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata":
39+
{"name": "loop-3-range-for-loop-4"}, "spec": {"iterateNumeric": "loop-item-param-3",
40+
"pipelineSpec": {"params": [{"name": "loop-item-param-1", "type": "string"},
41+
{"name": "loop-item-param-3", "type": "string"}], "tasks": [{"name": "loop-3-range-for-loop-6",
42+
"params": [{"name": "from", "value": "1"}, {"name": "loop-item-param-1", "value":
43+
"$(params.loop-item-param-1)"}, {"name": "loop-item-param-3", "value": "$(params.loop-item-param-3)"},
3744
{"name": "to", "value": "2"}], "taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1",
38-
"kind": "PipelineLoop", "name": "loop-3-range-for-loop-4"}}]}}}, {"apiVersion":
39-
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-4"},
40-
"spec": {"iterateNumeric": "loop-item-param-3", "pipelineSpec": {"params": [{"name":
41-
"loop-item-param-1", "type": "string"}, {"name": "loop-item-param-3", "type":
42-
"string"}], "tasks": [{"name": "loop-3-range-for-loop-6", "params": [{"name":
43-
"from", "value": "1"}, {"name": "loop-item-param-1", "value": "$(params.loop-item-param-1)"},
44-
{"name": "loop-item-param-3", "value": "$(params.loop-item-param-3)"}, {"name":
45-
"to", "value": "2"}], "taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1",
4645
"kind": "PipelineLoop", "name": "loop-3-range-for-loop-6"}}]}}}, {"apiVersion":
4746
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-6"},
4847
"spec": {"iterateNumeric": "loop-item-param-5", "pipelineSpec": {"params": [{"name":

‎sdk/python/tests/compiler/testdata/nested_loop_global_param.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ spec:
119119
pipelines.kubeflow.org/cache_enabled: "true"
120120
- name: empty-loop-for-loop-3
121121
params:
122+
- name: param
123+
value: $(params.param)
122124
- name: param-loop-item
123125
value: $(params.param)
124126
taskSpec:

‎sdk/python/tests/compiler/testdata/nested_loop_global_param_noninlined.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,5 @@ spec:
9999
params:
100100
- name: param-loop-item
101101
value: $(params.param)
102+
- name: param
103+
value: $(params.param)

‎sdk/python/tests/compiler/testdata/nested_loop_param.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ spec:
4848
params:
4949
- name: loop-item-param-2
5050
type: string
51-
- name: loop-item-param-6
52-
type: string
5351
tasks:
5452
- name: loop-3-range-for-loop-7
5553
params:

‎sdk/python/tests/compiler/testdata/nested_loop_param_noninlined.yaml

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -31,31 +31,30 @@ metadata:
3131
tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
3232
"kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-4"}, "spec":
3333
{"iterateParam": "loop-item-param-2", "iterateParamStringSeparator": "loop-item-param-3",
34-
"pipelineSpec": {"params": [{"name": "loop-item-param-2", "type": "string"},
35-
{"name": "loop-item-param-6", "type": "string"}], "tasks": [{"name": "loop-3-range-for-loop-7",
36-
"params": [{"name": "loop-item-param-2", "value": "$(params.loop-item-param-2)"},
37-
{"name": "loop-item-param-6", "value": "[A,B,C]"}], "taskRef": {"apiVersion":
38-
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "name": "loop-3-range-for-loop-7"}}]}}},
39-
{"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata":
40-
{"name": "loop-3-range-for-loop-7"}, "spec": {"iterateParam": "loop-item-param-6",
41-
"pipelineSpec": {"params": [{"name": "loop-item-param-2", "type": "string"},
42-
{"name": "loop-item-param-6", "type": "string"}], "tasks": [{"name": "loop-3-range-for-loop-11",
43-
"params": [{"name": "loop-item-param-10", "value": ","}, {"name": "loop-item-param-2",
34+
"pipelineSpec": {"params": [{"name": "loop-item-param-2", "type": "string"}],
35+
"tasks": [{"name": "loop-3-range-for-loop-7", "params": [{"name": "loop-item-param-2",
4436
"value": "$(params.loop-item-param-2)"}, {"name": "loop-item-param-6", "value":
45-
"$(params.loop-item-param-6)"}, {"name": "loop-item-param-9", "value": "1,2,3"}],
46-
"taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1", "kind": "PipelineLoop",
47-
"name": "loop-3-range-for-loop-11"}}]}}}, {"apiVersion": "custom.tekton.dev/v1alpha1",
48-
"kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-11"}, "spec":
49-
{"iterateParam": "loop-item-param-9", "iterateParamStringSeparator": "loop-item-param-10",
50-
"pipelineSpec": {"params": [{"name": "loop-item-param-2", "type": "string"},
51-
{"name": "loop-item-param-6", "type": "string"}, {"name": "loop-item-param-9",
52-
"type": "string"}], "tasks": [{"name": "print", "params": [{"name": "loop-item-param-2",
53-
"value": "$(params.loop-item-param-2)"}, {"name": "loop-item-param-6", "value":
54-
"$(params.loop-item-param-6)"}, {"name": "loop-item-param-9", "value": "$(params.loop-item-param-9)"}],
55-
"taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
56-
"{\"name\": \"print\", \"outputs\": [{\"description\": \"Represents an output
57-
paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
58-
\"print@sha256=c6e88bb19253b3bedeb9912855f4e324700cd80285e6b625b9ebcffb58677766\"}"},
37+
"[A,B,C]"}], "taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1", "kind":
38+
"PipelineLoop", "name": "loop-3-range-for-loop-7"}}]}}}, {"apiVersion": "custom.tekton.dev/v1alpha1",
39+
"kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-7"}, "spec":
40+
{"iterateParam": "loop-item-param-6", "pipelineSpec": {"params": [{"name": "loop-item-param-2",
41+
"type": "string"}, {"name": "loop-item-param-6", "type": "string"}], "tasks":
42+
[{"name": "loop-3-range-for-loop-11", "params": [{"name": "loop-item-param-10",
43+
"value": ","}, {"name": "loop-item-param-2", "value": "$(params.loop-item-param-2)"},
44+
{"name": "loop-item-param-6", "value": "$(params.loop-item-param-6)"}, {"name":
45+
"loop-item-param-9", "value": "1,2,3"}], "taskRef": {"apiVersion": "custom.tekton.dev/v1alpha1",
46+
"kind": "PipelineLoop", "name": "loop-3-range-for-loop-11"}}]}}}, {"apiVersion":
47+
"custom.tekton.dev/v1alpha1", "kind": "PipelineLoop", "metadata": {"name": "loop-3-range-for-loop-11"},
48+
"spec": {"iterateParam": "loop-item-param-9", "iterateParamStringSeparator":
49+
"loop-item-param-10", "pipelineSpec": {"params": [{"name": "loop-item-param-2",
50+
"type": "string"}, {"name": "loop-item-param-6", "type": "string"}, {"name":
51+
"loop-item-param-9", "type": "string"}], "tasks": [{"name": "print", "params":
52+
[{"name": "loop-item-param-2", "value": "$(params.loop-item-param-2)"}, {"name":
53+
"loop-item-param-6", "value": "$(params.loop-item-param-6)"}, {"name": "loop-item-param-9",
54+
"value": "$(params.loop-item-param-9)"}], "taskSpec": {"metadata": {"annotations":
55+
{"pipelines.kubeflow.org/component_spec_digest": "{\"name\": \"print\", \"outputs\":
56+
[{\"description\": \"Represents an output paramter.\", \"name\": \"output_value\",
57+
\"type\": \"String\"}], \"version\": \"print@sha256=c6e88bb19253b3bedeb9912855f4e324700cd80285e6b625b9ebcffb58677766\"}"},
5958
"labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "params": [{"name":
6059
"loop-item-param-2", "type": "string"}, {"name": "loop-item-param-6", "type":
6160
"string"}, {"name": "loop-item-param-9", "type": "string"}], "results": [{"description":

‎sdk/python/tests/compiler/testdata/nested_loop_same_arg.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ spec:
4747
value: $(params.param)
4848
- name: loop-item-param-3
4949
value: $(params.param)
50+
- name: param
51+
value: $(params.param)
5052
taskSpec:
5153
apiVersion: custom.tekton.dev/v1alpha1
5254
kind: PipelineLoop

0 commit comments

Comments
 (0)