fix(sdk): update loop dsl to handle same argument in nested loop (kubeflow#1052)

* update loop dsl to handle same argument in nested loop

* fix lint

* fix sdk dependency

Author: Tomcli

sdk/python/kfp_tekton/tekton.py

@@ -352,6 +352,10 @@ def __init__(self,
             self.items_is_string = True
         else:
             super().__init__(loop_args=loop_args, parallelism=parallelism)
+            self.loop_args = TektonLoopArguments(
+                loop_args,
+                code=self._next_id(),
+            )
             self.items_is_string = False
 
         self.separator = None

sdk/python/tests/compiler/compiler_tests.py

@@ -190,6 +190,13 @@ def test_recur_cond_workflow(self):
     from .testdata.tekton_loop_dsl import pipeline
     self._test_pipeline_workflow(pipeline, 'tekton_loop_dsl.yaml')
 
+  def test_nested_loop_same_arg_workflow(self):
+    """
+    Test compiling a nested loop with same argument workflow.
+    """
+    from .testdata.nested_loop_same_arg import loop_multi
+    self._test_pipeline_workflow(loop_multi, 'nested_loop_same_arg.yaml')
+
   def test_loop_with_numeric_workflow(self):
     """
     Test compiling a loop with numeric inputs in workflow.

sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml

@@ -136,17 +136,17 @@ spec:
         value: $(context.pipelineRun.name)
       - name: pipelineRun-namespace
         value: $(context.pipelineRun.namespace)
-    - name: any-sequencer-looped-for-loop-1
+    - name: any-sequencer-looped-for-loop-2
       params:
-      - name: param-loop-item
+      - name: loop-item-param-1
         value: $(params.param)
       taskSpec:
         apiVersion: custom.tekton.dev/v1alpha1
         kind: PipelineLoop
         spec:
           pipelineSpec:
             params:
-            - name: param-loop-item
+            - name: loop-item-param-1
               type: string
             tasks:
             - name: print-10
@@ -234,7 +234,7 @@ spec:
                 value: $(context.pipelineRun.name)
               - name: pipelineRun-namespace
                 value: $(context.pipelineRun.namespace)
-          iterateParam: param-loop-item
+          iterateParam: loop-item-param-1
         metadata:
           labels:
             pipelines.kubeflow.org/cache_enabled: "true"

sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml

@@ -42,9 +42,9 @@ metadata:
     pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
       \"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
     tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
-      "kind": "PipelineLoop", "metadata": {"name": "any-sequencer-looped-for-loop-1"},
-      "spec": {"iterateParam": "param-loop-item", "pipelineSpec": {"params": [{"name":
-      "param-loop-item", "type": "string"}], "tasks": [{"name": "print-10", "taskSpec":
+      "kind": "PipelineLoop", "metadata": {"name": "any-sequencer-looped-for-loop-2"},
+      "spec": {"iterateParam": "loop-item-param-1", "pipelineSpec": {"params": [{"name":
+      "loop-item-param-1", "type": "string"}], "tasks": [{"name": "print-10", "taskSpec":
       {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
       "{\"name\": \"print-10\", \"outputs\": [{\"description\": \"Represents an output
       paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
@@ -168,11 +168,11 @@ spec:
         value: $(context.pipelineRun.name)
       - name: pipelineRun-namespace
         value: $(context.pipelineRun.namespace)
-    - name: any-sequencer-looped-for-loop-1
+    - name: any-sequencer-looped-for-loop-2
       taskRef:
         apiVersion: custom.tekton.dev/v1alpha1
         kind: PipelineLoop
-        name: any-sequencer-looped-for-loop-1
+        name: any-sequencer-looped-for-loop-2
       params:
-      - name: param-loop-item
+      - name: loop-item-param-1
         value: $(params.param)

sdk/python/tests/compiler/testdata/loop_with_enumerate_basic.yaml

@@ -58,50 +58,48 @@ spec:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Produce list",
               "outputs": [{"name": "data_list", "type": "List"}], "version": "Produce
               list@sha256=3d80b314916a06826def6fa78eab66cd69199497d7d92a8d199bd5d906706abe"}'
-    - runAfter:
-      - produce-list
-      name: loop-with-enumerate-basic-for-loop-1
+    - name: loop-with-enumerate-basic-for-loop-2
       params:
-      - name: produce-list-data_list-loop-item
+      - name: loop-item-param-1
         value: $(tasks.produce-list.results.data-list)
       taskSpec:
         apiVersion: custom.tekton.dev/v1alpha1
         kind: PipelineLoop
         spec:
           pipelineSpec:
             params:
-            - name: iteration-number-2
+            - name: iteration-number-3
               type: string
-            - name: produce-list-data_list-loop-item
+            - name: loop-item-param-1
               type: string
             tasks:
             - name: consume-data
               params:
-              - name: iteration-number-2
-                value: $(params.iteration-number-2)
-              - name: produce-list-data_list-loop-item
-                value: $(params.produce-list-data_list-loop-item)
+              - name: iteration-number-3
+                value: $(params.iteration-number-3)
+              - name: loop-item-param-1
+                value: $(params.loop-item-param-1)
               taskSpec:
                 steps:
                 - name: main
                   command:
                   - echo
-                  - $(inputs.params.iteration-number-2)
-                  - $(inputs.params.produce-list-data_list-loop-item)
+                  - $(inputs.params.iteration-number-3)
+                  - $(inputs.params.loop-item-param-1)
                   image: busybox
                 params:
-                - name: iteration-number-2
+                - name: iteration-number-3
                   type: string
-                - name: produce-list-data_list-loop-item
+                - name: loop-item-param-1
                   type: string
                 metadata:
                   labels:
                     pipelines.kubeflow.org/cache_enabled: "true"
                   annotations:
                     pipelines.kubeflow.org/component_spec_digest: '{"name": "Consume
                       data", "outputs": [], "version": "Consume data@sha256=748cfc9398fb5667535b2ae5940d7947826385c62858c6b1b637b527d9b57e6e"}'
-          iterationNumberParam: iteration-number-2
-          iterateParam: produce-list-data_list-loop-item
+          iterationNumberParam: iteration-number-3
+          iterateParam: loop-item-param-1
         metadata:
           labels:
             pipelines.kubeflow.org/cache_enabled: "true"

sdk/python/tests/compiler/testdata/loop_with_enumerate_basic_noninlined.yaml

@@ -31,20 +31,18 @@ metadata:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Test pipeline to verify
       functions of par loop.", "name": "loop-with-enumerate-basic"}'
     tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
-      "kind": "PipelineLoop", "metadata": {"name": "loop-with-enumerate-basic-for-loop-1"},
-      "spec": {"iterateParam": "produce-list-data_list-loop-item", "iterationNumberParam":
-      "iteration-number-2", "pipelineSpec": {"params": [{"name": "iteration-number-2",
-      "type": "string"}, {"name": "produce-list-data_list-loop-item", "type": "string"}],
-      "tasks": [{"name": "consume-data", "params": [{"name": "iteration-number-2",
-      "value": "$(params.iteration-number-2)"}, {"name": "produce-list-data_list-loop-item",
-      "value": "$(params.produce-list-data_list-loop-item)"}], "taskSpec": {"metadata":
-      {"annotations": {"pipelines.kubeflow.org/component_spec_digest": "{\"name\":
-      \"Consume data\", \"outputs\": [], \"version\": \"Consume data@sha256=748cfc9398fb5667535b2ae5940d7947826385c62858c6b1b637b527d9b57e6e\"}"},
+      "kind": "PipelineLoop", "metadata": {"name": "loop-with-enumerate-basic-for-loop-2"},
+      "spec": {"iterateParam": "loop-item-param-1", "iterationNumberParam": "iteration-number-3",
+      "pipelineSpec": {"params": [{"name": "iteration-number-3", "type": "string"},
+      {"name": "loop-item-param-1", "type": "string"}], "tasks": [{"name": "consume-data",
+      "params": [{"name": "iteration-number-3", "value": "$(params.iteration-number-3)"},
+      {"name": "loop-item-param-1", "value": "$(params.loop-item-param-1)"}], "taskSpec":
+      {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"Consume data\", \"outputs\": [], \"version\": \"Consume data@sha256=748cfc9398fb5667535b2ae5940d7947826385c62858c6b1b637b527d9b57e6e\"}"},
       "labels": {"pipelines.kubeflow.org/cache_enabled": "true"}}, "params": [{"name":
-      "iteration-number-2", "type": "string"}, {"name": "produce-list-data_list-loop-item",
-      "type": "string"}], "steps": [{"command": ["echo", "$(inputs.params.iteration-number-2)",
-      "$(inputs.params.produce-list-data_list-loop-item)"], "image": "busybox", "name":
-      "main"}]}}]}}}]'
+      "iteration-number-3", "type": "string"}, {"name": "loop-item-param-1", "type":
+      "string"}], "steps": [{"command": ["echo", "$(inputs.params.iteration-number-3)",
+      "$(inputs.params.loop-item-param-1)"], "image": "busybox", "name": "main"}]}}]}}}]'
   labels:
     pipelines.kubeflow.org/pipelinename: ''
     pipelines.kubeflow.org/generation: ''
@@ -73,13 +71,11 @@ spec:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Produce list",
               "outputs": [{"name": "data_list", "type": "List"}], "version": "Produce
               list@sha256=3d80b314916a06826def6fa78eab66cd69199497d7d92a8d199bd5d906706abe"}'
-    - runAfter:
-      - produce-list
-      name: loop-with-enumerate-basic-for-loop-1
+    - name: loop-with-enumerate-basic-for-loop-2
       taskRef:
         apiVersion: custom.tekton.dev/v1alpha1
         kind: PipelineLoop
-        name: loop-with-enumerate-basic-for-loop-1
+        name: loop-with-enumerate-basic-for-loop-2
       params:
-      - name: produce-list-data_list-loop-item
+      - name: loop-item-param-1
         value: $(tasks.produce-list.results.data-list)