Rebase on KFP commit 7d8e921

mprahl · mprahl · commit a4c5b1a42a55 · 2025-02-28T10:09:26.000-05:00
diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml
@@ -74,6 +74,9 @@ jobs:
 
       - name: Run tests
         id: test
+        env:
+          PULL_NUMBER: ${{ github.event.pull_request.number }}
+          REPO_NAME: ${{ github.repository }}
         run: |
           export KFP_ENDPOINT="http://localhost:8888"
           export TIMEOUT_SECONDS=2700
diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml
@@ -48,6 +48,7 @@ jobs:
       id: tests
       env:
         PULL_NUMBER: ${{ github.event.pull_request.number }}
+        REPO_NAME: ${{ github.repository }}
       run: |
         ./backend/src/v2/test/sample-test.sh
       continue-on-error: true
diff --git a/.github/workflows/kfp-sdk-runtime-tests.yml b/.github/workflows/kfp-sdk-runtime-tests.yml
@@ -28,4 +28,5 @@ jobs:
       - name: Run KFP Runtime Code Tests
         run: |
           export PULL_NUMBER="${{ github.event.inputs.pull_number || github.event.pull_request.number }}"
+          export REPO_NAME="${{ github.repository }}"
           ./test/presubmit-test-kfp-runtime-code.sh
diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml
@@ -66,10 +66,13 @@ jobs:
 
       - name: Run tests
         id: tests
+        env:
+          PULL_NUMBER: ${{ github.event.pull_request.number }}
+          REPO_NAME: ${{ github.repository }}
         run: |
           export KFP_ENDPOINT="http://localhost:8888"
           export TIMEOUT_SECONDS=2700
-          pytest ./test/sdk-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS
+          pytest -v -n 5 ./test/sdk-execution-tests/sdk_execution_tests.py
         continue-on-error: true
 
       - name: Collect failed logs
diff --git a/OWNERS b/OWNERS
@@ -16,4 +16,4 @@ reviewers:
   - VaniHaripriya
   - mprahl
 emeritus_approvers:
-  - harshad16
+  - harshad16
diff --git a/backend/src/v2/test/sample-test.sh b/backend/src/v2/test/sample-test.sh
@@ -23,10 +23,12 @@ python3 -m pip install -r ./requirements-sample-test.txt
 
 popd
 
+REPO_NAME="${REPO_NAME:-kubeflow/pipelines}"
+
 if [[ -n "${PULL_NUMBER}" ]]; then
-  export KFP_PACKAGE_PATH="git+https://github.com/opendatahub-io/data-science-pipelines@refs/pull/${PULL_NUMBER}/merge#egg=kfp&subdirectory=sdk/python"
+  export KFP_PACKAGE_PATH="git+https://github.com/${REPO_NAME}@refs/pull/${PULL_NUMBER}/merge#egg=kfp&subdirectory=sdk/python"
 else
-  export KFP_PACKAGE_PATH='git+https://github.com/opendatahub-io/data-science-pipelines#egg=kfp&subdirectory=sdk/python'
+  export KFP_PACKAGE_PATH="git+https://github.com/${REPO_NAME}#egg=kfp&subdirectory=sdk/python"
 fi
 
 python3 -m pip install $KFP_PACKAGE_PATH
diff --git a/backend/src/v2/test/scripts/ci-env.sh b/backend/src/v2/test/scripts/ci-env.sh
@@ -18,11 +18,12 @@ GCS_ROOT="gs://${PROJECT}/${COMMIT_SHA}/v2-sample-test"
 GCR_ROOT="gcr.io/${PROJECT}/${COMMIT_SHA}/v2-sample-test"
 # This is kfp-ci endpoint.
 HOST=${HOST:-"https://$(curl https://raw.githubusercontent.com/kubeflow/testing/master/test-infra/kfp/endpoint)"}
+REPO_NAME="${REPO_NAME:-kubeflow/pipelines}"
 
 if [[ -z "${PULL_NUMBER}" ]]; then
-  KFP_PACKAGE_PATH='git+https://github.com/kubeflow/pipelines\#egg=kfp&subdirectory=sdk/python'
+  KFP_PACKAGE_PATH="git+https://github.com/${REPO_NAME}#egg=kfp&subdirectory=sdk/python"
 else
-  KFP_PACKAGE_PATH='git+https://github.com/kubeflow/pipelines@refs/pull/${PULL_NUMBER}/merge\#egg=kfp&subdirectory=sdk/python'
+  KFP_PACKAGE_PATH="git+https://github.com/${REPO_NAME}@refs/pull/${PULL_NUMBER}/merge#egg=kfp&subdirectory=sdk/python"
 fi
 
 cat <<EOF >kfp-ci.env
diff --git a/docs/versions.json b/docs/versions.json
@@ -1,12 +1,17 @@
 [
     {
-        "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.11.0/",
-        "title": "2.11.0",
+        "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.12.0/",
+        "title": "2.12.0",
         "aliases": [
             "stable",
             "latest"
         ]
     },
+    {
+        "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.11.0/",
+        "title": "2.11.0",
+        "aliases": []
+    },
     {
         "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.10.1/",
         "title": "2.10.1",
diff --git a/kubernetes_platform/python/README.md b/kubernetes_platform/python/README.md
@@ -289,3 +289,21 @@ def pipeline():
     task = simple_task()
     kubernetes.set_image_pull_policy(task, "Always")
 ```
+
+### ImagePullSecrets: Set secrets to authenticate image pulls
+```python
+from kfp import dsl
+from kfp import kubernetes
+
+@dsl.container_component
+def hello():
+    return dsl.ContainerSpec(
+        image='some-private-image:tag',
+        command=['echo', 'hello']
+    )
+
+@dsl.pipeline
+def pipeline():
+    task = hello()
+    kubernetes.set_image_pull_secrets(task, ['secret-name'])
+```
diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md
@@ -10,6 +10,26 @@
 
 ## Documentation updates
 
+# 2.12.0
+
+## Features
+
+* Add support for placeholders in resource limits [\#11501](https://github.com/kubeflow/pipelines/pull/11501)
+* Introduce cache_key to sdk [\#11466](https://github.com/kubeflow/pipelines/pull/11466)
+* Add support for importing models stored in the Modelcar format (sidecar) [\#11606](https://github.com/kubeflow/pipelines/pull/11606)
+
+## Breaking changes
+
+## Deprecations
+
+## Bug fixes and other changes
+
+* dsl.component docstring typo [\#11547](https://github.com/kubeflow/pipelines/pull/11547)
+* Update broken api-connect link [\#11521](https://github.com/kubeflow/pipelines/pull/11521)
+* Fix kfp-sdk-test for different python versions [\#11559](https://github.com/kubeflow/pipelines/pull/11559)
+
+## Documentation updates
+
 # 2.11.0
 
 ## Features
diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py
@@ -16,7 +16,7 @@
 # https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
 __path__ = __import__('pkgutil').extend_path(__path__, __name__)
 
-__version__ = '2.11.0'
+__version__ = '2.12.0'
 
 import sys
 import warnings
diff --git a/test/kfp-kubernetes-execution-tests/sdk_execution_tests.py b/test/kfp-kubernetes-execution-tests/sdk_execution_tests.py
@@ -110,10 +110,11 @@ def run(test_case: TestCase) -> Tuple[str, client.client.RunPipelineResult]:
 
 
 def get_kfp_package_path() -> str:
+    repo_name = os.environ.get('REPO_NAME', 'kubeflow/pipelines')
     if os.environ.get('PULL_NUMBER') is not None:
-        path = f'git+https://github.com/kubeflow/pipelines.git@refs/pull/{os.environ.get("PULL_NUMBER")}/merge#subdirectory=sdk/python'
+        path = f'git+https://github.com/{repo_name}.git@refs/pull/{os.environ["PULL_NUMBER"]}/merge#subdirectory=sdk/python'
     else:
-        path = 'git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python'
+        path = f'git+https://github.com/{repo_name}.git@master#subdirectory=sdk/python'
     print(f'Using the following KFP package path for tests: {path}')
     return path
 
diff --git a/test/presubmit-tests-sdk.sh b/test/presubmit-tests-sdk.sh
@@ -36,7 +36,8 @@ pytest sdk/python/kfp --cov=kfp
 set +x
 # export COVERALLS_REPO_TOKEN=$(gsutil cat gs://ml-pipeline-test-keys/coveralls_repo_token)
 set -x
-REPO_BASE="https://github.com/kubeflow/pipelines"
+REPO_NAME="${REPO_NAME:-kubeflow/pipelines}"
+REPO_BASE="https://github.com/${REPO_NAME}"
 export COVERALLS_SERVICE_NAME="prow"
 export COVERALLS_SERVICE_JOB_ID=$PROW_JOB_ID
 export CI_PULL_REQUEST="$REPO_BASE/pull/$PULL_NUMBER"
diff --git a/test/sdk-execution-tests/requirements.txt b/test/sdk-execution-tests/requirements.txt
@@ -1,3 +1,3 @@
 sdk/python
 pytest==8.3.2
-pytest-asyncio-cooperative==0.37.0
+pytest-xdist==2.5.0
diff --git a/test/sdk-execution-tests/sdk_execution_tests.py b/test/sdk-execution-tests/sdk_execution_tests.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import asyncio
 import dataclasses
 import functools
 import os
@@ -100,10 +99,11 @@ def run(test_case: TestCase) -> Tuple[str, client.client.RunPipelineResult]:
 
 
 def get_kfp_package_path() -> str:
+    repo_name = os.environ.get('REPO_NAME', 'kubeflow/pipelines')
     if os.environ.get('PULL_NUMBER') is not None:
-        path = f'git+https://github.com/kubeflow/pipelines.git@refs/pull/{os.environ.get("PULL_NUMBER")}/merge#subdirectory=sdk/python'
+        path = f'git+https://github.com/{repo_name}.git@refs/pull/{os.environ["PULL_NUMBER"]}/merge#subdirectory=sdk/python'
     else:
-        path = 'git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python'
+        path = f'git+https://github.com/{repo_name}.git@master#subdirectory=sdk/python'
     print(f'Using the following KFP package path for tests: {path}')
     return path
 
@@ -112,18 +112,15 @@ def get_kfp_package_path() -> str:
     dsl.component, kfp_package_path=get_kfp_package_path())
 
 
-@pytest.mark.asyncio_cooperative
 @pytest.mark.parametrize('test_case', create_test_case_parameters())
-async def test(test_case: TestCase) -> None:
-    """Asynchronously runs all samples and test that they succeed."""
-    event_loop = asyncio.get_running_loop()
+def test(test_case: TestCase) -> None:
     try:
         run_url, run_result = run(test_case)
     except Exception as e:
         raise RuntimeError(
             f'Error triggering pipeline {test_case.name}.') from e
 
-    api_run = await event_loop.run_in_executor(None, wait, run_result)
+    api_run = wait(run_result)
     assert api_run.state == test_case.expected_state, f'Pipeline {test_case.name} ended with incorrect status: {api_run.state}. More info: {run_url}'
 
 

Original file line number	Diff line number	Diff line change
`@@ -1,12 +1,17 @@`
`1`	`1`	`[`
`2`	`2`	`{`
`3`		`- "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.11.0/",`
`4`		`- "title": "2.11.0",`
	`3`	`+ "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.12.0/",`
	`4`	`+ "title": "2.12.0",`
`5`	`5`	`"aliases": [`
`6`	`6`	`"stable",`
`7`	`7`	`"latest"`
`8`	`8`	`]`
`9`	`9`	`},`
	`10`	`+ {`
	`11`	`+ "version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.11.0/",`
	`12`	`+ "title": "2.11.0",`
	`13`	`+ "aliases": []`
	`14`	`+ },`
`10`	`15`	`{`
`11`	`16`	`"version": "https://kubeflow-pipelines.readthedocs.io/en/sdk-2.10.1/",`
`12`	`17`	`"title": "2.10.1",`