Merge branch 'master' into grpc-options

Author: liferoad

.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml

@@ -73,6 +73,11 @@ jobs:
           - "python:container:distroless:py311:docker"
           - "python:container:distroless:py312:docker"
           - "python:container:distroless:py313:docker"
+          - "python:container:ml:py39:docker"
+          - "python:container:ml:py310:docker"
+          - "python:container:ml:py311:docker"
+          - "python:container:ml:py312:docker"
+          - "python:container:ml:py313:docker"
           - "java:expansion-service:container:docker"
     steps:
       - uses: actions/checkout@v4

.github/workflows/build_release_candidate.yml

@@ -259,14 +259,14 @@ jobs:
         # Any task which is skipped from a broader task must be explicitly included in this list to avoid accidentally missing new
         # tasks as they are added.
         images_to_publish: [
-          {"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push39", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push310", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push311", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push312", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:pushAll", "skip_flags": "-Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
-          {"gradle_task": ":pushAllSdkDockerImages", "skip_flags": "-Pskip-python-images"},
-          {"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
+          {"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:pushAll", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless -Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
+          {"gradle_task": ":pushAllSdkDockerImages", "include_skip_flags": "-Pskip-python-images"},
+          {"gradle_task": ":pushAllDockerImages", "include_skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
         ]
     steps:
       - name: Checkout
@@ -298,7 +298,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Push docker images
-        run: ./gradlew ${{ matrix.images_to_publish.gradle_task }} -PisRelease -Pdocker-pull-licenses -Pprune-images ${{ matrix.images_to_publish.skip_flags }} -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} --no-daemon --no-parallel
+        run: ./gradlew ${{ matrix.images_to_publish.gradle_task }} -PisRelease -Pdocker-pull-licenses -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} --no-daemon --no-parallel
 
   beam_site_pr:
     if:  ${{ fromJson(github.event.inputs.STAGE).beam_site_pr == 'yes'}}

.github/workflows/republish_released_docker_containers.yml

@@ -46,14 +46,14 @@ jobs:
         # Any task which is skipped from a broader task must be explicitly included in this list to avoid accidentally missing new
         # tasks as they are added.
         images_to_publish: [
-          {"gradle_task": ":pushAllRunnersDockerImages", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push39", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push310", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push311", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:push312", "skip_flags": ""},
-          {"gradle_task": ":sdks:python:container:pushAll", "skip_flags": "-Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
-          {"gradle_task": ":pushAllSdkDockerImages", "skip_flags": "-Pskip-python-images"},
-          {"gradle_task": ":pushAllDockerImages", "skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
+          {"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"},
+          {"gradle_task": ":sdks:python:container:pushAll", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless -Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"},
+          {"gradle_task": ":pushAllSdkDockerImages", "include_skip_flags": "-Pskip-python-images"},
+          {"gradle_task": ":pushAllDockerImages", "include_skip_flags": "-Pskip-runner-images -Pskip-sdk-images"}
         ]
     steps:
       - name: Checkout
@@ -94,7 +94,7 @@ jobs:
           ./gradlew ${{ matrix.images_to_publish.gradle_task }} \
           -PisRelease \
           -Pdocker-pull-licenses \
-          -Pprune-images ${{ matrix.images_to_publish.skip_flags }} \
+          -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} \
           -Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \
           -Pdocker-tag-list=${{ env.release }},${{ github.sha }},$(date +'%Y-%m-%d') \
           --no-daemon \

CHANGES.md

@@ -73,6 +73,8 @@
 ## New Features / Improvements
 
 * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Python examples added for Milvus search enrichment handler on [Beam Website](https://beam.apache.org/documentation/transforms/python/elementwise/enrichment-milvus/)
+  including jupyter notebook example (Python) ([#36176](https://github.com/apache/beam/issues/36176)).
 
 ## Breaking Changes
 
@@ -112,17 +114,17 @@
 * Added official support for Python 3.13 ([#34869](https://github.com/apache/beam/issues/34869)).
 * Added an optional output_schema verification to all YAML transforms ([#35952](https://github.com/apache/beam/issues/35952)).
 * Support for encryption when using GroupByKey added, along with `--gbek` pipeline option to automatically replace all GroupByKey transforms (Java/Python) ([#36214](https://github.com/apache/beam/issues/36214)).
+* In Python SDK, the `--element_processing_timeout_minutes` option will also interrupt the SDK process if slowness happens during DoFn initialization, for example in `DoFn.setup()` ([#36518](https://github.com/apache/beam/issues/36518)).
 
 ## Breaking Changes
 
-* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
 * (Python) `dill` is no longer a required, default dependency for Apache Beam ([#21298](https://github.com/apache/beam/issues/21298)).
   - This change only affects pipelines that explicitly use the `pickle_library=dill` pipeline option.
   - While `dill==0.3.1.1` is still pre-installed on the official Beam SDK base images, it is no longer a direct dependency of the apache-beam Python package. This means it can be overridden by other dependencies in your environment.
   - If your pipeline uses `pickle_library=dill`, you must manually ensure `dill==0.3.1.1` is installed in both your submission and runtime environments.
     - Submission environment: Install the dill extra in your local environment `pip install apache-beam[gcpdill]`.
     - Runtime (worker) environment: Your action depends on how you manage your worker's environment.
-      - If using default containers or custom containers with the official Beam base image e.g. `FROM apache/beam_python3.10_sdk:2.69`
+      - If using default containers or custom containers with the official Beam base image e.g. `FROM apache/beam_python3.10_sdk:2.69.0`
         - Add `dill==0.3.1.1` to your worker's requirements file (e.g., requirements.txt)
         - Pass this file to your pipeline using the --requirements_file requirements.txt pipeline option (For more details see [managing Dataflow dependencies](https://cloud.google.com/dataflow/docs/guides/manage-dependencies#py-custom-containers)).
       - If custom containers with a non-Beam base image e.g. `FROM python:3.9-slim`
@@ -140,10 +142,6 @@
 * (Java) DoFn OutputReceiver now requires implementing a builder method as part of extended metadata support for elements ([#34902](https://github.com/apache/beam/issues/34902)).
 * (Java) Removed ProcessContext outputWindowedValue introduced in 2.68 that allowed setting offset and record Id. Use OutputReceiver's builder to set those field ([#36523]https://github.com/apache/beam/pull/36523).
 
-## Deprecations
-
-* X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)).
-
 ## Bugfixes
 
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).