Kafka Consumer Group Properties

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow.json

@@ -1,4 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2,
+  "https://github.com/apache/beam/pull/34294": "noting that PR #34294 should run this test"
 }

.github/workflows/beam_PreCommit_Flink_Container.yml

@@ -77,6 +77,10 @@ env:
   HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
   JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
   ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{ github.run_id }}
+  DOCKER_REGISTRY: gcr.io
+  DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request_target' && 'gcr.io/apache-beam-testing/beam-sdk-pr' || 'gcr.io/apache-beam-testing/beam-sdk' }}
+  PYTHON_VERSION: 3.9
+  PYTHON_SDK_IMAGE_TAG: latest
 
 jobs:
   beam_PreCommit_Flink_Container:
@@ -87,7 +91,7 @@ jobs:
       github.event_name == 'pull_request_target' ||
       github.event.comment.body == 'Run Flink Container PreCommit'
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 45
+    timeout-minutes: 90
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:
       matrix:
@@ -105,6 +109,24 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           python-version: default
+      - name: GCloud Docker credential helper
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          gcloud auth configure-docker ${{ env.DOCKER_REGISTRY }}
+      - name: Set PYTHON_SDK_IMAGE_TAG unique variable based on timestamp
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: echo "PYTHON_SDK_IMAGE_TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_ENV
+      - name: Build and push to registry
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:container:py39:docker
+          arguments: |
+            -PpythonVersion=${{ env.PYTHON_VERSION }} \
+            -Pdocker-repository-root=${{ env.DOCKER_REPOSITORY_ROOT }} \
+            -Pdocker-tag=${{ env.PYTHON_SDK_IMAGE_TAG }} \
+            -PuseBuildx \
+            -Ppush-containers
       - name: Prepare test arguments
         uses: ./.github/actions/test-arguments-action
         with:
@@ -141,11 +163,11 @@ jobs:
           arguments: |
             -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
             -Prunner=FlinkRunner \
-            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --job_name=flink-tests-python-${{env.NOW_UTC}}'
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --environment_config=${{ env.DOCKER_REPOSITORY_ROOT }}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{ env.PYTHON_SDK_IMAGE_TAG }} --job_name=flink-tests-python-${{env.NOW_UTC}}'
 
       # Run a Java Combine load test to verify the Flink container
       - name: Run Flink Container Test with Java Combine
-        timeout-minutes: 10
+        timeout-minutes: 20
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:testing:load-tests:run
@@ -158,3 +180,8 @@ jobs:
         if: always()
         run: |
           ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete
+
+      - name: Cleanup Python SDK Container
+        if: ${{ always() && github.event_name == 'pull_request_target' }}
+        run: |
+          gcloud container images delete ${{ env.DOCKER_REPOSITORY_ROOT }}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{ env.PYTHON_SDK_IMAGE_TAG }} --force-delete-tags --quiet

.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt

@@ -18,6 +18,5 @@
 --parallelism=2
 --job_endpoint=localhost:8099
 --environment_type=DOCKER
---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest
 --top_count=10
 --runner=PortableRunner

.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh

@@ -22,7 +22,7 @@ set -euo pipefail
 # Clean up private registry (us.gcr.io)
 # Images more than 5 day old and not the latest (either has latest label or newest)
 
-PUBLIC_REPOSITORIES=(beam-sdk beam_portability beamgrafana beammetricssyncjenkins beammetricssyncgithub)
+PUBLIC_REPOSITORIES=(beam-sdk beam-sdk-pr beam_portability beamgrafana beammetricssyncjenkins beammetricssyncgithub)
 PRIVATE_REPOSITORIES=(java-postcommit-it python-postcommit-it jenkins github-actions)
 # set as the same as 6-week release period
 if [[ $OSTYPE == "linux-gnu"* ]]; then

.test-infra/tools/stale_k8s_workload_cleaner.sh

@@ -25,6 +25,7 @@ set -euo pipefail
 PROJECT=apache-beam-testing
 LOCATION=us-central1-a
 CLUSTER=io-datastores
+MEMSQL_CLUSTER_RESOURCE="memsqlclusters.memsql.com/sdb-cluster"
 
 function should_teardown() {
   if [[ $1 =~ ^([0-9]+)([a-z]) ]]; then
@@ -43,10 +44,12 @@ function should_teardown() {
 gcloud container clusters get-credentials io-datastores --zone us-central1-a --project apache-beam-testing
 
 while read NAME STATUS AGE; do
-  # Regex has temporary workaround to avoid trying to delete beam-performancetests-singlestoreio-* to avoid getting stuck in a terminal state
-  # See https://github.com/apache/beam/pull/33545 for context.
-  # This may be safe to remove if https://cloud.google.com/knowledge/kb/deleted-namespace-remains-in-terminating-status-000004867 has been resolved, just try it before checking in :)
-  if [[ $NAME =~ ^beam-.+(test|-it)(?!s-singlestoreio) ]] && should_teardown $AGE; then
+  if [[ $NAME =~ ^beam-.+(test|-it) ]] && should_teardown $AGE; then
+    # For namespaces containing "-singlestoreio-", remove the finalizers from the sdb-cluster resource
+    # to ensure it can be fully deleted and not block namespace removal.
+    if [[ $NAME == *-singlestoreio-* ]]; then
+      kubectl patch $MEMSQL_CLUSTER_RESOURCE -n $NAME -p '[{"op": "remove", "path": "/metadata/finalizers"}]' --type=json
+    fi
     kubectl delete namespace $NAME
   fi
 done < <( kubectl get namespaces --context=gke_${PROJECT}_${LOCATION}_${CLUSTER} )

CHANGES.md

@@ -66,13 +66,17 @@
 
 * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
 * [Java] Use API compatible with both com.google.cloud.bigdataoss:util 2.x and 3.x in BatchLoads ([#34105](https://github.com/apache/beam/pull/34105))
+* [IcebergIO] Address edge case where bundle retry following a successful data commit results in data duplication ([#34264](https://github.com/apache/beam/pull/34264))
 
 ## New Features / Improvements
 
-* Support custom coders in Reshuffle ([#29908](https://github.com/apache/beam/issues/29908), [#33356](https://github.com/apache/beam/issues/33356)).
+* [Python] Support custom coders in Reshuffle ([#29908](https://github.com/apache/beam/issues/29908), [#33356](https://github.com/apache/beam/issues/33356)).
 * [Java] Upgrade SLF4J to 2.0.16. Update default Spark version to 3.5.0. ([#33574](https://github.com/apache/beam/pull/33574))
+* [Java] Support for `--add-modules` JVM option is added through a new pipeline option `JdkAddRootModules`. This allows extending the module graph with optional modules such as SDK incubator modules. Sample usage: `<pipeline invocation> --jdkAddRootModules=jdk.incubator.vector` ([#30281](https://github.com/apache/beam/issues/30281)).
 * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
 * Managed API for [Java](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/managed/Managed.html) and [Python](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.managed.html#module-apache_beam.transforms.managed) supports [key I/O connectors](https://beam.apache.org/documentation/io/connectors/) Iceberg, Kafka, and BigQuery.
+* Prism now supports event time triggers for most common cases. ([#31438](https://github.com/apache/beam/issues/31438))
+  * Prism does not yet support triggered side inputs, or triggers on merging windows (such as session windows).
 
 ## Breaking Changes
 
@@ -91,7 +95,9 @@
 * (Java) Fixed TIME field encodings for BigQuery Storage API writes on GenericRecords ([#34059](https://github.com/apache/beam/pull/34059)).
 * (Java) Fixed a race condition in JdbcIO which could cause hangs trying to acquire a connection ([#34058](https://github.com/apache/beam/pull/34058)).
 * (Java) Fix BigQuery Storage Write compatibility with Avro 1.8 ([#34281](https://github.com/apache/beam/pull/34281)).
-* Fixed checkpoint recovery and streaming behavior in Spark Classic and Portable runner's Flatten transform by replacing queueStream with SingleEmitInputDStream ([#34080](https://github.com/apache/beam/pull/34080), [#18144](https://github.com/apache/beam/issues/18144), [#20426](https://github.com/apache/beam/issues/20426)).
+* Fixed checkpoint recovery and streaming behavior in Spark Classic and Portable runner's Flatten transform by replacing queueStream with SingleEmitInputDStream ([#34080](https://github.com/apache/beam/pull/34080), [#18144](https://github.com/apache/beam/issues/18144), [#20426](https://github.com/apache/beam/issues/20426))
+* (Java) Fixed Read caching of UnboundedReader objects to effectively cache across multiple DoFns and avoid checkpointing unstarted reader. [#34146](https://github.com/apache/beam/pull/34146) [#33901](https://github.com/apache/beam/pull/33901)
+
 
 ## Security Fixes
 * Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).
@@ -133,7 +139,7 @@
 * [Dataflow Streaming] Enable Windmill GetWork Response Batching by default ([#33847](https://github.com/apache/beam/pull/33847)).
   * With this change user workers will request batched GetWork responses from backend and backend will send multiple WorkItems in the same response proto.
   * The feature can be disabled by passing `--windmillRequestBatchedGetWorkResponse=false`
-
+* Added supports for staging arbitrary files via `--files_to_stage` flag (Python) ([#34208](https://github.com/apache/beam/pull/34208))
 ## Breaking Changes
 
 * AWS V1 I/Os have been removed (Java). As part of this, x-lang Python Kinesis I/O has been updated to consume the V2 IO and it also no longer supports setting producer_properties ([#33430](https://github.com/apache/beam/issues/33430)).

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -573,7 +573,7 @@ class BeamModulePlugin implements Plugin<Project> {
     }
 
     project.ext.useBuildx = {
-      return project.containerArchitectures() != [project.nativeArchitecture()]
+      return (project.containerArchitectures() != [project.nativeArchitecture()]) || project.rootProject.hasProperty("useBuildx")
     }
 
     /** ***********************************************************************************************/

examples/notebooks/beam-ml/run_inference_vllm.ipynb

@@ -338,9 +338,9 @@
         "RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install --upgrade pip\n",
         "\n",
         "# Copy the Apache Beam worker dependencies from the Beam Python 3.10 SDK image.\n",
-        "COPY --from=apache/beam_python3.10_sdk:2.60.0 /opt/apache/beam /opt/apache/beam\n",
+        "COPY --from=apache/beam_python3.10_sdk:2.61.0 /opt/apache/beam /opt/apache/beam\n",
         "\n",
-        "RUN pip install --no-cache-dir -vvv apache-beam[gcp]==2.60.0\n",
+        "RUN pip install --no-cache-dir -vvv apache-beam[gcp]==2.61.0\n",
         "RUN pip install openai>=1.52.2 vllm>=0.6.3 triton>=3.1.0\n",
         "\n",
         "RUN apt install libcairo2-dev pkg-config python3-dev -y\n",

learning/katas/go/go.mod

@@ -19,7 +19,7 @@ go 1.14
 
 require (
 	github.com/apache/beam/sdks/v2 v2.40.0
-	github.com/google/go-cmp v0.5.8
-	golang.org/x/net v0.23.0 // indirect
+	github.com/google/go-cmp v0.6.0
+	golang.org/x/net v0.36.0 // indirect
 	google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959 // indirect
 )