Merge branch 'apache:master' into add-watermark-filter

Author: changliiu

.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml

@@ -83,15 +83,25 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           java-version: 17
+      - name: Set DOCKER_TAG unique variable based on timestamp
+        run: echo "DOCKER_TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_ENV
+      - name: GCloud Docker credential helper
+        run: |
+          gcloud auth configure-docker us.gcr.io
       - name: run PostCommit Java Distroless Examples Dataflow V2 script
         uses: ./.github/actions/gradle-command-self-hosted-action
+        env:
+          USER: github-actions
         with:
             gradle-command: :runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTestDistroless
             max-workers: 12
-            arguments:
-              -PskipCheckerFramework
-              -PtestJavaVersion=17
-              -Pjava17Home=$JAVA_HOME_17_X64
+            arguments: |
+              -PdisableSpotlessCheck=true \
+              -PdisableCheckStyle=true \
+              -PskipCheckerFramework \
+              -PtestJavaVersion=java17 \
+              -Pjava17Home=$JAVA_HOME_17_X64 \
+              -PdockerTag=${{ env.DOCKER_TAG }} \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml

@@ -20,6 +20,7 @@ on:
     tags: ['v*']
     branches: ['master', 'release-*']
     paths:
+      - "buildSrc/**"
       - "runners/core-java/**"
       - "sdks/java/core/src/main/**"
       - "sdks/java/extensions/arrow/**"
@@ -33,6 +34,7 @@ on:
   pull_request_target:
     branches: ['master', 'release-*']
     paths:
+      - "buildSrc/**"
       - "runners/core-java/**"
       - 'release/trigger_all_tests.json'
       - '.github/trigger_files/beam_PreCommit_Java_GCP_IO_Direct.json'

CHANGES.md

@@ -86,6 +86,7 @@
 * Yapf version upgraded to 0.43.0 for formatting (Python) ([#34801](https://github.com/apache/beam/pull/34801/)).
 * Python: Added JupyterLab 4.x extension compatibility for enhanced notebook integration ([#34495](https://github.com/apache/beam/pull/34495)).
 * Python: Argument abbreviation is no longer enabled within Beam. If you previously abbreviated arguments (e.g. `--r` for `--runner`), you will now need to specify the whole argument ([#34934](https://github.com/apache/beam/pull/34934)).
+* Java: Users of ReadFromKafkaViaSDF transform might encounter pipeline graph compatibility issues when updating the pipeline. To mitigate, set the `updateCompatibilityVersion` option to the SDK version used for the original pipeline, example `--updateCompatabilityVersion=2.64.0`
 
 ## Deprecations
 
@@ -94,6 +95,7 @@
 ## Bugfixes
 
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* (Java) Fixed cassandraIO ReadAll does not let a pipeline handle or retry exceptions ([#34191](https://github.com/apache/beam/pull/34191)).
 
 ## Security Fixes
 * Fixed [CVE-YYYY-NNNN](https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN) (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -611,12 +611,12 @@ class BeamModulePlugin implements Plugin<Project> {
     def dbcp2_version = "2.9.0"
     def errorprone_version = "2.10.0"
     // [bomupgrader] determined by: com.google.api:gax, consistent with: google_cloud_platform_libraries_bom
-    def gax_version = "2.63.1"
+    def gax_version = "2.65.0"
     def google_ads_version = "33.0.0"
     def google_clients_version = "2.0.0"
     def google_cloud_bigdataoss_version = "2.2.26"
     // [bomupgrader] determined by: com.google.cloud:google-cloud-spanner, consistent with: google_cloud_platform_libraries_bom
-    def google_cloud_spanner_version = "6.89.0"
+    def google_cloud_spanner_version = "6.93.0"
     def google_code_gson_version = "2.10.1"
     def google_oauth_clients_version = "1.34.1"
     // [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
@@ -681,6 +681,7 @@ class BeamModulePlugin implements Plugin<Project> {
         antlr_runtime                               : "org.antlr:antlr4-runtime:4.7",
         args4j                                      : "args4j:args4j:2.33",
         auto_value_annotations                      : "com.google.auto.value:auto-value-annotations:$autovalue_version",
+        // TODO: https://github.com/apache/beam/issues/34993 after stopping supporting Java 8
         avro                                        : "org.apache.avro:avro:1.11.4",
         avro_tests                                  : "org.apache.avro:avro:1.11.3:tests",
         aws_java_sdk2_apache_client                 : "software.amazon.awssdk:apache-client:$aws_java_sdk2_version",
@@ -739,12 +740,12 @@ class BeamModulePlugin implements Plugin<Project> {
         google_api_client_gson                      : "com.google.api-client:google-api-client-gson:$google_clients_version",
         google_api_client_java6                     : "com.google.api-client:google-api-client-java6:$google_clients_version",
         google_api_common                           : "com.google.api:api-common", // google_cloud_platform_libraries_bom sets version
-        google_api_services_bigquery                : "com.google.apis:google-api-services-bigquery:v2-rev20250313-2.0.0",  // [bomupgrader] sets version
+        google_api_services_bigquery                : "com.google.apis:google-api-services-bigquery:v2-rev20250427-2.0.0",  // [bomupgrader] sets version
         google_api_services_cloudresourcemanager    : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20240310-2.0.0",  // [bomupgrader] sets version
         google_api_services_dataflow                : "com.google.apis:google-api-services-dataflow:v1b3-rev20250106-$google_clients_version",
         google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20240130-$google_clients_version",
         google_api_services_pubsub                  : "com.google.apis:google-api-services-pubsub:v1-rev20220904-$google_clients_version",
-        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20250224-2.0.0",  // [bomupgrader] sets version
+        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20250424-2.0.0",  // [bomupgrader] sets version
         google_auth_library_credentials             : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version
         google_auth_library_oauth2_http             : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version
         google_cloud_bigquery                       : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version
@@ -756,13 +757,13 @@ class BeamModulePlugin implements Plugin<Project> {
         google_cloud_core_grpc                      : "com.google.cloud:google-cloud-core-grpc", // google_cloud_platform_libraries_bom sets version
         google_cloud_datacatalog_v1beta1            : "com.google.cloud:google-cloud-datacatalog", // google_cloud_platform_libraries_bom sets version
         google_cloud_dataflow_java_proto_library_all: "com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304",
-        google_cloud_datastore_v1_proto_client      : "com.google.cloud.datastore:datastore-v1-proto-client:2.27.1",   // [bomupgrader] sets version
+        google_cloud_datastore_v1_proto_client      : "com.google.cloud.datastore:datastore-v1-proto-client:2.28.1",   // [bomupgrader] sets version
         google_cloud_firestore                      : "com.google.cloud:google-cloud-firestore", // google_cloud_platform_libraries_bom sets version
         google_cloud_pubsub                         : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version
         google_cloud_pubsublite                     : "com.google.cloud:google-cloud-pubsublite",  // google_cloud_platform_libraries_bom sets version
         // [bomupgrader] the BOM version is set by scripts/tools/bomupgrader.py. If update manually, also update
         // libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
-        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.57.0",
+        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.60.0",
         google_cloud_secret_manager                 : "com.google.cloud:google-cloud-secretmanager", // google_cloud_platform_libraries_bom sets version
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
         google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",

contributor-docs/release-guide.md

@@ -587,7 +587,7 @@ Verify that third party licenses are included in Docker. You can do this with a
           -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l'
     done
 
-    for javaver in 8 11 17; do
+    for javaver in 8 11 17 21; do
       docker run --rm --entrypoint sh \
           apache/beam_java${javaver}_sdk:${RC_TAG} \
           -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l'
@@ -646,8 +646,8 @@ redirects for the 'current' version, merge these PRs in the order listed.  Once
 the PR is merged, the new contents will get picked up automatically and served
 to the Beam website, usually within an hour.  A committer can manually trigger
 the
-[beam_PostCommit_Website_Publish](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/)
-task in Jenkins to avoid waiting.
+[beam_Publish Website](https://github.com/apache/beam/actions/workflows/beam_Publish_Website.yml)
+task in GitHub Actions to avoid waiting.
 
 **PR 1: apache/beam-site**
 
@@ -802,7 +802,7 @@ You can (optionally) also do additional verification by:
 - [ ] Check signatures (e.g. `gpg --verify apache-beam-1.2.3-python.tar.gz.asc
   apache-beam-1.2.3-python.tar.gz`)
 - [ ] `grep` for legal headers in each file.
-- [ ] Run all jenkins suites and include links to passing tests in the voting
+- [ ] Run all GitHub Action suites and include links to passing tests in the voting
   email.
 - [ ] Pull docker images to make sure they are pullable. (e.g. `docker pull apache/beam_python3.12_sdk:2.64.0rc2`)
 - [ ] [Google Internal] You have performed the internal Dataflow container release workflows (these take ~3 hours).
@@ -1189,7 +1189,7 @@ A PR should have already been created (and possibly merged) by github-actions bo
 by looking at open PRs from that bot - https://github.com/apache/beam/pulls/app%2Fgithub-actions
 
 If a PR has not been merged, drive it to completion.
-If no PR was created, triage any failures in https://github.com/apache/beam/actions/workflows/beam_Publish_Website.yml and manually regenerate dependencies,
+If no PR was created, triage any failures in https://github.com/apache/beam/actions/workflows/update_python_dependencies.yml and manually regenerate dependencies,
 following https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-HowtoupdatedependenciesthatareinstalledinPythoncontainerimages
 
 ### Update the Java starter repo