Merge branch 'master' of https://github.com/apache/beam into users/damccorm/redisExtra

Author: damccorm

.github/REVIEWERS.yml

@@ -23,23 +23,20 @@ labels:
       - jrmccluskey
       - lostluck
       - shunping
-      - liferoad
     exclusionList: []
   - name: Python
     reviewers:
       - claudevdm
       - damccorm
       - jrmccluskey
       - tvalentyn
-      - liferoad
       - shunping
     exclusionList: []
   - name: Java
     reviewers:
       - Abacn
       - ahmedabu98
       - chamikaramj
-      - m-trieu
       - kennknowles
     exclusionList: []
   - name: spanner
@@ -64,16 +61,16 @@ labels:
     exclusionList: []
   - name: website
     reviewers:
+      - Abacn
       - damccorm
-      - liferoad
       - kennknowles
+      - shunping
     exclusionList: []
 fallbackReviewers:
   - Abacn
   - chamikaramj
   - damccorm
   - jrmccluskey
   - kennknowles
-  - liferoad
   - shunping
   - tvalentyn

.github/actions/dind-up-action/action.yml

@@ -43,7 +43,7 @@ inputs:
   storage-driver:
     default: overlay2
   additional-dockerd-args:
-    default: ""
+    default: "--tls=false"
   use-host-network:
     description: "Run DinD with --network host instead of publishing a TCP port."
     default: "false"
@@ -57,6 +57,9 @@ inputs:
     default: 10s
   wait-timeout:
     default: "180"
+  dind-image:
+    description: "DinD image. Use a fixed version tag to avoid issues."
+    default: "docker:27-dind"
 
   # --- NEW: Optional Setup & Verification Steps ---
   cleanup-dind-on-start:
@@ -129,7 +132,11 @@ runs:
 
         docker volume create --name "${STORAGE_VOL}" --label "com.github.dind=1" --label "com.github.repo=${GITHUB_REPOSITORY}" >/dev/null
         docker volume create --name "${EXECROOT_VOL}" --label "com.github.dind=1" --label "com.github.repo=${GITHUB_REPOSITORY}" >/dev/null
+
+        # Clean up any existing DinD containers
+        docker ps -a -q --filter "label=com.github.dind=1" | xargs -r docker rm -f -v 2>/dev/null || true
         docker rm -f -v "$NAME" 2>/dev/null || true
+        sleep 2
 
         NET_ARGS=""
         PUBLISH_ARGS="-p ${BIND}:${PORT}:${PORT}"
@@ -138,6 +145,8 @@ runs:
           PUBLISH_ARGS=""
         fi
 
+        IMAGE="${{ inputs.dind-image || 'docker:27-dind' }}"
+
         docker run -d --privileged --name "$NAME" \
           --cgroupns=host \
           -e DOCKER_TLS_CERTDIR= \
@@ -152,10 +161,11 @@ runs:
           --health-interval=${HI} \
           --health-retries=${HR} \
           --health-start-period=${HSP} \
-          docker:dind \
+          "${IMAGE}" \
             --host=tcp://0.0.0.0:${PORT} \
             --host=unix:///var/run/docker.sock \
             --storage-driver=${SD} \
+            --iptables=false \
             --exec-root=/execroot ${EXTRA}
 
         {
@@ -206,20 +216,20 @@ runs:
       run: |
         set -euo pipefail
         NAME="${{ inputs.container-name || 'dind-daemon' }}"
-        
+
         # Use host daemon to inspect the DinD container
         nm=$(docker inspect -f '{{.HostConfig.NetworkMode}}' "$NAME")
         echo "DinD NetworkMode=${nm}"
 
         # Try to find the bridge network IP
         ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$NAME" || true)
-        
+
         # If still empty, likely host networking -> use loopback
         if [[ -z "${ip}" || "${nm}" == "host" ]]; then
           echo "No bridge IP found or using host network. Falling back to 127.0.0.1."
           ip="127.0.0.1"
         fi
-        
+
         echo "Discovered DinD IP: ${ip}"
         echo "dind-ip=${ip}" >> "$GITHUB_OUTPUT"
 
@@ -237,7 +247,7 @@ runs:
         hostport=$(docker port redis-smoke 6379/tcp | sed 's/.*://')
         echo "Redis container started, mapped to host port ${hostport}"
         echo "Probing connection to ${DIND_IP}:${hostport} ..."
-        
+
         timeout 5 bash -c 'exec 3<>/dev/tcp/$DIND_IP/'"$hostport"
         if [[ $? -eq 0 ]]; then
           echo "TCP connection successful. Port mapping is working."
@@ -272,4 +282,4 @@ runs:
       shell: bash
       run: |
         echo "DOCKER_HOST=${{ steps.set-output.outputs.docker-host }}" >> "$GITHUB_ENV"
-        echo "DIND_IP=${{ steps.discover-ip.outputs.dind-ip }}" >> "$GITHUB_ENV"
+        echo "DIND_IP=${{ steps.discover-ip.outputs.dind-ip }}" >> "$GITHUB_ENV"

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 3
+    "modification": 4
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json

@@ -4,5 +4,6 @@
   "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
   "https://github.com/apache/beam/pull/31268": "noting that PR #31268 should run this test",
   "https://github.com/apache/beam/pull/31490": "noting that PR #31490 should run this test",
-  "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface"
+  "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface",
+  "https://github.com/apache/beam/pull/36631": "dofn lifecycle",
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json

@@ -4,5 +4,6 @@
   "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
   "https://github.com/apache/beam/pull/31268": "noting that PR #31268 should run this test",
   "https://github.com/apache/beam/pull/31490": "noting that PR #31490 should run this test",
-  "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface"
+  "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface",
+  "https://github.com/apache/beam/pull/36631": "dofn lifecycle validation",
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json

@@ -1,6 +1,4 @@
 {
-  "https://github.com/apache/beam/pull/35213": "Eliminating getPane() in favor of getPaneInfo()",
-  "https://github.com/apache/beam/pull/35177": "Introducing WindowedValueReceiver to runners",
   "comment": "Modify this file in a trivial way to cause this test suite to run",
   "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
   "https://github.com/apache/beam/pull/31798": "noting that PR #31798 should run this test",

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 13
+  "modification": 1
 }

.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml

@@ -77,13 +77,9 @@ jobs:
           python-version: default
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      - name: GCloud Docker credential helper
-        run: |
-          gcloud auth configure-docker us.gcr.io
       - name: run XVR GoUsingJava Dataflow script
         env:
           USER: github-actions
-          CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -97,6 +97,8 @@ jobs:
         id: dind
         if: contains(matrix.os, 'self-hosted')
         with:
+          # Pin to stable Docker version to avoid compatibility issues
+          dind-image: "docker:27-dind"
           # Enable all the new features
           cleanup-dind-on-start: "true"
           smoke-test-port-mapping: "true"

.test-infra/mock-apis/go.mod

@@ -46,12 +46,12 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.35.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 	google.golang.org/api v0.128.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect