Merge remote-tracking branch 'upstream/master' into revertUbuntuLatestPR35734

Author: mohamedawnallah

.asf.yaml

@@ -51,6 +51,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.67.0-postrelease: {}
     release-2.67: {}
     release-2.66.0-postrelease: {}
     release-2.66: {}

.github/actions/build-push-docker-action/action.yml

@@ -0,0 +1,45 @@
+# This is a composite action to build and push a Docker image.
+name: 'Docker Build and Push'
+description: 'Builds and pushes a Docker image to a container registry.'
+
+inputs:
+  dockerfile_path:
+    description: 'Path to the Dockerfile'
+    required: true
+  image_name:
+    description: 'Base name for the Docker image (e.g., gcr.io/my-project/my-app)'
+    required: true
+  image_tag:
+    description: 'Tag for the Docker image (e.g., latest, or a git sha)'
+    required: true
+  build_context:
+    description: 'The build context for the Docker build command'
+    required: false
+    default: '.'
+
+outputs:
+  image_url:
+    description: "The full URL of the pushed image, including the tag"
+    value: ${{ steps.build-push.outputs.image_url }} # the value is set from a step's output
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure Docker to use Google Cloud credentials
+      shell: bash
+      run: gcloud auth configure-docker --quiet
+
+    - name: Build and Push Docker Image
+      id: build-push # give the step an ID to reference its output
+      shell: bash
+      run: |
+        # Construct the full image URL from the inputs
+        FULL_IMAGE_URL="${{ inputs.image_name }}:${{ inputs.image_tag }}"
+        echo "Building image: $FULL_IMAGE_URL"
+        
+        # Build the image
+        docker build -t $FULL_IMAGE_URL -f ${{ inputs.dockerfile_path }} ${{ inputs.build_context }}
+        # Push the image
+        docker push $FULL_IMAGE_URL
+        # Set the output value for this action
+        echo "image_url=$FULL_IMAGE_URL" >> $GITHUB_OUTPUT

.github/trigger_files/beam_PostCommit_Python.json

@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 101
+  "modification": 35
 }
 

.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "revision": 5
+  "revision": 6
 }

.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml

@@ -55,7 +55,7 @@ jobs:
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run Inference Benchmarks'
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 900
+    timeout-minutes: 1000
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:
       matrix:
@@ -72,7 +72,12 @@ jobs:
       - name: Setup Python environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: default
           python-version: '3.10'
+      - name: Package Python SDK using Gradle
+        run: ./gradlew :sdks:python:sdist -PpythonVersion=3.10
+      - name: Configure Docker for Artifact Registry
+        run: gcloud auth configure-docker us-docker.pkg.dev
       - name: Prepare test arguments
         uses: ./.github/actions/test-arguments-action
         with:
@@ -86,9 +91,28 @@ jobs:
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Streaming_DistilBert_Base_Uncased.txt
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Batch_DistilBert_Base_Uncased.txt
+            ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt
       # The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
       - name: get current time
         run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
+      - name: Build VLLM Development Image
+        id: build_vllm_image
+        uses: ./.github/actions/build-push-docker-action
+        with:
+          dockerfile_path: 'sdks/python/apache_beam/ml/inference/test_resources/vllm.dockerfile'
+          image_name: 'us-docker.pkg.dev/apache-beam-testing/beam-temp/beam-vllm-gpu-base'
+          image_tag: ${{ github.sha }}
+      - name: Run VLLM Gemma Batch Test
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        timeout-minutes: 180
+        with:
+          gradle-command: :sdks:python:apache_beam:testing:load_tests:run
+          arguments: |
+            -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.vllm_gemma_benchmarks \
+            -Prunner=DataflowRunner \
+            -PsdkLocationOverride=false \
+            -PpythonVersion=3.10 \
+            -PloadTest.requirementsTxtFile=apache_beam/ml/inference/vllm_tests_requirements.txt '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_8 }} --mode=batch --job_name=benchmark-tests-vllm-with-gemma-2b-it-batch-${{env.NOW_UTC}} --sdk_container_image=${{ steps.build_vllm_image.outputs.image_url }}'
       - name: run Pytorch Sentiment Streaming using Hugging Face distilbert-base-uncased model
         uses: ./.github/actions/gradle-command-self-hosted-action
         timeout-minutes: 180

.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml

@@ -100,7 +100,7 @@ jobs:
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
-            --no-parallel \
+          max-workers: 4
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

.github/workflows/build_release_candidate.yml

@@ -40,14 +40,17 @@ on:
           beam_site_pr: create the documentation update PR against apache/beam-site.
           --
           prism: build and upload the artifacts to the release for this tag
+          --
+          managed_io_docs_pr: create the managed-io.md update PR against apache/beam.
         required: true
         default: |
            {java_artifacts: "no",
             java_source: "no",
             docker_artifacts: "no",
             python_artifacts: "no",
             beam_site_pr: "no",
-            prism: "no"}
+            prism: "no",
+            managed_io_docs_pr: "no"}
 
 env:
   DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
@@ -545,3 +548,69 @@ jobs:
           svn add --force --parents prism
           svn status
           svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
+
+  managed_io_docs_pr:
+    if:  ${{ fromJson(github.event.inputs.STAGE).managed_io_docs_pr == 'yes'}}
+    runs-on: ubuntu-22.04
+    env:
+      BRANCH_NAME: updates_managed_io_docs_${{ github.event.inputs.RELEASE }}_rc${{ github.event.inputs.RC }}
+      BEAM_ROOT_DIR: ${{ github.workspace }}/beam
+      MANAGED_IO_DOCS_PATH: website/www/site/content/en/documentation/io/managed-io.md
+    steps:
+      - name: Checkout Beam Repo
+        uses: actions/checkout@v4
+        with:
+          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
+          repository: apache/beam
+          path: beam
+          token: ${{ github.event.inputs.REPO_TOKEN }}
+          persist-credentials: false
+      - name: Install Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install Java 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Remove default github maven configuration
+        # This step is a workaround to avoid a decryption issue of Beam's
+        # net.linguica.gradle.maven.settings plugin and github's provided maven
+        # settings.xml file
+        run: rm ~/.m2/settings.xml || true
+      - name: Install SDK
+        working-directory: beam/sdks/python
+        run: |
+          pip install -e.
+      - name: Build Expansion Service Jar
+        working-directory: beam
+        run: |
+          ./gradlew sdks:java:io:expansion-service:shadowJar
+      - name: Build GCP Expansion Service Jar
+        working-directory: beam
+        run: |
+          ./gradlew sdks:java:io:google-cloud-platform:expansion-service:shadowJar
+      - name: Generate Managed IO Docs
+        working-directory: beam/sdks/python
+        run: |
+          python gen_managed_doc.py --output_location ${{ runner.temp }}/managed-io.md
+      - name: Create commit on beam branch
+        working-directory: beam
+        run: |
+          git fetch origin master
+          git checkout -b $BRANCH_NAME origin/master
+          mv ${{ runner.temp }}/managed-io.md ${{ env.MANAGED_IO_DOCS_PATH }}
+          git config user.name $GITHUB_ACTOR
+          git config user.email actions@"$RUNNER_NAME".local
+          git add ${{ env.MANAGED_IO_DOCS_PATH }}
+          git commit --allow-empty -m "Update managed-io.md for release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}."
+          git push -f --set-upstream origin $BRANCH_NAME
+      - name: Create beam PR
+        working-directory: beam
+        env:
+          GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }}
+          PR_TITLE: "Update managed-io.md for release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
+          PR_BODY: "Content generated from release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}."
+        run: |
+          gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base master --repo apache/beam

.github/workflows/finalize_release.yml

@@ -93,8 +93,10 @@ jobs:
         echo "::add-mask::$PYPI_PASSWORD"
     - name: Validate PyPi id/password
       run: |
-        echo "::add-mask::${{ github.event.inputs.PYPI_API_TOKEN }}"
-        if [ "${{ github.event.inputs.PYPI_API_TOKEN }}" == "" ]
+        # Workaround for Actions bug - https://github.com/actions/runner/issues/643
+        PYPI_API_TOKEN=$(jq -r '.inputs.PYPI_API_TOKEN' $GITHUB_EVENT_PATH)
+        echo "::add-mask::$PYPI_API_TOKEN"
+        if [ "$PYPI_API_TOKEN" == "" ]
         then
           echo "Must provide a PyPi password to publish artifacts to PyPi"
           exit 1

.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--runner=DataflowRunner
+--region=us-central1
+--temp_location=gs://temp-storage-for-perf-tests/loadtests
+--staging_location=gs://temp-storage-for-perf-tests/loadtests
+--input=gs://apache-beam-ml/testing/inputs/sentences_50k.txt
+--machine_type=n1-standard-8
+--worker_zone=us-central1-b
+--disk_size_gb=50
+--input_options={}
+--num_workers=8
+--max_num_workers=25
+--autoscaling_algorithm=THROUGHPUT_BASED
+--publish_to_big_query=true
+--sdk_location=container
+--output_table=apache-beam-testing.beam_run_inference.result_gemma_vllm_batch
+--metrics_dataset=beam_run_inference
+--metrics_table=gemma_vllm_batch
+--influx_measurement=gemma_vllm_batch
+--model_gcs_path=gs://apache-beam-ml/models/gemma-2b-it
+--dataflow_service_options=worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver
+--experiments=use_runner_v2

.github/workflows/republish_released_docker_containers.yml

@@ -32,7 +32,7 @@ on:
   - cron: "0 6 * * 1"
 env:
   docker_registry: gcr.io
-  release: "${{ github.event.inputs.RELEASE || '2.66.0' }}"
+  release: "${{ github.event.inputs.RELEASE || '2.67.0' }}"
   rc: "${{ github.event.inputs.RC || '2' }}"
 
 jobs: