From 6295e711c54d6102cd38636f85cf29b1b093fbb7 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 27 May 2025 14:22:44 -0400 Subject: [PATCH 01/23] Update GitHub Actions to run Python 3.13 Tests --- .github/workflows/beam_PostCommit_Python.yml | 2 +- .github/workflows/beam_PostCommit_Python_Arm.yml | 2 +- .../workflows/beam_PostCommit_Python_Examples_Dataflow.yml | 2 +- .github/workflows/beam_PostCommit_Python_Examples_Direct.yml | 2 +- .github/workflows/beam_PostCommit_Python_Examples_Flink.yml | 2 +- .github/workflows/beam_PostCommit_Python_Examples_Spark.yml | 2 +- .github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml | 2 +- .../beam_PostCommit_Python_ValidatesContainer_Dataflow.yml | 2 +- ..._PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml | 2 +- .../beam_PostCommit_Python_ValidatesRunner_Dataflow.yml | 2 +- .../beam_PostCommit_Python_ValidatesRunner_Flink.yml | 2 +- .../beam_PostCommit_Python_ValidatesRunner_Samza.yml | 2 +- .../beam_PostCommit_Python_ValidatesRunner_Spark.yml | 2 +- .../workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 2 +- .github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 2 +- .../workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 2 +- .github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml | 2 +- .github/workflows/beam_PostCommit_TransformService_Direct.yml | 2 +- .github/workflows/beam_PostCommit_XVR_Direct.yml | 2 +- .github/workflows/beam_PostCommit_XVR_Flink.yml | 2 +- .../beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 2 +- .../beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 2 +- .../beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 2 +- .github/workflows/beam_PostCommit_XVR_Samza.yml | 2 +- .github/workflows/beam_PostCommit_XVR_Spark3.yml | 2 +- .github/workflows/beam_PreCommit_Prism_Python.yml | 2 +- .github/workflows/beam_PreCommit_Python.yml | 2 +- .github/workflows/beam_PreCommit_PythonDocker.yml | 2 +- .github/workflows/beam_PreCommit_Python_Dataframes.yml | 2 +- .github/workflows/beam_PreCommit_Python_Examples.yml | 2 +- .github/workflows/beam_PreCommit_Python_Integration.yml | 2 +- .github/workflows/beam_PreCommit_Python_ML.yml | 2 +- .github/workflows/beam_PreCommit_Python_PVR_Flink.yml | 2 +- .github/workflows/beam_PreCommit_Python_Runners.yml | 2 +- .github/workflows/beam_PreCommit_Python_Transforms.yml | 2 +- .github/workflows/beam_Publish_Beam_SDK_Snapshots.yml | 2 ++ .../workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +- .github/workflows/build_wheels.yml | 4 ++-- .github/workflows/python_dependency_tests.yml | 1 + .github/workflows/python_tests.yml | 3 ++- .github/workflows/update_python_dependencies.yml | 1 + CHANGES.md | 1 + 42 files changed, 45 insertions(+), 39 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index 03c76f7253ed..5c1d716a1a7d 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -61,7 +61,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python] job_phrase: [Run Python PostCommit] - python_version: ['3.9', '3.10', '3.11', '3.12'] + python_version: ['3.9', '3.10', '3.11', '3.12','3.13'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml index 8b990ea01cf5..8bebebc96e91 100644 --- a/.github/workflows/beam_PostCommit_Python_Arm.yml +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: [beam_PostCommit_Python_Arm] job_phrase: [Run Python PostCommit Arm] - python_version: ['3.9', '3.10', '3.11', '3.12'] + python_version: ['3.9', '3.10', '3.11', '3.12', '3.13'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml index 3abed56ab8a2..4a9e55beb9ab 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: 3.12 + python-version: 3.13 - name: Run examplesPostCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml index 390aac1ab42d..8e559d286700 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Direct"] job_phrase: ["Run Python Examples_Direct"] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml index ffac141694b1..137ba7ef1b18 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Flink"] job_phrase: ["Run Python Examples_Flink"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml index c2a4132e8c2e..3e155eca0e3c 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_Examples_Spark"] job_phrase: ["Run Python Examples_Spark"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml index 3a12b2d31787..8c0fcf61e6a4 100644 --- a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml +++ b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml @@ -74,7 +74,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: 3.12 + python-version: 3.13 - name: Run mongodbioIT script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index 9077803aeb07..a3803ef2a12a 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesContainer"] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index 985c2a705fbb..a393e423649e 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] job_phrase: ["Run Python RC Dataflow ValidatesContainer"] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml index f37d36b2c0ab..7988ebdbdae0 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] job_phrase: ["Run Python Dataflow ValidatesRunner"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml index e887def73d87..f9e16ae82366 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] job_phrase: ["Run Python Flink ValidatesRunner"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml index 300becc9ddb6..dcfc1663f9af 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Samza"] job_phrase: ["Run Python Samza ValidatesRunner"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml index 030a1dba70d2..127d0cd9e48b 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] job_phrase: ["Run Python Spark ValidatesRunner"] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index 71e032597e6c..1e4fef31368a 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.9 - 3.12 + 3.13 - name: run PostCommit Python Xlang Gcp Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index d6582cea858a..7fb252362f12 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -76,7 +76,7 @@ jobs: with: python-version: | 3.9 - 3.12 + 3.13 - name: Install docker compose run: | sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index 6c543fa2cdbe..50d3055f790a 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -75,7 +75,7 @@ jobs: with: python-version: | 3.9 - 3.12 + 3.13 - name: run PostCommit Python Xlang IO Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml index 35c250e8627b..6c9afba221f8 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml @@ -75,7 +75,7 @@ jobs: with: python-version: | 3.9 - 3.12 + 3.13 - name: run PostCommit Python Xlang IO Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index e2d3220ae6a2..3880fb935472 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_TransformService_Direct"] job_phrase: ["Run TransformService_Direct PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index b2bd02dccbc9..ad4606407e18 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Direct"] job_phrase: ["Run XVR_Direct PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index 50e737d4671e..015ca692d2d9 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -63,7 +63,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Flink"] job_phrase: ["Run XVR_Flink PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index 775c46a82cff..dcc44d740ee7 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] job_phrase: ["Run XVR_JavaUsingPython_Dataflow PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml index 4458cc42ce25..af2057862b1a 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml @@ -73,7 +73,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - python-version: 3.12 + python-version: 3.13 - name: run PostCommit XVR PythonUsingJavaSQL Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index 45f21c426164..a7dacd739c05 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] job_phrase: ["Run XVR_PythonUsingJava_Dataflow PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml index e75eb1e3c2d9..7cca9d9ec0f9 100644 --- a/.github/workflows/beam_PostCommit_XVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Samza"] job_phrase: ["Run XVR_Samza PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index 14a275550f13..4f9e9f6c7cb5 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -62,7 +62,7 @@ jobs: matrix: job_name: ["beam_PostCommit_XVR_Spark3"] job_phrase: ["Run XVR_Spark3 PostCommit"] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] steps: - uses: actions/checkout@v4 - name: Setup repository diff --git a/.github/workflows/beam_PreCommit_Prism_Python.yml b/.github/workflows/beam_PreCommit_Prism_Python.yml index ddb822c2ca28..1d2c125abb71 100644 --- a/.github/workflows/beam_PreCommit_Prism_Python.yml +++ b/.github/workflows/beam_PreCommit_Prism_Python.yml @@ -76,7 +76,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Prism_Python'] job_phrase: ['Run Prism_Python PreCommit'] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml index 3ad9020f17f7..a8de7dd4ddde 100644 --- a/.github/workflows/beam_PreCommit_Python.yml +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python'] job_phrase: ['Run Python PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml index 9cf336f1535c..3824dcd0d2fe 100644 --- a/.github/workflows/beam_PreCommit_PythonDocker.yml +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ["beam_PreCommit_PythonDocker"] job_phrase: ["Run PythonDocker PreCommit"] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml index 14b60c1a5af1..0b1f39aaded6 100644 --- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Dataframes'] job_phrase: ['Run Python_Dataframes PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml index c76d140eadeb..8483883890c8 100644 --- a/.github/workflows/beam_PreCommit_Python_Examples.yml +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Examples'] job_phrase: ['Run Python_Examples PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml index d3c5bf69aab0..c61a23b828a7 100644 --- a/.github/workflows/beam_PreCommit_Python_Integration.yml +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Integration'] job_phrase: ['Run Python_Integration PreCommit'] - python_version: ['3.9', '3.12'] + python_version: ['3.9', '3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index 50ae079d3db3..c6ed29c2c728 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -65,7 +65,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_ML'] job_phrase: ['Run Python_ML PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml index 2010b2ff6f42..1886ad74db70 100644 --- a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml +++ b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml @@ -100,7 +100,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - python-version: 3.12 + python-version: 3.13 - name: run Python PVR Flink PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action env: diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml index 514d8bc57e00..cd7bb4a21ba6 100644 --- a/.github/workflows/beam_PreCommit_Python_Runners.yml +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Runners'] job_phrase: ['Run Python_Runners PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml index 1a16e9b61756..a81b12d91cba 100644 --- a/.github/workflows/beam_PreCommit_Python_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ['beam_PreCommit_Python_Transforms'] job_phrase: ['Run Python_Transforms PreCommit'] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 8aead7937cf5..8eb6958a2476 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -72,10 +72,12 @@ jobs: - "python:container:py310" - "python:container:py311" - "python:container:py312" + - "python:container:py313" - "python:container:distroless:py39" - "python:container:distroless:py310" - "python:container:distroless:py311" - "python:container:distroless:py312" + - "python:container:distroless:py313" - "java:expansion-service:container" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml index e70ec88d1abd..1e9952c533a2 100644 --- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -54,7 +54,7 @@ jobs: matrix: job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] job_phrase: [Run Python ValidatesContainer Dataflow ARM] - python_version: ['3.9','3.10','3.11','3.12'] + python_version: ['3.9','3.10','3.11','3.12','3.13'] if: | github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 51087dadd244..691bc520ea39 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -49,7 +49,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} # Keep in sync with py_version matrix value below - if changed, change that as well. - PY_VERSIONS_FULL: "cp39-* cp310-* cp311-* cp312-*" + PY_VERSIONS_FULL: "cp39-* cp310-* cp311-* cp312-* cp313-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} @@ -230,7 +230,7 @@ jobs: {"os": "ubuntu-20.04", "runner": "ubuntu-22.04", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "aarch64" } ] # Keep in sync (remove asterisks) with PY_VERSIONS_FULL env var above - if changed, change that as well. - py_version: ["cp39-", "cp310-", "cp311-", "cp312-"] + py_version: ["cp39-", "cp310-", "cp311-", "cp312-", "cp313-"] steps: - name: Download python source distribution from artifacts uses: actions/download-artifact@v4.1.8 diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index d8a8ab8c44bf..2f95ea4f48f8 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -30,6 +30,7 @@ jobs: {"py_ver": "3.10", "py_env": "py310" }, { "py_ver": "3.11", "py_env": "py311" }, { "py_ver": "3.12", "py_env": "py312" }, + { "py_ver": "3.13", "py_env": "py313" }, ] steps: - name: Checkout code diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index fc6d4566ea5d..82275ef9dfc1 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -97,6 +97,7 @@ jobs: { "py_ver": "3.10", "tox_env": "py310" }, { "py_ver": "3.11", "tox_env": "py311" }, { "py_ver": "3.12", "tox_env": "py312" }, + { "py_ver": "3.13", "tox_env": "py313" }, ] steps: - name: Checkout code @@ -133,7 +134,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest] - python: ["3.9", "3.10", "3.11", "3.12"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index d45aa2a08c91..dea5f2ac7653 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -60,6 +60,7 @@ jobs: 3.10 3.11 3.12 + 3.13 java-version: default go-version: default disable-cache: true diff --git a/CHANGES.md b/CHANGES.md index a8925df49d72..65562eb7be77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -79,6 +79,7 @@ * [Python] Prism runner now auto-enabled for some Python pipelines using the direct runner ([#34921](https://github.com/apache/beam/pull/34921)). * [YAML] WriteToTFRecord and ReadFromTFRecord Beam YAML support * Python: Added JupyterLab 4.x extension compatibility for enhanced notebook integration ([#34495](https://github.com/apache/beam/pull/34495)). +* [Python] Apache Beam adds Python 3.13 support ([#34869](https://github.com/apache/beam/issues/34869)). ## Breaking Changes From 70b11812c3ad2239f3bfdf0b82185af93e4d71ff Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 27 May 2025 14:27:21 -0400 Subject: [PATCH 02/23] fix container gradle build commands --- sdks/python/container/build.gradle | 2 +- sdks/python/container/distroless/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index 861d90b6e6be..c8266e93aff0 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -22,7 +22,7 @@ applyGoNature() description = "Apache Beam :: SDKs :: Python :: Container" // Keep these values in sync with sdks/python/container/distroless/build.gradle. int min_python_version=9 -int max_python_version=12 +int max_python_version=13 configurations { sdkSourceTarball diff --git a/sdks/python/container/distroless/build.gradle b/sdks/python/container/distroless/build.gradle index a967a80f4fc0..d8277c01b3fe 100644 --- a/sdks/python/container/distroless/build.gradle +++ b/sdks/python/container/distroless/build.gradle @@ -21,7 +21,7 @@ plugins { id 'org.apache.beam.module' } description = "Apache Beam :: SDKs :: Python :: Container :: Distroless" // Keep these values in sync with sdks/python/container/build.gradle. int min_python_version=9 -int max_python_version=12 +int max_python_version=13 tasks.register("buildAll") { From 17633690b1e6dbf0bbd679a9f07df0f116772778 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 27 May 2025 14:34:48 -0400 Subject: [PATCH 03/23] bump cibuildwheel --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 691bc520ea39..a3d0d1befd1c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -252,7 +252,7 @@ jobs: name: Set up QEMU - name: Install cibuildwheel # note: sync cibuildwheel version with gradle task sdks:python:bdistPy* steps - run: pip install cibuildwheel==2.17.0 setuptools + run: pip install cibuildwheel==2.23.3 setuptools - name: Build wheel # Only build wheel if it is one of the target versions for this platform, otherwise no-op if: ${{ contains(matrix.os_python.python, matrix.py_version) }} From b71e8697225191127ca462b08b2f6362287ae891 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 27 May 2025 14:40:21 -0400 Subject: [PATCH 04/23] bump distlib --- sdks/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 97a9fe6141ea..9f5c4788e200 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -25,7 +25,7 @@ requires = [ "grpcio-tools==1.71.0; python_version >= '3.13'", "mypy-protobuf==3.5.0", # Avoid https://github.com/pypa/virtualenv/issues/2006 - "distlib==0.3.7", + "distlib==0.3.9", # Numpy headers "numpy>=1.14.3,<2.3.0", # Update setup.py as well. # having cython here will create wheels that are platform dependent. From 7028c7cf7b535a4d1cb93c1c70a7c67068052d2e Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Wed, 28 May 2025 09:51:55 -0400 Subject: [PATCH 05/23] update gradle.properties --- sdks/python/test-suites/gradle.properties | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index d027cd3144d3..99352a61c401 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -23,34 +23,34 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 # Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.9,3.12 +dataflow_precommit_it_task_py_versions=3.9,3.13 dataflow_mongodbio_it_task_py_versions=3.9 dataflow_chicago_taxi_example_task_py_versions=3.9 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests=3.9,3.12 -dataflow_validates_runner_streaming_tests=3.9,3.12 -dataflow_examples_postcommit_py_versions=3.12 +dataflow_validates_runner_batch_tests=3.9,3.13 +dataflow_validates_runner_streaming_tests=3.9,3.13 +dataflow_examples_postcommit_py_versions=3.13 # TFX_BSL is not yet supported on Python 3.10. dataflow_cloudml_benchmark_tests_py_versions=3.9 # direct runner test-suites -direct_mongodbio_it_task_py_versions=3.12 +direct_mongodbio_it_task_py_versions=3.13 # flink runner test-suites -flink_validates_runner_precommit_py_versions=3.12 -flink_validates_runner_postcommit_py_versions=3.9,3.12 -flink_examples_postcommit_py_versions=3.9,3.12 +flink_validates_runner_precommit_py_versions=3.13 +flink_validates_runner_postcommit_py_versions=3.9,3.13 +flink_examples_postcommit_py_versions=3.9,3.13 # samza runner test-suites -samza_validates_runner_postcommit_py_versions=3.9,3.12 +samza_validates_runner_postcommit_py_versions=3.9,3.13 # spark runner test-suites -spark_examples_postcommit_py_versions=3.9,3.12 +spark_examples_postcommit_py_versions=3.9,3.13 # prism runner test-suites -prism_validates_runner_precommit_py_versions=3.12 -prism_validates_runner_postcommit_py_versions=3.9,3.12 -prism_examples_postcommit_py_versions=3.9,3.12 +prism_validates_runner_precommit_py_versions=3.13 +prism_validates_runner_postcommit_py_versions=3.9,3.13 +prism_examples_postcommit_py_versions=3.9,3.13 # cross language postcommit python test suites -cross_language_validates_py_versions=3.9,3.12 +cross_language_validates_py_versions=3.9,3.13 From 98c473c2beea1e05d0ea347f571e556eaa92442b Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Wed, 28 May 2025 12:10:54 -0400 Subject: [PATCH 06/23] fix postcommit gradle commands --- build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index a0844249bf08..104f26535e51 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -552,8 +552,8 @@ tasks.register("python313PostCommit") { dependsOn(":sdks:python:test-suites:dataflow:py313:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py313:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py313:hdfsIntegrationTest") - dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy312") - dependsOn(":sdks:python:test-suites:dataflow:py313:inferencePostCommitITPy312") + dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy313") + dependsOn(":sdks:python:test-suites:dataflow:py313:inferencePostCommitITPy313") } tasks.register("portablePythonPreCommit") { From 40f182c4828da5c201150895e0dca3cef725b92f Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Wed, 28 May 2025 12:24:43 -0400 Subject: [PATCH 07/23] remove inference postcommit reference --- build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 104f26535e51..eb9270e9afee 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -553,7 +553,6 @@ tasks.register("python313PostCommit") { dependsOn(":sdks:python:test-suites:direct:py313:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py313:hdfsIntegrationTest") dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy313") - dependsOn(":sdks:python:test-suites:dataflow:py313:inferencePostCommitITPy313") } tasks.register("portablePythonPreCommit") { From 6dac5cb26bfe431daa8626dddc46b5876fa7181f Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Thu, 29 May 2025 14:22:57 -0400 Subject: [PATCH 08/23] try stepping confest var back for python postcommit --- sdks/python/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/conftest.py b/sdks/python/conftest.py index 37c4a0434e75..0b8881ebb5a8 100644 --- a/sdks/python/conftest.py +++ b/sdks/python/conftest.py @@ -22,7 +22,7 @@ from apache_beam.options import pipeline_options from apache_beam.testing.test_pipeline import TestPipeline -MAX_SUPPORTED_PYTHON_VERSION = (3, 13) +MAX_SUPPORTED_PYTHON_VERSION = (3, 8) def pytest_addoption(parser): From c636c85d7a2724b03a76ecf201ba9e0d5ea2020d Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 18:26:58 +0000 Subject: [PATCH 09/23] fix conftest, CHANGES.md --- CHANGES.md | 3 ++- sdks/python/conftest.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 946ef27c1ef5..38f0554354f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -77,7 +77,8 @@ * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). * Python examples added for CloudSQL enrichment handler on [Beam website](https://beam.apache.org/documentation/transforms/python/elementwise/enrichment-cloudsql/) (Python) ([#35473](https://github.com/apache/beam/issues/36095)). * Support for batch mode execution in WriteToPubSub transform added (Python) ([#35990](https://github.com/apache/beam/issues/35990)). -* +* Added official support for Python 3.13 ([#34869](https://github.com/apache/beam/issues/34869)). + ## Breaking Changes * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). diff --git a/sdks/python/conftest.py b/sdks/python/conftest.py index 62acc3246556..855af55911a1 100644 --- a/sdks/python/conftest.py +++ b/sdks/python/conftest.py @@ -26,7 +26,7 @@ from apache_beam.options import pipeline_options from apache_beam.testing.test_pipeline import TestPipeline -MAX_SUPPORTED_PYTHON_VERSION = (3, 8) +MAX_SUPPORTED_PYTHON_VERSION = (3, 13) def pytest_addoption(parser): From cd0a92da48fc2cdec5820b3abef06f4c3b019891 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 18:27:48 +0000 Subject: [PATCH 10/23] pin apitools --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index c23d69225d52..fbd0786ab158 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -458,7 +458,7 @@ def get_portability_package_data(): 'cachetools>=3.1.0,<7', 'google-api-core>=2.0.0,<3', 'google-apitools>=0.5.31,<0.5.32; python_version < "3.13"', - 'google-apitools>=0.5.32,<0.5.33; python_version >= "3.13"', + 'google-apitools>=0.5.35; python_version >= "3.13"', # NOTE: Maintainers, please do not require google-auth>=2.x.x # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 From d60fa62b61e4c2c3400ae86999f5139fbc748409 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 19:59:08 +0000 Subject: [PATCH 11/23] remove py313-win env, restore ML precommit exclusions to previous version --- .github/workflows/beam_PreCommit_Python_ML.yml | 4 +--- sdks/python/tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index f16df0ab91f4..e78a52416361 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -73,7 +73,7 @@ jobs: # Context: https://github.com/apache/beam/pull/35585. os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]] exclude: - # Temporary exclude Python 3.9, 3.10, 3.11, 3.12 from ubuntu-latest. + # Temporary exclude Python 3.9, 3.10, 3.11, from ubuntu-latest. # This results in pip dependency resolution exceeded maximum depth issue. # Context: https://github.com/apache/beam/pull/35816. - python_version: '3.9' @@ -82,8 +82,6 @@ jobs: os: [ubuntu-latest] - python_version: '3.11' os: [ubuntu-latest] - - python_version: '3.12' - os: [ubuntu-latest] if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 9e428ba251a5..8a06bf4f3262 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -89,7 +89,7 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311,312,313}-win] +[testenv:py{39,310,311,312}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" From 3cb78e1c16c1a9fa3816f10cffe04facecd6985b Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 20:11:31 +0000 Subject: [PATCH 12/23] update postcommit dependency workflow to use 3.13 --- .github/workflows/beam_PostCommit_Python_Dependency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index 609271cda75d..c1dfb34e0153 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -59,7 +59,7 @@ jobs: matrix: job_name: ['beam_PostCommit_Python_Dependency'] job_phrase: ['Run Python PostCommit Dependency'] - python_version: ['3.9','3.12'] + python_version: ['3.9','3.13'] timeout-minutes: 180 if: | github.event_name == 'workflow_dispatch' || From 26636097ffd5feeb00f82c6adece4cab6b9f4909 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 20:21:54 +0000 Subject: [PATCH 13/23] restore tox 313-win env --- sdks/python/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 8a06bf4f3262..9e428ba251a5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -89,7 +89,7 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{39,310,311,312}-win] +[testenv:py{39,310,311,312,313}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" From 346c9364a9e810c839a3de72407da40f0d4baf8a Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 30 Sep 2025 20:44:01 +0000 Subject: [PATCH 14/23] bump pytest upper bound --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index fbd0786ab158..ea1a99d4b2dc 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -437,7 +437,7 @@ def get_portability_package_data(): 'pyhamcrest>=1.9,!=1.10.0,<3.0.0', 'requests_mock>=1.7,<2.0', 'tenacity>=8.0.0,<9', - 'pytest>=7.1.2,<8.0', + 'pytest>=7.1.2,<9.0', 'pytest-xdist>=2.5.0,<4', 'pytest-timeout>=2.1.0,<3', 'scikit-learn>=0.20.0', From 2b897ad888ffa47ce7dd5c8afd7bc62c12e4a0c2 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 1 Oct 2025 14:03:06 +0000 Subject: [PATCH 15/23] comment out unneccesary golden_size kwarg --- .../interactive/testing/integration/tests/screen_diff_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py b/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py index a3f8ace0b53f..0470e94fbff5 100644 --- a/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py +++ b/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py @@ -28,7 +28,7 @@ @pytest.mark.timeout(300) class DataFramesTest(BaseTestCase): def __init__(self, *args, **kwargs): - kwargs['golden_size'] = (1024, 10000) + # kwargs['golden_size'] = (1024, 10000) super().__init__(*args, **kwargs) def explicit_wait(self): @@ -50,7 +50,7 @@ def test_dataframes(self): @pytest.mark.timeout(300) class InitSquareCubeTest(BaseTestCase): def __init__(self, *args, **kwargs): - kwargs['golden_size'] = (1024, 10000) + # kwargs['golden_size'] = (1024, 10000) super().__init__(*args, **kwargs) def test_init_square_cube_notebook(self): From 1e844d2bce20546e236cb9f74e944f80fa450eb7 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 1 Oct 2025 14:15:05 +0000 Subject: [PATCH 16/23] update base image requirements --- .../py310/base_image_requirements.txt | 92 ++++++++-------- .../py311/base_image_requirements.txt | 94 ++++++++-------- .../py312/base_image_requirements.txt | 94 ++++++++-------- .../py313/base_image_requirements.txt | 101 +++++++++--------- .../py39/base_image_requirements.txt | 82 +++++++------- 5 files changed, 236 insertions(+), 227 deletions(-) diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index 63d947772c2b..3db0e132e8e9 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -26,26 +26,26 @@ aiohappyeyeballs==2.6.1 aiohttp==3.12.15 aiosignal==1.4.0 annotated-types==0.7.0 -anyio==4.10.0 +anyio==4.11.0 asn1crypto==1.5.1 async-timeout==5.0.1 attrs==25.3.0 backports.tarfile==1.2.0 beartype==0.21.0 -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==5.5.2 +cachetools==6.2.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 -click==8.2.1 +click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==45.0.6 -Cython==3.1.3 +cryptography==46.0.2 +Cython==3.1.4 dill==0.3.1.1 -dnspython==2.7.0 +dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 @@ -57,29 +57,30 @@ freezegun==1.5.5 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.179.0 +google-api-python-client==2.183.0 google-apitools==0.5.31 -google-auth==2.40.3 +google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.110.0 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-bigtable==2.32.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.31.0 +google-cloud-dlp==3.32.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.14.2 -google-cloud-spanner==3.57.0 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.31.0 +google-genai==1.39.1 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.4 @@ -93,7 +94,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.138.3 +hypothesis==6.140.2 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -102,32 +103,32 @@ jaraco.context==6.0.1 jaraco.functools==4.3.0 jeepney==0.9.0 Jinja2==3.1.6 -joblib==1.5.1 +joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 -MarkupSafe==3.0.2 +MarkupSafe==3.0.3 milvus-lite==2.5.1 mmh3==5.2.0 mock==5.2.0 -more-itertools==10.7.0 +more-itertools==10.8.0 multidict==6.6.4 -nltk==3.9.1 +nltk==3.9.2 numpy==2.2.6 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 oracledb==3.3.0 -orjson==3.11.2 +orjson==3.11.3 overrides==7.7.0 packaging==25.0 pandas==2.2.3 parameterized==0.9.0 -pg8000==1.31.4 +pg8000==1.31.5 pip==25.2 pluggy==1.6.0 propcache==0.3.2 @@ -138,52 +139,53 @@ pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 -pycparser==2.22 -pydantic==2.11.7 +pycparser==2.23 +pydantic==2.11.9 pydantic_core==2.33.2 pydot==1.4.2 +Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.5.15 -pymongo==4.14.1 +pymilvus==2.5.16 +pymongo==4.15.1 PyMySQL==1.1.2 -pyparsing==3.2.3 +pyparsing==3.2.5 pyproject_hooks==1.2.0 -pytest==7.4.4 +pytest==8.4.2 pytest-timeout==2.4.0 pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 -python-tds==1.17.0 +python-tds==1.17.1 pytz==2025.2 -PyYAML==6.0.2 +PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.7.34 +regex==2025.9.18 requests==2.32.5 requests-mock==1.12.1 -rpds-py==0.27.0 +rpds-py==0.27.1 rsa==4.9.1 -scikit-learn==1.7.1 +scikit-learn==1.7.2 scipy==1.15.3 scramp==1.4.6 -SecretStorage==3.3.3 +SecretStorage==3.4.0 setuptools==80.9.0 -shapely==2.1.1 +shapely==2.1.2 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -soupsieve==2.7 +soupsieve==2.8 SQLAlchemy==2.0.43 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 tenacity==8.5.0 -testcontainers==4.12.0 +testcontainers==4.13.1 threadpoolctl==3.6.0 tomli==2.2.1 tqdm==4.67.1 -typing-inspection==0.4.1 -typing_extensions==4.14.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 uritemplate==4.2.0 @@ -194,4 +196,4 @@ wheel==0.45.1 wrapt==1.17.3 yarl==1.20.1 zipp==3.23.0 -zstandard==0.24.0 +zstandard==0.25.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 6ba596eeed3d..0986570ba125 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -26,25 +26,25 @@ aiohappyeyeballs==2.6.1 aiohttp==3.12.15 aiosignal==1.4.0 annotated-types==0.7.0 -anyio==4.10.0 +anyio==4.11.0 asn1crypto==1.5.1 attrs==25.3.0 backports.tarfile==1.2.0 beartype==0.21.0 -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==5.5.2 +cachetools==6.2.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 -click==8.2.1 +click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==45.0.6 -Cython==3.1.3 +cryptography==46.0.2 +Cython==3.1.4 dill==0.3.1.1 -dnspython==2.7.0 +dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 @@ -55,29 +55,30 @@ freezegun==1.5.5 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.179.0 +google-api-python-client==2.183.0 google-apitools==0.5.31 -google-auth==2.40.3 +google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.110.0 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-bigtable==2.32.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.31.0 +google-cloud-dlp==3.32.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.14.2 -google-cloud-spanner==3.57.0 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.31.0 +google-genai==1.39.1 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.4 @@ -91,7 +92,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.138.3 +hypothesis==6.140.2 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -100,32 +101,32 @@ jaraco.context==6.0.1 jaraco.functools==4.3.0 jeepney==0.9.0 Jinja2==3.1.6 -joblib==1.5.1 +joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 -MarkupSafe==3.0.2 +MarkupSafe==3.0.3 milvus-lite==2.5.1 mmh3==5.2.0 mock==5.2.0 -more-itertools==10.7.0 +more-itertools==10.8.0 multidict==6.6.4 -nltk==3.9.1 +nltk==3.9.2 numpy==2.2.6 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 oracledb==3.3.0 -orjson==3.11.2 +orjson==3.11.3 overrides==7.7.0 packaging==25.0 pandas==2.2.3 parameterized==0.9.0 -pg8000==1.31.4 +pg8000==1.31.5 pip==25.2 pluggy==1.6.0 propcache==0.3.2 @@ -136,51 +137,52 @@ pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 -pycparser==2.22 -pydantic==2.11.7 +pycparser==2.23 +pydantic==2.11.9 pydantic_core==2.33.2 pydot==1.4.2 +Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.5.15 -pymongo==4.14.1 +pymilvus==2.5.16 +pymongo==4.15.1 PyMySQL==1.1.2 -pyparsing==3.2.3 +pyparsing==3.2.5 pyproject_hooks==1.2.0 -pytest==7.4.4 +pytest==8.4.2 pytest-timeout==2.4.0 pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 -python-tds==1.17.0 +python-tds==1.17.1 pytz==2025.2 -PyYAML==6.0.2 +PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.7.34 +regex==2025.9.18 requests==2.32.5 requests-mock==1.12.1 -rpds-py==0.27.0 +rpds-py==0.27.1 rsa==4.9.1 -scikit-learn==1.7.1 -scipy==1.16.1 +scikit-learn==1.7.2 +scipy==1.16.2 scramp==1.4.6 -SecretStorage==3.3.3 +SecretStorage==3.4.0 setuptools==80.9.0 -shapely==2.1.1 +shapely==2.1.2 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -soupsieve==2.7 +soupsieve==2.8 SQLAlchemy==2.0.43 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 tenacity==8.5.0 -testcontainers==4.12.0 +testcontainers==4.13.1 threadpoolctl==3.6.0 tqdm==4.67.1 -typing-inspection==0.4.1 -typing_extensions==4.14.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 uritemplate==4.2.0 @@ -191,4 +193,4 @@ wheel==0.45.1 wrapt==1.17.3 yarl==1.20.1 zipp==3.23.0 -zstandard==0.24.0 +zstandard==0.25.0 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index c709b57164a8..840ba4a5cac9 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -26,24 +26,24 @@ aiohappyeyeballs==2.6.1 aiohttp==3.12.15 aiosignal==1.4.0 annotated-types==0.7.0 -anyio==4.10.0 +anyio==4.11.0 asn1crypto==1.5.1 attrs==25.3.0 beartype==0.21.0 -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==5.5.2 +cachetools==6.2.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 -click==8.2.1 +click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==45.0.6 -Cython==3.1.3 +cryptography==46.0.2 +Cython==3.1.4 dill==0.3.1.1 -dnspython==2.7.0 +dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 @@ -54,29 +54,30 @@ freezegun==1.5.5 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.179.0 +google-api-python-client==2.183.0 google-apitools==0.5.31 -google-auth==2.40.3 +google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.110.0 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-bigtable==2.32.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.31.0 +google-cloud-dlp==3.32.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.14.2 -google-cloud-spanner==3.57.0 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.31.0 +google-genai==1.39.1 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.4 @@ -90,7 +91,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.138.3 +hypothesis==6.140.2 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -99,32 +100,32 @@ jaraco.context==6.0.1 jaraco.functools==4.3.0 jeepney==0.9.0 Jinja2==3.1.6 -joblib==1.5.1 +joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 -MarkupSafe==3.0.2 +MarkupSafe==3.0.3 milvus-lite==2.5.1 mmh3==5.2.0 mock==5.2.0 -more-itertools==10.7.0 +more-itertools==10.8.0 multidict==6.6.4 -nltk==3.9.1 +nltk==3.9.2 numpy==2.2.6 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 oracledb==3.3.0 -orjson==3.11.2 +orjson==3.11.3 overrides==7.7.0 packaging==25.0 pandas==2.2.3 parameterized==0.9.0 -pg8000==1.31.4 +pg8000==1.31.5 pip==25.2 pluggy==1.6.0 propcache==0.3.2 @@ -135,51 +136,52 @@ pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 -pycparser==2.22 -pydantic==2.11.7 +pycparser==2.23 +pydantic==2.11.9 pydantic_core==2.33.2 pydot==1.4.2 +Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.5.15 -pymongo==4.14.1 +pymilvus==2.5.16 +pymongo==4.15.1 PyMySQL==1.1.2 -pyparsing==3.2.3 +pyparsing==3.2.5 pyproject_hooks==1.2.0 -pytest==7.4.4 +pytest==8.4.2 pytest-timeout==2.4.0 pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 -python-tds==1.17.0 +python-tds==1.17.1 pytz==2025.2 -PyYAML==6.0.2 +PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.7.34 +regex==2025.9.18 requests==2.32.5 requests-mock==1.12.1 -rpds-py==0.27.0 +rpds-py==0.27.1 rsa==4.9.1 -scikit-learn==1.7.1 -scipy==1.16.1 +scikit-learn==1.7.2 +scipy==1.16.2 scramp==1.4.6 -SecretStorage==3.3.3 +SecretStorage==3.4.0 setuptools==80.9.0 -shapely==2.1.1 +shapely==2.1.2 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -soupsieve==2.7 +soupsieve==2.8 SQLAlchemy==2.0.43 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 tenacity==8.5.0 -testcontainers==4.12.0 +testcontainers==4.13.1 threadpoolctl==3.6.0 tqdm==4.67.1 -typing-inspection==0.4.1 -typing_extensions==4.14.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 uritemplate==4.2.0 @@ -190,4 +192,4 @@ wheel==0.45.1 wrapt==1.17.3 yarl==1.20.1 zipp==3.23.0 -zstandard==0.24.0 +zstandard==0.25.0 diff --git a/sdks/python/container/py313/base_image_requirements.txt b/sdks/python/container/py313/base_image_requirements.txt index 7d73bf53a928..83bb7090ba75 100644 --- a/sdks/python/container/py313/base_image_requirements.txt +++ b/sdks/python/container/py313/base_image_requirements.txt @@ -26,24 +26,24 @@ aiohappyeyeballs==2.6.1 aiohttp==3.12.15 aiosignal==1.4.0 annotated-types==0.7.0 -anyio==4.10.0 +anyio==4.11.0 asn1crypto==1.5.1 attrs==25.3.0 beartype==0.21.0 -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==5.5.2 +cachetools==6.2.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 -click==8.2.1 +click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==45.0.6 -Cython==3.1.3 +cryptography==46.0.2 +Cython==3.1.4 dill==0.3.1.1 -dnspython==2.7.0 +dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 @@ -54,41 +54,42 @@ freezegun==1.5.5 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-apitools==0.5.32 -google-auth==2.40.3 +google-apitools==0.5.35 +google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.110.0 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-bigtable==2.32.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.31.0 +google-cloud-dlp==3.32.0 google-cloud-language==2.17.2 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.14.2 -google-cloud-spanner==3.57.0 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.31.0 +google-genai==1.39.1 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.4 grpc-google-iam-v1==0.14.2 grpc-interceptor==0.15.4 -grpcio==1.74.0 -grpcio-status==1.71.2 +grpcio==1.75.1 +grpcio-status==1.75.1 guppy3==3.1.5 h11==0.16.0 hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.138.3 +hypothesis==6.140.2 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -97,87 +98,87 @@ jaraco.context==6.0.1 jaraco.functools==4.3.0 jeepney==0.9.0 Jinja2==3.1.6 -joblib==1.5.1 +joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 -MarkupSafe==3.0.2 -milvus-lite==2.5.1 +MarkupSafe==3.0.3 mmh3==5.2.0 mock==5.2.0 -more-itertools==10.7.0 +more-itertools==10.8.0 multidict==6.6.4 -nltk==3.9.1 +nltk==3.9.2 numpy==2.2.6 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 oracledb==3.3.0 -orjson==3.11.2 +orjson==3.11.3 overrides==7.7.0 packaging==25.0 pandas==2.2.3 parameterized==0.9.0 -pg8000==1.31.4 +pg8000==1.31.5 pip==25.2 pluggy==1.6.0 propcache==0.3.2 proto-plus==1.26.1 -protobuf==5.29.5 +protobuf==6.32.1 psycopg2-binary==2.9.10 pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 -pycparser==2.22 -pydantic==2.11.7 +pycparser==2.23 +pydantic==2.11.9 pydantic_core==2.33.2 pydot==1.4.2 +Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.6.0 -pymongo==4.14.1 +pymilvus==2.6.2 +pymongo==4.15.1 PyMySQL==1.1.2 -pyparsing==3.2.3 +pyparsing==3.2.5 pyproject_hooks==1.2.0 -pytest==7.4.4 +pytest==8.4.2 pytest-timeout==2.4.0 pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 -python-tds==1.17.0 +python-tds==1.17.1 pytz==2025.2 -PyYAML==6.0.2 +PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.7.34 +regex==2025.9.18 requests==2.32.5 requests-mock==1.12.1 -rpds-py==0.27.0 +rpds-py==0.27.1 rsa==4.9.1 -scikit-learn==1.7.1 -scipy==1.16.1 +scikit-learn==1.7.2 +scipy==1.16.2 scramp==1.4.6 -SecretStorage==3.3.3 +SecretStorage==3.4.0 setuptools==80.9.0 -shapely==2.1.1 +shapely==2.1.2 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -soupsieve==2.7 +soupsieve==2.8 SQLAlchemy==2.0.43 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 tenacity==8.5.0 -testcontainers==4.12.0 +testcontainers==4.13.1 threadpoolctl==3.6.0 tqdm==4.67.1 -typing-inspection==0.4.1 -typing_extensions==4.14.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 urllib3==2.5.0 @@ -187,4 +188,4 @@ wheel==0.45.1 wrapt==1.17.3 yarl==1.20.1 zipp==3.23.0 -zstandard==0.24.0 +zstandard==0.25.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 810dfcc2a6e5..9fdab376e541 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -26,24 +26,24 @@ aiohappyeyeballs==2.6.1 aiohttp==3.12.15 aiosignal==1.4.0 annotated-types==0.7.0 -anyio==4.10.0 +anyio==4.11.0 asn1crypto==1.5.1 async-timeout==5.0.1 attrs==25.3.0 backports.tarfile==1.2.0 beartype==0.21.0 -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==5.5.2 +cachetools==6.2.0 certifi==2025.8.3 -cffi==1.17.1 +cffi==2.0.0 charset-normalizer==3.4.3 click==8.1.8 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==45.0.6 -Cython==3.1.3 +cryptography==46.0.2 +Cython==3.1.4 dill==0.3.1.1 dnspython==2.7.0 docker==7.1.0 @@ -57,29 +57,30 @@ freezegun==1.5.5 frozenlist==1.7.0 future==1.0.0 google-api-core==2.25.1 -google-api-python-client==2.179.0 +google-api-python-client==2.183.0 google-apitools==0.5.31 -google-auth==2.40.3 +google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.110.0 -google-cloud-bigquery==3.36.0 -google-cloud-bigquery-storage==2.32.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 google-cloud-bigtable==2.32.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.31.0 +google-cloud-dlp==3.32.0 google-cloud-language==2.17.2 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.14.2 -google-cloud-spanner==3.57.0 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 google-cloud-videointelligence==2.16.2 google-cloud-vision==3.10.2 google-crc32c==1.7.1 -google-genai==1.31.0 +google-genai==1.39.1 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 greenlet==3.2.4 @@ -93,7 +94,7 @@ hdfs==2.7.3 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 -hypothesis==6.138.3 +hypothesis==6.140.2 idna==3.10 importlib_metadata==8.7.0 iniconfig==2.1.0 @@ -102,32 +103,32 @@ jaraco.context==6.0.1 jaraco.functools==4.3.0 jeepney==0.9.0 Jinja2==3.1.6 -joblib==1.5.1 +joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 -MarkupSafe==3.0.2 +MarkupSafe==3.0.3 milvus-lite==2.5.1 mmh3==5.2.0 mock==5.2.0 -more-itertools==10.7.0 +more-itertools==10.8.0 multidict==6.6.4 -nltk==3.9.1 +nltk==3.9.2 numpy==2.0.2 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 oracledb==3.3.0 -orjson==3.11.2 +orjson==3.11.3 overrides==7.7.0 packaging==25.0 pandas==2.2.3 parameterized==0.9.0 -pg8000==1.31.4 +pg8000==1.31.5 pip==25.2 pluggy==1.6.0 propcache==0.3.2 @@ -138,31 +139,32 @@ pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 -pycparser==2.22 -pydantic==2.11.7 +pycparser==2.23 +pydantic==2.11.9 pydantic_core==2.33.2 pydot==1.4.2 +Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.5.15 -pymongo==4.14.1 +pymilvus==2.5.16 +pymongo==4.15.1 PyMySQL==1.1.2 -pyparsing==3.2.3 +pyparsing==3.2.5 pyproject_hooks==1.2.0 -pytest==7.4.4 +pytest==8.4.2 pytest-timeout==2.4.0 pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 -python-tds==1.17.0 +python-tds==1.17.1 pytz==2025.2 -PyYAML==6.0.2 +PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.7.34 +regex==2025.9.18 requests==2.32.5 requests-mock==1.12.1 -rpds-py==0.27.0 +rpds-py==0.27.1 rsa==4.9.1 scikit-learn==1.6.1 scipy==1.13.1 @@ -173,17 +175,17 @@ shapely==2.0.7 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -soupsieve==2.7 +soupsieve==2.8 SQLAlchemy==2.0.43 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 tenacity==8.5.0 -testcontainers==4.12.0 +testcontainers==4.13.1 threadpoolctl==3.6.0 tomli==2.2.1 tqdm==4.67.1 -typing-inspection==0.4.1 -typing_extensions==4.14.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 uritemplate==4.2.0 @@ -194,4 +196,4 @@ wheel==0.45.1 wrapt==1.17.3 yarl==1.20.1 zipp==3.23.0 -zstandard==0.24.0 +zstandard==0.25.0 From 82e6f9bb16424ccdc3505a8bda6191fed6d6b6f5 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 1 Oct 2025 17:39:27 +0000 Subject: [PATCH 17/23] bump cibuildwheel to 2.23.3 --- sdks/python/build.gradle | 2 +- sdks/python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index c8f02262d3fd..970020da8605 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -220,7 +220,7 @@ platform_identifiers_map.each { platform, idsuffix -> args '-c', ". ${envdir}/bin/activate && " + // note: sync cibuildwheel version with GitHub Action // .github/workflows/build_wheel.yml:build_wheels "Install cibuildwheel" step - "pip install cibuildwheel==2.17.0 setuptools && " + + "pip install cibuildwheel==2.23.3 setuptools && " + "cibuildwheel --print-build-identifiers --platform ${platform} --archs ${archs} && " + "cibuildwheel --output-dir ${buildDir} --platform ${platform} --archs ${archs} " } diff --git a/sdks/python/setup.py b/sdks/python/setup.py index ea1a99d4b2dc..105c58848c5e 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -280,7 +280,7 @@ def get_portability_package_data(): python_requires = '>=3.9' -if sys.version_info.major == 3 and sys.version_info.minor >= 13: +if sys.version_info.major == 3 and sys.version_info.minor >= 14: warnings.warn( 'This version of Apache Beam has not been sufficiently tested on ' 'Python %s.%s. You may encounter bugs or missing features.' % From e1725f2717f1264098f03520a97bf2e9b70dffce Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 1 Oct 2025 19:35:43 +0000 Subject: [PATCH 18/23] introduce ML image requirements --- .../ml/py313/ml_image_requirements.txt | 240 ++++++++++++++++++ sdks/python/setup.py | 2 +- 2 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 sdks/python/container/ml/py313/ml_image_requirements.txt diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py313/ml_image_requirements.txt new file mode 100644 index 000000000000..960772cb9e93 --- /dev/null +++ b/sdks/python/container/ml/py313/ml_image_requirements.txt @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py313 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.1 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +anyio==4.11.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.3.0 +beartype==0.21.0 +beautifulsoup4==4.14.2 +bs4==0.0.2 +build==1.3.0 +cachetools==6.2.0 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.3.0 +cloud-sql-python-connector==1.18.4 +crcmod==1.7 +cryptography==46.0.2 +Cython==3.1.4 +dill==0.3.1.1 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.17.0 +execnet==2.1.1 +fastavro==1.12.0 +fasteners==0.20 +filelock==3.19.1 +flatbuffers==25.9.23 +freezegun==1.5.5 +frozenlist==1.7.0 +fsspec==2025.9.0 +future==1.0.0 +gast==0.6.0 +google-api-core==2.25.1 +google-apitools==0.5.35 +google-auth==2.41.1 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.118.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigtable==2.32.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.32.0 +google-cloud-language==2.17.2 +google-cloud-pubsub==2.31.1 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.14.2 +google-cloud-secret-manager==2.24.0 +google-cloud-spanner==3.58.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.16.2 +google-cloud-vision==3.10.2 +google-crc32c==1.7.1 +google-genai==1.39.1 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.4 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio==1.75.1 +grpcio-status==1.75.1 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.10 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.35.3 +hypothesis==6.140.2 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.2 +jsonpickle==3.4.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keras==3.11.3 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.9 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +ml_dtypes==0.5.3 +mmh3==5.2.0 +mock==5.2.0 +more-itertools==10.8.0 +mpmath==1.3.0 +multidict==6.6.4 +namex==0.1.0 +networkx==3.5 +nltk==3.9.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-cupti-cu12==12.6.80 +nvidia-cuda-nvrtc-cu12==12.6.77 +nvidia-cuda-runtime-cu12==12.6.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-cufft-cu12==11.3.0.4 +nvidia-cufile-cu12==1.11.1.6 +nvidia-curand-cu12==10.3.7.77 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cusparse-cu12==12.5.4.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-nccl-cu12==2.26.2 +nvidia-nvjitlink-cu12==12.6.85 +nvidia-nvtx-cu12==12.6.77 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 +opentelemetry-semantic-conventions==0.58b0 +opt_einsum==3.4.0 +optree==0.17.0 +oracledb==3.3.0 +orjson==3.11.3 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.5 +pillow==11.3.0 +pip==25.2 +pluggy==1.6.0 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==6.33.0rc1 +psycopg2-binary==2.9.10 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.10.1 +pymilvus==2.6.2 +pymongo==4.15.1 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pyproject_hooks==1.2.0 +pytest==8.4.2 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.17.1 +pytz==2025.2 +PyYAML==6.0.3 +redis==5.3.1 +referencing==0.36.2 +regex==2025.9.18 +requests==2.32.5 +requests-mock==1.12.1 +rich==14.1.0 +rpds-py==0.27.1 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.16.2 +scramp==1.4.6 +SecretStorage==3.4.0 +setuptools==80.9.0 +shapely==2.1.2 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.8 +SQLAlchemy==2.0.43 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tensorflow==2.20.0 +tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.13.1 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +torch==2.7.1 +tqdm==4.67.1 +transformers==4.55.4 +triton==3.3.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +ujson==5.11.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.3 +yarl==1.20.1 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 105c58848c5e..35febcce2467 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -587,7 +587,7 @@ def get_portability_package_data(): # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing 'torch': ['torch>=1.9.0,<2.8.0'], - 'tensorflow': ['tensorflow>=2.12rc1,<2.17'], + 'tensorflow': ['tensorflow>=2.12rc1,<2.21.0'], 'transformers': [ 'transformers>=4.28.0,<4.56.0', 'tensorflow>=2.12.0', From ee637cdfd7d445a7888d7d37651ce81bcad919f3 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 2 Oct 2025 13:55:46 +0000 Subject: [PATCH 19/23] add env for py313-ml --- sdks/python/tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 9e428ba251a5..89ad8c095b11 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -116,8 +116,8 @@ commands = /bin/sh -c "pip freeze | grep -E tensorflow" bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py312-ml] -# many packages do not support py3.12 +[testenv:py{312,313}-ml] +# many packages do not support py3.12+ # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. deps = accelerate>=1.6.0 From 9308dccd98abfc5553c66d463e80cf8fb2e0e3fb Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 2 Oct 2025 14:10:15 +0000 Subject: [PATCH 20/23] remove datatables dep in 313ml --- sdks/python/setup.py | 14 ++++++++++++++ sdks/python/tox.ini | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 7f69e345c64d..2f4a2d56dda1 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -557,6 +557,20 @@ def get_portability_package_data(): 'torch', 'transformers', ], + 'p313_ml_test': [ + 'embeddings', + 'onnxruntime', + 'langchain', + 'sentence-transformers', + 'skl2onnx', + 'pillow', + 'pyod', + 'tensorflow', + 'tensorflow-hub', + 'tf2onnx', + 'torch', + 'transformers', + ], 'aws': ['boto3>=1.9,<2'], 'azure': [ 'azure-storage-blob>=12.3.2,<13', diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 89ad8c095b11..411ab1add416 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -116,8 +116,8 @@ commands = /bin/sh -c "pip freeze | grep -E tensorflow" bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{312,313}-ml] -# many packages do not support py3.12+ +[testenv:py312-ml] +# many packages do not support py3.12 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. deps = accelerate>=1.6.0 @@ -128,6 +128,18 @@ commands = /bin/sh -c "pip freeze | grep -E tensorflow" bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" +[testenv:py313-ml] +# many packages do not support py3.13, and datatables breaks after 3.12. +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +deps = + accelerate>=1.6.0 +setenv = +extras = test,gcp,dataframe,p313_ml_test +commands = + # Log tensorflow version for debugging + /bin/sh -c "pip freeze | grep -E tensorflow" + bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" + [testenv:py{39,310,311,31,313}-dask] extras = test,dask,dataframes commands_pre = From be11a3ef913e522217dd58c6ea53ac24041f12a6 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 2 Oct 2025 17:43:42 +0000 Subject: [PATCH 21/23] deleted commented out lines --- .../interactive/testing/integration/tests/screen_diff_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py b/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py index 0470e94fbff5..dbb978b44619 100644 --- a/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py +++ b/sdks/python/apache_beam/runners/interactive/testing/integration/tests/screen_diff_test.py @@ -28,7 +28,6 @@ @pytest.mark.timeout(300) class DataFramesTest(BaseTestCase): def __init__(self, *args, **kwargs): - # kwargs['golden_size'] = (1024, 10000) super().__init__(*args, **kwargs) def explicit_wait(self): @@ -50,7 +49,6 @@ def test_dataframes(self): @pytest.mark.timeout(300) class InitSquareCubeTest(BaseTestCase): def __init__(self, *args, **kwargs): - # kwargs['golden_size'] = (1024, 10000) super().__init__(*args, **kwargs) def test_init_square_cube_notebook(self): From ffe239f54d886fcfaf8109d45e7229831dc3ef9d Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 2 Oct 2025 18:44:53 +0000 Subject: [PATCH 22/23] modify setup.py for ml dependency clarity --- sdks/python/setup.py | 82 +++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 2f4a2d56dda1..32f3a397b91c 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -162,6 +162,21 @@ def cythonize(*args, **kwargs): milvus_dependency = ['pymilvus>=2.5.10,<3.0.0'] +ml_base = [ + 'embeddings', + 'onnxruntime', + 'langchain', + 'sentence-transformers', + 'skl2onnx', + 'pillow', + 'pyod', + 'tensorflow', + 'tensorflow-hub', + 'tf2onnx', + 'torch', + 'transformers', +], + def find_by_ext(root_dir, ext): for root, _, files in os.walk(root_dir): @@ -407,13 +422,13 @@ def get_portability_package_data(): # BEAM-8840: Do NOT use tests_require or setup_requires. extras_require={ 'dill': [ - # Dill doesn't have forwards-compatibility guarantees within minor - # version. Pickles created with a new version of dill may not - # unpickle using older version of dill. It is best to use the same - # version of dill on client and server, therefore list of allowed - # versions is very narrow. - # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', + # Dill doesn't have forwards-compatibility guarantees within minor + # version. Pickles created with a new version of dill may not + # unpickle using older version of dill. It is best to use the same + # version of dill on client and server, therefore list of allowed + # versions is very narrow. + # See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', ], 'docs': [ 'jinja2>=3.0,<3.2', @@ -521,56 +536,19 @@ def get_portability_package_data(): # can find out early when Beam doesn't work with new versions. 'ml_test': [ 'datatable', - 'embeddings', - 'langchain', - 'onnxruntime', - 'sentence-transformers', - 'skl2onnx', - 'pillow', - 'pyod', - 'tensorflow', - 'tensorflow-hub', # tensorflow-transform requires dill, but doesn't set dill as a # hard requirement in setup.py. 'dill', 'tensorflow-transform', - 'tf2onnx', - 'torch', - 'transformers', # Comment out xgboost as it is breaking presubmit python ml # tests due to tag check introduced since pip 24.2 # https://github.com/apache/beam/issues/31285 # 'xgboost<2.0', # https://github.com/apache/beam/issues/31252 - ], + ] + ml_base, 'p312_ml_test': [ 'datatable', - 'embeddings', - 'onnxruntime', - 'langchain', - 'sentence-transformers', - 'skl2onnx', - 'pillow', - 'pyod', - 'tensorflow', - 'tensorflow-hub', - 'tf2onnx', - 'torch', - 'transformers', - ], - 'p313_ml_test': [ - 'embeddings', - 'onnxruntime', - 'langchain', - 'sentence-transformers', - 'skl2onnx', - 'pillow', - 'pyod', - 'tensorflow', - 'tensorflow-hub', - 'tf2onnx', - 'torch', - 'transformers', - ], + ] + ml_base, + 'p313_ml_test': ml_base, 'aws': ['boto3>=1.9,<2'], 'azure': [ 'azure-storage-blob>=12.3.2,<13', @@ -608,10 +586,12 @@ def get_portability_package_data(): 'torch>=1.9.0' ], 'tft': [ - 'tensorflow_transform>=1.14.0,<1.15.0' - # tensorflow-transform requires dill, but doesn't set dill as a - # hard requirement in setup.py. - , 'dill'], + 'tensorflow_transform>=1.14.0,<1.15.0' + # tensorflow-transform requires dill, but doesn't set dill as a + # hard requirement in setup.py. + , + 'dill' + ], 'onnx': [ 'onnxruntime==1.13.1', 'torch==1.13.1', From c6e701b3485b8b89e1e547563d1417e63964ad6f Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 2 Oct 2025 18:49:36 +0000 Subject: [PATCH 23/23] remove errant comma --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 32f3a397b91c..4ad898d4b7cb 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -175,7 +175,7 @@ def cythonize(*args, **kwargs): 'tf2onnx', 'torch', 'transformers', -], +] def find_by_ext(root_dir, ext):