Skip to content

Commit 2d617c0

Browse files
jrmccluskeyjrmccluskey
authored
Remove Python 3.9 support from Apache Beam (#36665)
* Remove Python 3.9 support from Apache Beam * add version removal to CHANGES.md * wrap python version in quotes in build_wheels * wrap remaining bare version sets in quotes * fix typo * change default expansion service container in BeamModulePlugin.groovy * bump manual container versions * revert python_requires change to allow 3.9 workflows to execute on PR * modify linting versions to fix pylint breakage * update pylintrc to remove new checks * fix typo * fix extra disables * try new ignore format * more linting changes * uncap cores for linting, explicitly cover cloudbuild files in exclusions * add s3 client to exclusion list * build out isort config * exclude .pyx files * revert errant version bump * standardize bools * initial import ordering changes * fix circulat dependencies, clear ungrouped imports * last bit of pylint fixes * exclude portability api files * extra exclusion of portability/api/__init__.py * fix python version in flink container * make the whitespace precommit happy * restore io init * restore vertex test ordering
1 parent 0448a5b commit 2d617c0

File tree

295 files changed

+961
-596
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

295 files changed

+961
-596
lines changed

‎.github/actions/setup-default-test-properties/test-properties.json‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
22
"PythonTestProperties": {
3-
"ALL_SUPPORTED_VERSIONS": ["3.9", "3.10", "3.11", "3.12", "3.13"],
4-
"LOWEST_SUPPORTED": ["3.9"],
3+
"ALL_SUPPORTED_VERSIONS": ["3.10", "3.11", "3.12", "3.13"],
4+
"LOWEST_SUPPORTED": ["3.10"],
55
"HIGHEST_SUPPORTED": ["3.13"],
6-
"ESSENTIAL_VERSIONS": ["3.9", "3.12"],
7-
"CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.9", "3.12", "3.13"],
6+
"ESSENTIAL_VERSIONS": ["3.10", "3.13"],
7+
"CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.10", "3.12", "3.13"],
88
"CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"],
9-
"VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12", "3.13"],
10-
"LOAD_TEST_PYTHON_VERSION": "3.9",
11-
"CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.9",
12-
"DEFAULT_INTERPRETER": "python3.9",
9+
"VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.10", "3.11", "3.12", "3.13"],
10+
"LOAD_TEST_PYTHON_VERSION": "3.10",
11+
"CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.10",
12+
"DEFAULT_INTERPRETER": "python3.10",
1313
"TOX_ENV": ["Cloud", "Cython"]
1414
},
1515
"JavaTestProperties": {

‎.github/actions/setup-environment-action/action.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ runs:
5050
if: ${{ inputs.python-version != '' }}
5151
uses: actions/setup-python@v5
5252
with:
53-
python-version: ${{ inputs.python-version == 'default' && '3.9' || inputs.python-version }}
53+
python-version: ${{ inputs.python-version == 'default' && '3.10' || inputs.python-version }}
5454
cache: ${{ inputs.python-cache && 'pip' || 'none' }}
5555
cache-dependency-path: |
5656
sdks/python/setup.py
@@ -64,10 +64,10 @@ runs:
6464
sdks/python/target/.tox
6565
!sdks/python/target/.tox/**/log
6666
!sdks/python/target/.tox/.package_cache
67-
key: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-${{ hashFiles('sdks/python/setup.py') }}
67+
key: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-${{ hashFiles('sdks/python/setup.py') }}
6868
restore-keys: |
69-
tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-
70-
tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-
69+
tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-
70+
tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '310' || inputs.python-version }}-
7171
7272
- name: Install Java
7373
if: ${{ inputs.java-version != '' }}

‎.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ jobs:
7474
uses: ./.github/actions/setup-environment-action
7575
with:
7676
python-version: |
77-
3.9
7877
3.10
78+
3.11
7979
- name: Prepare test arguments
8080
uses: ./.github/actions/test-arguments-action
8181
with:
@@ -89,6 +89,6 @@ jobs:
8989
with:
9090
gradle-command: :sdks:python:test-suites:dataflow:tftTests
9191
arguments: |
92-
-PpythonVersion=3.9 \
92+
-PpythonVersion=3.10 \
9393
-Prunner=DataflowRunner \
9494
'-Popts=${{ env.beam_CloudML_Benchmarks_Dataflow_test_arguments_1 }}'

‎.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ jobs:
9696
--info \
9797
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
9898
-Prunner=DataflowRunner \
99-
-PpythonVersion=3.9 \
99+
-PpythonVersion=3.10 \
100100
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
101101
- name: run CoGBK 2GB of 100B records with multiple keys
102102
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -105,7 +105,7 @@ jobs:
105105
arguments: |
106106
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
107107
-Prunner=DataflowRunner \
108-
-PpythonVersion=3.9 \
108+
-PpythonVersion=3.10 \
109109
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
110110
- name: run CoGBK reiterate 4 times 10kB values
111111
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -114,7 +114,7 @@ jobs:
114114
arguments: |
115115
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
116116
-Prunner=DataflowRunner \
117-
-PpythonVersion=3.9 \
117+
-PpythonVersion=3.10 \
118118
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \
119119
- name: run CoGBK reiterate 4 times 2MB values
120120
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -123,5 +123,5 @@ jobs:
123123
arguments: |
124124
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
125125
-Prunner=DataflowRunner \
126-
-PpythonVersion=3.9 \
126+
-PpythonVersion=3.10 \
127127
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}'

‎.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ jobs:
9595
arguments: |
9696
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
9797
-Prunner=DataflowRunner \
98-
-PpythonVersion=3.9 \
98+
-PpythonVersion=3.10 \
9999
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
100100
- name: run CoGBK 2GB of 100B records with multiple keys
101101
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -104,7 +104,7 @@ jobs:
104104
arguments: |
105105
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
106106
-Prunner=DataflowRunner \
107-
-PpythonVersion=3.9 \
107+
-PpythonVersion=3.10 \
108108
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
109109
- name: run CoGBK reiterate 4 times 10kB values
110110
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -113,7 +113,7 @@ jobs:
113113
arguments: |
114114
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
115115
-Prunner=DataflowRunner \
116-
-PpythonVersion=3.9 \
116+
-PpythonVersion=3.10 \
117117
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \
118118
- name: run CoGBK reiterate 4 times 2MB values
119119
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -122,5 +122,5 @@ jobs:
122122
arguments: |
123123
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
124124
-Prunner=DataflowRunner \
125-
-PpythonVersion=3.9 \
125+
-PpythonVersion=3.10 \
126126
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}'

‎.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ jobs:
108108
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
109109
arguments: |
110110
--info \
111-
-PpythonVersion=3.9 \
111+
-PpythonVersion=3.10 \
112112
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
113113
-Prunner=PortableRunner \
114114
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
@@ -118,7 +118,7 @@ jobs:
118118
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
119119
arguments: |
120120
--info \
121-
-PpythonVersion=3.9 \
121+
-PpythonVersion=3.10 \
122122
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
123123
-Prunner=PortableRunner \
124124
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
@@ -128,7 +128,7 @@ jobs:
128128
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
129129
arguments: |
130130
--info \
131-
-PpythonVersion=3.9 \
131+
-PpythonVersion=3.10 \
132132
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
133133
-Prunner=PortableRunner \
134134
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \

‎.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ jobs:
9292
arguments: |
9393
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
9494
-Prunner=DataflowRunner \
95-
-PpythonVersion=3.9 \
95+
-PpythonVersion=3.10 \
9696
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \
9797
- name: run Combine Dataflow Batch Python Load Test 2 (fanout 4)
9898
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -101,7 +101,7 @@ jobs:
101101
arguments: |
102102
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
103103
-Prunner=DataflowRunner \
104-
-PpythonVersion=3.9 \
104+
-PpythonVersion=3.10 \
105105
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \
106106
- name: run Combine Dataflow Batch Python Load Test 3 (fanout 8)
107107
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -110,5 +110,5 @@ jobs:
110110
arguments: |
111111
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
112112
-Prunner=DataflowRunner \
113-
-PpythonVersion=3.9 \
113+
-PpythonVersion=3.10 \
114114
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}'

‎.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ jobs:
9292
arguments: |
9393
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
9494
-Prunner=DataflowRunner \
95-
-PpythonVersion=3.9 \
95+
-PpythonVersion=3.10 \
9696
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \
9797
- name: run 2GB Fanout 4 test
9898
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -101,7 +101,7 @@ jobs:
101101
arguments: |
102102
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
103103
-Prunner=DataflowRunner \
104-
-PpythonVersion=3.9 \
104+
-PpythonVersion=3.10 \
105105
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \
106106
- name: run 2GB Fanout 8 test
107107
uses: ./.github/actions/gradle-command-self-hosted-action
@@ -110,5 +110,5 @@ jobs:
110110
arguments: |
111111
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
112112
-Prunner=DataflowRunner \
113-
-PpythonVersion=3.9 \
113+
-PpythonVersion=3.10 \
114114
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}'

‎.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ jobs:
107107
with:
108108
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
109109
arguments: |
110-
-PpythonVersion=3.9 \
110+
-PpythonVersion=3.10 \
111111
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
112112
-Prunner=PortableRunner \
113113
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \
@@ -121,7 +121,7 @@ jobs:
121121
with:
122122
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
123123
arguments: |
124-
-PpythonVersion=3.9 \
124+
-PpythonVersion=3.10 \
125125
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
126126
-Prunner=PortableRunner \
127127
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \
@@ -130,7 +130,7 @@ jobs:
130130
with:
131131
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
132132
arguments: |
133-
-PpythonVersion=3.9 \
133+
-PpythonVersion=3.10 \
134134
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
135135
-Prunner=PortableRunner \
136136
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \

‎.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ jobs:
109109
with:
110110
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
111111
arguments: |
112-
-PpythonVersion=3.9 \
112+
-PpythonVersion=3.10 \
113113
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
114114
-Prunner=PortableRunner \
115115
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \
@@ -118,7 +118,7 @@ jobs:
118118
with:
119119
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
120120
arguments: |
121-
-PpythonVersion=3.9 \
121+
-PpythonVersion=3.10 \
122122
-PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
123123
-Prunner=PortableRunner \
124124
'-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \

0 commit comments

Comments
 (0)