Skip to content

Commit 0c4a82d

Browse files
jrmccluskeyjrmccluskey
committed
generate base image requirements, related changes
1 parent 64d038d commit 0c4a82d

File tree

6 files changed

+235
-30
lines changed

6 files changed

+235
-30
lines changed

‎build.gradle.kts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,7 @@ tasks.register("pythonPreCommit") {
494494
dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311")
495495
dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312")
496496
dependsOn(":sdks:python:test-suites:tox:py313:preCommitPy313")
497+
dependsOn(":sdks:python:test-suites:tox:py314:preCommitPy314")
497498
}
498499

499500
tasks.register("pythonPreCommitIT") {
@@ -510,6 +511,7 @@ tasks.register("pythonDockerBuildPreCommit") {
510511
dependsOn(":sdks:python:container:py311:docker")
511512
dependsOn(":sdks:python:container:py312:docker")
512513
dependsOn(":sdks:python:container:py313:docker")
514+
dependsOn(":sdks:python:container:py314:docker")
513515
}
514516

515517
tasks.register("pythonLintPreCommit") {
@@ -820,14 +822,21 @@ tasks.register("python313PostCommit") {
820822
dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy313")
821823
}
822824

825+
tasks.register("python314PostCommit") {
826+
dependsOn(":sdks:python:test-suites:dataflow:py314:postCommitIT")
827+
dependsOn(":sdks:python:test-suites:direct:py314:postCommitIT")
828+
dependsOn(":sdks:python:test-suites:direct:py314:hdfsIntegrationTest")
829+
dependsOn(":sdks:python:test-suites:portable:py314:postCommitPy314")
830+
}
831+
823832
tasks.register("portablePythonPreCommit") {
824833
dependsOn(":sdks:python:test-suites:portable:py310:preCommitPy310")
825-
dependsOn(":sdks:python:test-suites:portable:py313:preCommitPy313")
834+
dependsOn(":sdks:python:test-suites:portable:py314:preCommitPy314")
826835
}
827836

828837
tasks.register("pythonSparkPostCommit") {
829838
dependsOn(":sdks:python:test-suites:portable:py310:sparkValidatesRunner")
830-
dependsOn(":sdks:python:test-suites:portable:py313:sparkValidatesRunner")
839+
dependsOn(":sdks:python:test-suites:portable:py314:sparkValidatesRunner")
831840
}
832841

833842
tasks.register("websitePreCommit") {

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,4 @@ docker_image_default_repo_prefix=beam_
4141
# supported flink versions
4242
flink_versions=1.17,1.18,1.19,1.20,2.0
4343
# supported python versions
44-
python_versions=3.10,3.11,3.12,3.13
44+
python_versions=3.10,3.11,3.12,3.13,3.14

‎sdks/python/container/common.gradle‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,23 +46,24 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") {
4646
"${pipExtraOptions}"
4747
}
4848
// Generate versions for ML dependencies
49-
exec {
50-
executable 'sh'
51-
args '-c', "cd ${rootDir} && ${runScriptsPath} " +
52-
"${project.ext.pythonVersion} " +
53-
"${files(configurations.sdkSourceTarball.files).singleFile} " +
54-
"base_image_requirements.txt " +
49+
// Skip for 3.14 since there isn't a tensorflow release that supports it
50+
// yet.
51+
if ("${project.ext.pythonVersion}" != "3.14") {
52+
exec {
53+
executable 'sh'
54+
args '-c', "cd ${rootDir} && ${runScriptsPath} " +
55+
"${project.ext.pythonVersion} " +
56+
"${files(configurations.sdkSourceTarball.files).singleFile} " +
57+
"base_image_requirements.txt " +
5558
"container/ml " +
5659
"[gcp,dataframe,test,ml_cpu,tfrecord,yaml] " +
5760
"${pipExtraOptions}"
5861
}
62+
}
5963
// TODO(https://github.com/apache/beam/issues/36637)
60-
// Skip generating Python 3.13 requirements for now since not all 3.13
61-
// wheels are available/buildable.
62-
// Also skip 3.9 because there are some dependency version conflicts. This
63-
// is fine since 3.9 will be EoL by the next release, and we can remove
64-
// this condition once we remove support entirely.
65-
if ("${project.ext.pythonVersion}" != "3.13" && "${project.ext.pythonVersion}" != "3.9") {
64+
// Skip generating Python 3.13 and 3.14 requirements for now since not all 3.13
65+
// and 3.14 wheels are available/buildable.
66+
if ("${project.ext.pythonVersion}" != "3.13" && "${project.ext.pythonVersion}" != "3.14") {
6667
// GPU requirements not used for any containers directly due to
6768
// licensing, but can be picked up by customers or other consumers for
6869
// use.

‎sdks/python/container/py314/base_image_requirements.txt‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Autogenerated requirements file for Apache Beam py314 container image.
17+
# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update.
18+
# Do not edit manually, adjust ../base_image_requirements_manual.txt or
19+
# Apache Beam's setup.py instead, and regenerate the list.
20+
# You will need Python interpreters for all versions supported by Beam, see:
21+
# https://s.apache.org/beam-python-dev-wiki
22+
# Reach out to a committer if you need help.
23+
24+
aiofiles==25.1.0
25+
aiohappyeyeballs==2.6.1
26+
aiohttp==3.13.3
27+
aiosignal==1.4.0
28+
annotated-types==0.7.0
29+
anyio==4.12.1
30+
asn1crypto==1.5.1
31+
attrs==25.4.0
32+
beartype==0.22.9
33+
beautifulsoup4==4.14.3
34+
betterproto==2.0.0b6
35+
bs4==0.0.2
36+
build==1.4.0
37+
cachetools==6.2.6
38+
certifi==2026.1.4
39+
cffi==2.0.0
40+
charset-normalizer==3.4.4
41+
click==8.3.1
42+
cloud-sql-python-connector==1.20.0
43+
crcmod==1.7
44+
cryptography==46.0.4
45+
Cython==3.2.4
46+
dill==0.3.1.1
47+
distro==1.9.0
48+
dnspython==2.8.0
49+
docker==7.1.0
50+
docstring_parser==0.17.0
51+
envoy_data_plane==1.0.3
52+
execnet==2.1.2
53+
fastavro==1.12.1
54+
fasteners==0.20
55+
freezegun==1.5.5
56+
frozenlist==1.8.0
57+
future==1.0.0
58+
google-api-core==2.29.0
59+
google-apitools==0.5.35
60+
google-auth==2.49.0.dev0
61+
google-auth-httplib2==0.2.1
62+
google-cloud-aiplatform==1.135.0
63+
google-cloud-bigquery==3.40.0
64+
google-cloud-bigquery-storage==2.36.0
65+
google-cloud-bigtable==2.35.0
66+
google-cloud-build==3.35.0
67+
google-cloud-core==2.5.0
68+
google-cloud-datastore==2.23.0
69+
google-cloud-dlp==3.34.0
70+
google-cloud-kms==3.10.0
71+
google-cloud-language==2.19.0
72+
google-cloud-monitoring==2.29.0
73+
google-cloud-pubsub==2.34.0
74+
google-cloud-recommendations-ai==0.10.18
75+
google-cloud-resource-manager==1.16.0
76+
google-cloud-secret-manager==2.26.0
77+
google-cloud-spanner==3.62.0
78+
google-cloud-storage==2.19.0
79+
google-cloud-videointelligence==2.18.0
80+
google-cloud-vision==3.12.0
81+
google-crc32c==1.8.0
82+
google-genai==1.61.0
83+
google-resumable-media==2.8.0
84+
googleapis-common-protos==1.72.0
85+
greenlet==3.3.1
86+
grpc-google-iam-v1==0.14.3
87+
grpc-interceptor==0.15.4
88+
grpcio==1.76.0
89+
grpcio-status==1.76.0
90+
grpcio-tools==1.76.0
91+
grpclib==0.4.9
92+
guppy3==3.1.6
93+
h11==0.16.0
94+
h2==4.3.0
95+
hpack==4.1.0
96+
httpcore==1.0.9
97+
httplib2==0.31.2
98+
httpx==0.28.1
99+
hyperframe==6.1.0
100+
hypothesis==6.148.3
101+
idna==3.11
102+
importlib_metadata==8.7.1
103+
iniconfig==2.3.0
104+
jaraco.classes==3.4.0
105+
jaraco.context==6.1.0
106+
jaraco.functools==4.4.0
107+
jeepney==0.9.0
108+
Jinja2==3.1.6
109+
joblib==1.5.3
110+
jsonpickle==3.4.2
111+
jsonschema==4.26.0
112+
jsonschema-specifications==2025.9.1
113+
keyring==25.7.0
114+
keyrings.google-artifactregistry-auth==1.1.2
115+
MarkupSafe==3.0.3
116+
mmh3==5.2.0
117+
mock==5.2.0
118+
more-itertools==10.8.0
119+
multidict==6.7.1
120+
nltk==3.9.2
121+
numpy==2.4.2
122+
oauth2client==4.1.3
123+
objsize==0.7.1
124+
opentelemetry-api==1.39.1
125+
opentelemetry-resourcedetector-gcp==1.11.0a0
126+
opentelemetry-sdk==1.39.1
127+
opentelemetry-semantic-conventions==0.60b1
128+
oracledb==3.4.2
129+
orjson==3.11.7
130+
packaging==26.0
131+
pandas==2.2.3
132+
parameterized==0.9.0
133+
pg8000==1.31.5
134+
pip==26.0
135+
pluggy==1.6.0
136+
propcache==0.4.1
137+
proto-plus==1.27.1
138+
protobuf==6.33.5
139+
psycopg2-binary==2.9.11
140+
pyarrow==23.0.0
141+
pyarrow-hotfix==0.7
142+
pyasn1==0.6.2
143+
pyasn1_modules==0.4.2
144+
pycparser==3.0
145+
pydantic==2.12.5
146+
pydantic_core==2.41.5
147+
Pygments==2.19.2
148+
PyHamcrest==2.1.0
149+
pymilvus==2.6.8
150+
pymongo==4.16.0
151+
PyMySQL==1.1.2
152+
pyparsing==3.3.2
153+
pyproject_hooks==1.2.0
154+
pytest==8.4.2
155+
pytest-timeout==2.4.0
156+
pytest-xdist==3.8.0
157+
python-dateutil==2.9.0.post0
158+
python-dotenv==1.2.1
159+
python-tds==1.17.1
160+
pytz==2025.2
161+
PyYAML==6.0.3
162+
referencing==0.37.0
163+
regex==2026.1.15
164+
requests==2.32.5
165+
requests-mock==1.12.1
166+
rpds-py==0.30.0
167+
rsa==4.9.1
168+
scikit-learn==1.7.2
169+
scipy==1.17.0
170+
scramp==1.4.8
171+
SecretStorage==3.5.0
172+
setuptools==80.10.2
173+
six==1.17.0
174+
sniffio==1.3.1
175+
sortedcontainers==2.4.0
176+
soupsieve==2.8.3
177+
SQLAlchemy==2.0.46
178+
sqlalchemy_pytds==1.0.2
179+
sqlparse==0.5.5
180+
tenacity==8.5.0
181+
testcontainers==4.14.1
182+
threadpoolctl==3.6.0
183+
tqdm==4.67.3
184+
typing-inspection==0.4.2
185+
typing_extensions==4.15.0
186+
tzdata==2025.3
187+
urllib3==2.6.3
188+
virtualenv-clone==0.5.7
189+
websockets==15.0.1
190+
wheel==0.46.3
191+
wrapt==2.1.1
192+
yarl==1.22.0
193+
zipp==3.23.0
194+
zstandard==0.25.0

‎sdks/python/setup.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,8 @@ def cythonize(*args, **kwargs):
145145
pyarrow_dependency = ['']
146146
else:
147147
pyarrow_dependency = [
148-
'pyarrow>=3.0.0,<19.0.0',
148+
'pyarrow>=3.0.0,<19.0.0; python_version <= "3.13"',
149+
'pyarrow>=22.0.0,<24.0.0; python_version >= "3.14"',
149150
# NOTE(https://github.com/apache/beam/issues/29392): We can remove this
150151
# once Beam increases the pyarrow lower bound to a version that fixes CVE.
151152
# (lower bound >= 14.0.1)

‎sdks/python/test-suites/gradle.properties‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,34 +23,34 @@
2323
# dataflow test-suites
2424
# (TODO): https://github.com/apache/beam/issues/21971
2525
# Add python 3.10 to dataflow test-suites
26-
dataflow_precommit_it_task_py_versions=3.10,3.13
26+
dataflow_precommit_it_task_py_versions=3.10,3.14
2727
dataflow_mongodbio_it_task_py_versions=3.10
2828
dataflow_chicago_taxi_example_task_py_versions=3.10
2929

3030
# TODO: Enable following tests after making sure we have enough capacity.
31-
dataflow_validates_runner_batch_tests=3.10,3.13
32-
dataflow_validates_runner_streaming_tests=3.10,3.13
33-
dataflow_examples_postcommit_py_versions=3.13
31+
dataflow_validates_runner_batch_tests=3.10,3.14
32+
dataflow_validates_runner_streaming_tests=3.10,3.14
33+
dataflow_examples_postcommit_py_versions=3.14
3434
# TFX_BSL is not yet supported on Python 3.10.
3535
dataflow_cloudml_benchmark_tests_py_versions=3.10
3636
# direct runner test-suites
37-
direct_mongodbio_it_task_py_versions=3.13
37+
direct_mongodbio_it_task_py_versions=3.14
3838

3939
# flink runner test-suites
40-
flink_validates_runner_precommit_py_versions=3.13
41-
flink_validates_runner_postcommit_py_versions=3.10,3.13
42-
flink_examples_postcommit_py_versions=3.10,3.13
40+
flink_validates_runner_precommit_py_versions=3.14
41+
flink_validates_runner_postcommit_py_versions=3.10,3.14
42+
flink_examples_postcommit_py_versions=3.10,3.14
4343

4444
# samza runner test-suites
45-
samza_validates_runner_postcommit_py_versions=3.10,3.13
45+
samza_validates_runner_postcommit_py_versions=3.10,3.14
4646

4747
# spark runner test-suites
48-
spark_examples_postcommit_py_versions=3.10,3.13
48+
spark_examples_postcommit_py_versions=3.10,3.14
4949

5050
# prism runner test-suites
51-
prism_validates_runner_precommit_py_versions=3.13
52-
prism_validates_runner_postcommit_py_versions=3.10,3.13
53-
prism_examples_postcommit_py_versions=3.10,3.13
51+
prism_validates_runner_precommit_py_versions=3.14
52+
prism_validates_runner_postcommit_py_versions=3.10,3.14
53+
prism_examples_postcommit_py_versions=3.10,3.14
5454

5555
# cross language postcommit python test suites
56-
cross_language_validates_py_versions=3.10,3.13
56+
cross_language_validates_py_versions=3.10,3.14

0 commit comments

Comments
 (0)