From c7e9df89cbdf72734668308863110f6fa8c0f75d Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 23 Oct 2025 22:52:06 +0400 Subject: [PATCH] Cherry-pick #36583 --- sdks/python/container/Dockerfile | 2 +- sdks/python/container/common.gradle | 2 +- .../ml/py310/ml_image_requirements.txt | 233 ++++++++++++++++++ .../ml/py311/ml_image_requirements.txt | 230 +++++++++++++++++ .../ml/py312/ml_image_requirements.txt | 229 +++++++++++++++++ .../ml/py313/ml_image_requirements.txt | 47 ++-- .../ml/py39/ml_image_requirements.txt | 233 ++++++++++++++++++ .../container/run_generate_requirements.sh | 5 +- sdks/python/setup.py | 6 + 9 files changed, 953 insertions(+), 34 deletions(-) create mode 100644 sdks/python/container/ml/py310/ml_image_requirements.txt create mode 100644 sdks/python/container/ml/py311/ml_image_requirements.txt create mode 100644 sdks/python/container/ml/py312/ml_image_requirements.txt create mode 100644 sdks/python/container/ml/py39/ml_image_requirements.txt diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index efd5a4a90d8a..56df970f75c2 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -50,7 +50,7 @@ RUN \ # Install required packages for Beam Python SDK and common dependencies used by users. # use --no-deps to ensure the list includes all transitive dependencies. - pip install --no-deps -r /tmp/base_image_requirements.txt && \ + pip install --no-deps -r /tmp/base_image_requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu && \ rm -rf /tmp/base_image_requirements.txt && \ python -c "import nltk; nltk.download('stopwords')" && \ rm /root/nltk_data/corpora/stopwords.zip && \ diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 0648bf4fa2e6..a67a9f0a7d83 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -53,7 +53,7 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { "${files(configurations.sdkSourceTarball.files).singleFile} " + "ml_image_requirements.txt " + "container/ml " + - "[gcp,dataframe,test,tensorflow,torch,transformers] " + + "[gcp,dataframe,test,ml_cpu] " + "${pipExtraOptions}" } } diff --git a/sdks/python/container/ml/py310/ml_image_requirements.txt b/sdks/python/container/ml/py310/ml_image_requirements.txt new file mode 100644 index 000000000000..59fe869e02d6 --- /dev/null +++ b/sdks/python/container/ml/py310/ml_image_requirements.txt @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py310 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.1 +aiofiles==25.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.1 +aiosignal==1.4.0 +annotated-types==0.7.0 +anyio==4.11.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.4.0 +backports.tarfile==1.2.0 +beartype==0.21.0 +beautifulsoup4==4.14.2 +bs4==0.0.2 +build==1.3.0 +cachetools==6.2.1 +certifi==2025.10.5 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.0 +cloud-sql-python-connector==1.18.5 +crcmod==1.7 +cryptography==46.0.3 +Cython==3.1.5 +dill==0.3.1.1 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.17.0 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.12.1 +fasteners==0.20 +filelock==3.20.0 +flatbuffers==25.9.23 +freezegun==1.5.5 +frozenlist==1.8.0 +fsspec==2025.9.0 +future==1.0.0 +gast==0.6.0 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 +google-auth==2.41.1 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.121.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigtable==2.33.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.31.1 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 +google-cloud-spanner==3.58.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 +google-crc32c==1.7.1 +google-genai==1.45.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.71.0 +greenlet==3.2.4 +grpc-google-iam-v1==0.14.3 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.65.5 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.15.1 +hdfs==2.7.3 +hf-xet==1.1.10 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.35.3 +hypothesis==6.142.2 +idna==3.11 +importlib_metadata==8.7.0 +iniconfig==2.3.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.2 +jsonpickle==3.4.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keras==3.11.3 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.9 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml_dtypes==0.5.3 +mmh3==5.2.0 +mock==5.2.0 +more-itertools==10.8.0 +mpmath==1.3.0 +multidict==6.7.0 +namex==0.1.0 +networkx==3.4.2 +nltk==3.9.2 +numpy==2.2.6 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 +opt_einsum==3.4.0 +optree==0.17.0 +oracledb==3.4.0 +orjson==3.11.3 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.5 +pillow==12.0.0 +pip==25.2 +pluggy==1.6.0 +propcache==0.4.1 +proto-plus==1.26.1 +protobuf==5.29.5 +psycopg2-binary==2.9.11 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.12.3 +pydantic_core==2.41.4 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.10.1 +pymilvus==2.5.16 +pymongo==4.15.3 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pyproject_hooks==1.2.0 +pytest==8.4.2 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.17.1 +pytz==2025.2 +PyYAML==6.0.3 +redis==5.3.1 +referencing==0.37.0 +regex==2025.10.23 +requests==2.32.5 +requests-mock==1.12.1 +rich==14.2.0 +rpds-py==0.27.1 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +scramp==1.4.6 +SecretStorage==3.4.0 +setuptools==80.9.0 +shapely==2.1.2 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.8 +SQLAlchemy==2.0.44 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tensorflow==2.20.0 +tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.13.2 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +tomli==2.3.0 +torch==2.8.0+cpu +tqdm==4.67.1 +transformers==4.55.4 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +ujson==5.11.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==2.0.0 +yarl==1.22.0 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/sdks/python/container/ml/py311/ml_image_requirements.txt b/sdks/python/container/ml/py311/ml_image_requirements.txt new file mode 100644 index 000000000000..351eac254c0e --- /dev/null +++ b/sdks/python/container/ml/py311/ml_image_requirements.txt @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py311 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.1 +aiofiles==25.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.1 +aiosignal==1.4.0 +annotated-types==0.7.0 +anyio==4.11.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.4.0 +backports.tarfile==1.2.0 +beartype==0.21.0 +beautifulsoup4==4.14.2 +bs4==0.0.2 +build==1.3.0 +cachetools==6.2.1 +certifi==2025.10.5 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.0 +cloud-sql-python-connector==1.18.5 +crcmod==1.7 +cryptography==46.0.3 +Cython==3.1.5 +dill==0.3.1.1 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.17.0 +execnet==2.1.1 +fastavro==1.12.1 +fasteners==0.20 +filelock==3.20.0 +flatbuffers==25.9.23 +freezegun==1.5.5 +frozenlist==1.8.0 +fsspec==2025.9.0 +future==1.0.0 +gast==0.6.0 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 +google-auth==2.41.1 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.121.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigtable==2.33.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.31.1 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 +google-cloud-spanner==3.58.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 +google-crc32c==1.7.1 +google-genai==1.45.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.71.0 +greenlet==3.2.4 +grpc-google-iam-v1==0.14.3 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.65.5 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.15.1 +hdfs==2.7.3 +hf-xet==1.1.10 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.35.3 +hypothesis==6.142.2 +idna==3.11 +importlib_metadata==8.7.0 +iniconfig==2.3.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.2 +jsonpickle==3.4.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keras==3.11.3 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.9 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml_dtypes==0.5.3 +mmh3==5.2.0 +mock==5.2.0 +more-itertools==10.8.0 +mpmath==1.3.0 +multidict==6.7.0 +namex==0.1.0 +networkx==3.5 +nltk==3.9.2 +numpy==2.2.6 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 +opt_einsum==3.4.0 +optree==0.17.0 +oracledb==3.4.0 +orjson==3.11.3 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.5 +pillow==12.0.0 +pip==25.2 +pluggy==1.6.0 +propcache==0.4.1 +proto-plus==1.26.1 +protobuf==5.29.5 +psycopg2-binary==2.9.11 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.12.3 +pydantic_core==2.41.4 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.10.1 +pymilvus==2.5.16 +pymongo==4.15.3 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pyproject_hooks==1.2.0 +pytest==8.4.2 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.17.1 +pytz==2025.2 +PyYAML==6.0.3 +redis==5.3.1 +referencing==0.37.0 +regex==2025.10.23 +requests==2.32.5 +requests-mock==1.12.1 +rich==14.2.0 +rpds-py==0.27.1 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.16.2 +scramp==1.4.6 +SecretStorage==3.4.0 +setuptools==80.9.0 +shapely==2.1.2 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.8 +SQLAlchemy==2.0.44 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tensorflow==2.20.0 +tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.13.2 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +torch==2.8.0+cpu +tqdm==4.67.1 +transformers==4.55.4 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +ujson==5.11.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==2.0.0 +yarl==1.22.0 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/sdks/python/container/ml/py312/ml_image_requirements.txt b/sdks/python/container/ml/py312/ml_image_requirements.txt new file mode 100644 index 000000000000..e4b64d509dd5 --- /dev/null +++ b/sdks/python/container/ml/py312/ml_image_requirements.txt @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py312 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.1 +aiofiles==25.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.1 +aiosignal==1.4.0 +annotated-types==0.7.0 +anyio==4.11.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +attrs==25.4.0 +beartype==0.21.0 +beautifulsoup4==4.14.2 +bs4==0.0.2 +build==1.3.0 +cachetools==6.2.1 +certifi==2025.10.5 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.0 +cloud-sql-python-connector==1.18.5 +crcmod==1.7 +cryptography==46.0.3 +Cython==3.1.5 +dill==0.3.1.1 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.17.0 +execnet==2.1.1 +fastavro==1.12.1 +fasteners==0.20 +filelock==3.20.0 +flatbuffers==25.9.23 +freezegun==1.5.5 +frozenlist==1.8.0 +fsspec==2025.9.0 +future==1.0.0 +gast==0.6.0 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 +google-auth==2.41.1 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.121.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigtable==2.33.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.31.1 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 +google-cloud-spanner==3.58.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 +google-crc32c==1.7.1 +google-genai==1.45.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.71.0 +greenlet==3.2.4 +grpc-google-iam-v1==0.14.3 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.65.5 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.15.1 +hdfs==2.7.3 +hf-xet==1.1.10 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.35.3 +hypothesis==6.142.2 +idna==3.11 +importlib_metadata==8.7.0 +iniconfig==2.3.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.2 +jsonpickle==3.4.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keras==3.11.3 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.9 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml_dtypes==0.5.3 +mmh3==5.2.0 +mock==5.2.0 +more-itertools==10.8.0 +mpmath==1.3.0 +multidict==6.7.0 +namex==0.1.0 +networkx==3.5 +nltk==3.9.2 +numpy==2.2.6 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 +opt_einsum==3.4.0 +optree==0.17.0 +oracledb==3.4.0 +orjson==3.11.3 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.5 +pillow==12.0.0 +pip==25.2 +pluggy==1.6.0 +propcache==0.4.1 +proto-plus==1.26.1 +protobuf==5.29.5 +psycopg2-binary==2.9.11 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.12.3 +pydantic_core==2.41.4 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.10.1 +pymilvus==2.5.16 +pymongo==4.15.3 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pyproject_hooks==1.2.0 +pytest==8.4.2 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.17.1 +pytz==2025.2 +PyYAML==6.0.3 +redis==5.3.1 +referencing==0.37.0 +regex==2025.10.23 +requests==2.32.5 +requests-mock==1.12.1 +rich==14.2.0 +rpds-py==0.27.1 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.16.2 +scramp==1.4.6 +SecretStorage==3.4.0 +setuptools==80.9.0 +shapely==2.1.2 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.8 +SQLAlchemy==2.0.44 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tensorflow==2.20.0 +tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.13.2 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +torch==2.8.0+cpu +tqdm==4.67.1 +transformers==4.55.4 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +ujson==5.11.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==2.0.0 +yarl==1.22.0 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py313/ml_image_requirements.txt index ebc712ee6a70..a974d8606987 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py313/ml_image_requirements.txt @@ -42,8 +42,8 @@ charset-normalizer==3.4.3 click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 dnspython==2.8.0 docker==7.1.0 @@ -74,22 +74,22 @@ google-cloud-language==2.17.2 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 google-genai==1.39.1 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 grpc-google-iam-v1==0.14.2 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.76.0 +grpcio-status==1.76.0 guppy3==3.1.5 h11==0.16.0 h5py==3.14.0 @@ -99,8 +99,8 @@ httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.142.2 +idna==3.11 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 @@ -130,20 +130,6 @@ namex==0.1.0 networkx==3.5 nltk==3.9.2 numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.37.0 @@ -190,8 +176,8 @@ python-tds==1.17.1 pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 -referencing==0.36.2 -regex==2025.9.18 +referencing==0.37.0 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 rich==14.1.0 @@ -221,10 +207,9 @@ termcolor==3.1.0 testcontainers==4.13.1 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +torch==2.8.0+cpu tqdm==4.67.1 transformers==4.55.4 -triton==3.3.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 @@ -234,7 +219,7 @@ virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/ml/py39/ml_image_requirements.txt b/sdks/python/container/ml/py39/ml_image_requirements.txt new file mode 100644 index 000000000000..8e92499a3b1e --- /dev/null +++ b/sdks/python/container/ml/py39/ml_image_requirements.txt @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam py39 container image. +# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. +# Do not edit manually, adjust ../base_image_requirements_manual.txt or +# Apache Beam's setup.py instead, and regenerate the list. +# You will need Python interpreters for all versions supported by Beam, see: +# https://s.apache.org/beam-python-dev-wiki +# Reach out to a committer if you need help. + +absl-py==2.3.1 +aiofiles==25.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.1 +aiosignal==1.4.0 +annotated-types==0.7.0 +anyio==4.11.0 +asn1crypto==1.5.1 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==25.4.0 +backports.tarfile==1.2.0 +beartype==0.21.0 +beautifulsoup4==4.14.2 +bs4==0.0.2 +build==1.3.0 +cachetools==6.2.1 +certifi==2025.10.5 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.1.8 +cloud-sql-python-connector==1.18.5 +crcmod==1.7 +cryptography==46.0.3 +Cython==3.1.5 +dill==0.3.1.1 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.17.0 +exceptiongroup==1.3.0 +execnet==2.1.1 +fastavro==1.12.1 +fasteners==0.20 +filelock==3.19.1 +flatbuffers==25.9.23 +freezegun==1.5.5 +frozenlist==1.8.0 +fsspec==2025.9.0 +future==1.0.0 +gast==0.6.0 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 +google-auth==2.41.1 +google-auth-httplib2==0.2.0 +google-cloud-aiplatform==1.121.0 +google-cloud-bigquery==3.38.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigtable==2.33.0 +google-cloud-core==2.4.3 +google-cloud-datastore==2.21.0 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.31.1 +google-cloud-pubsublite==1.12.0 +google-cloud-recommendations-ai==0.10.18 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 +google-cloud-spanner==3.58.0 +google-cloud-storage==2.19.0 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 +google-crc32c==1.7.1 +google-genai==1.45.0 +google-pasta==0.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.71.0 +greenlet==3.2.4 +grpc-google-iam-v1==0.14.3 +grpc-interceptor==0.15.4 +grpcio==1.65.5 +grpcio-status==1.65.5 +guppy3==3.1.5 +h11==0.16.0 +h5py==3.14.0 +hdfs==2.7.3 +hf-xet==1.1.10 +httpcore==1.0.9 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.35.3 +hypothesis==6.141.1 +idna==3.11 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.6 +joblib==1.5.2 +jsonpickle==3.4.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keras==3.10.0 +keyring==25.6.0 +keyrings.google-artifactregistry-auth==1.1.2 +libclang==18.1.1 +Markdown==3.9 +markdown-it-py==3.0.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +milvus-lite==2.5.1 +ml_dtypes==0.5.3 +mmh3==5.2.0 +mock==5.2.0 +more-itertools==10.8.0 +mpmath==1.3.0 +multidict==6.7.0 +namex==0.1.0 +networkx==3.2.1 +nltk==3.9.2 +numpy==2.0.2 +oauth2client==4.1.3 +objsize==0.7.1 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 +opt_einsum==3.4.0 +optree==0.17.0 +oracledb==3.4.0 +orjson==3.11.3 +overrides==7.7.0 +packaging==25.0 +pandas==2.2.3 +parameterized==0.9.0 +pg8000==1.31.5 +pillow==11.3.0 +pip==25.2 +pluggy==1.6.0 +propcache==0.4.1 +proto-plus==1.26.1 +protobuf==5.29.5 +psycopg2-binary==2.9.9 +pyarrow==18.1.0 +pyarrow-hotfix==0.7 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.12.3 +pydantic_core==2.41.4 +pydot==1.4.2 +Pygments==2.19.2 +PyHamcrest==2.1.0 +PyJWT==2.10.1 +pymilvus==2.5.16 +pymongo==4.15.3 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pyproject_hooks==1.2.0 +pytest==8.4.2 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +python-tds==1.17.1 +pytz==2025.2 +PyYAML==6.0.3 +redis==5.3.1 +referencing==0.36.2 +regex==2025.10.23 +requests==2.32.5 +requests-mock==1.12.1 +rich==14.2.0 +rpds-py==0.27.1 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.6.1 +scipy==1.13.1 +scramp==1.4.6 +SecretStorage==3.3.3 +setuptools==80.9.0 +shapely==2.0.7 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soupsieve==2.8 +SQLAlchemy==2.0.44 +sqlalchemy_pytds==1.0.2 +sqlparse==0.5.3 +sympy==1.14.0 +tenacity==8.5.0 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tensorflow==2.20.0 +tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" +termcolor==3.1.0 +testcontainers==4.13.2 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +tomli==2.3.0 +torch==2.8.0+cpu +tqdm==4.67.1 +transformers==4.54.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +ujson==5.11.0 +uritemplate==4.2.0 +urllib3==2.5.0 +virtualenv-clone==0.5.7 +websockets==15.0.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==2.0.0 +yarl==1.22.0 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index de14cbff2d50..d05783ebbdd0 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -80,7 +80,10 @@ pip install --upgrade pip setuptools wheel # Install dataframe deps to add have Dataframe support in released images. # Install test deps since some integration tests need dependencies, # such as pytest, installed in the runner environment. -pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" +# Force torch dependencies to be pulled from the PyTorch CPU wheel +# repository so that they don't include GPU dependencies with +# non-compliant licenses +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" --extra-index-url https://download.pytorch.org/whl/cpu pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt pip uninstall -y apache-beam diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 719d188ed266..8feb1fe1f019 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -593,6 +593,12 @@ def get_portability_package_data(): 'tensorflow>=2.12.0', 'torch>=1.9.0' ], + 'ml_cpu': [ + 'tensorflow>=2.12.0', + 'torch==2.8.0+cpu', + 'transformers>=4.28.0,<4.55.0; python_version < "3.10"', + 'transformers>=4.28.0,<4.56.0; python_version >= "3.10"' + ], 'tft': [ 'tensorflow_transform>=1.14.0,<1.15.0' # tensorflow-transform requires dill, but doesn't set dill as a