[FLINK-38231][python] Standardise use of uv for PyFlink building, testing & linting

Author: autophagy

docs/content/docs/flinkDev/building.md

@@ -79,6 +79,15 @@ The `fast` and `skip-webui-build` profiles have a significant impact on the buil
     # the version printed here must be 3.9, 3.10, 3.11 or 3.12
     ```
 
+3. Uv for building, installing and managing Python environments (optional)
+
+    The tool [uv](https://docs.astral.sh/uv/) is a Python package and project manager that can make building and installing PyFlink from source much easier.
+    To verify that it is installed:
+
+    ```shell
+    $ uv --version
+    ```
+
 3. Build PyFlink with Cython extension support (optional)
 
     To build PyFlink with Cython extension support, you’ll need a C compiler. It's a little different on how to install the C compiler on different operating systems:
@@ -87,21 +96,49 @@ The `fast` and `skip-webui-build` profiles have a significant impact on the buil
 
     * **Mac OS X** To install GCC on Mac OS X, you need to download and install "[Command Line Tools for Xcode](https://developer.apple.com/downloads/index.action)", which is available in Apple’s developer page.
 
-    You also need to install the dependencies with following command:
+    You also need to install the dependencies with following commands. Either with `uv`:
+
+    ```shell
+    $ uv pip install --group flink-python/pyproject.toml:dev
+    ```
+
+    or with `python` and `pip`:
 
     ```shell
     $ python -m pip install --group flink-python/pyproject.toml:dev
     ```
 
 #### Installation
 
-Then go to the root directory of flink source code and run this command to build the sdist package and wheel package of `apache-flink` and `apache-flink-libraries`:
+##### With `uv`
+
+To build and install the `apache-flink` and `apache-flink-libraries` packages with `uv`, go to the root directory of flink source code and run the following:
+
+```bash
+cd flink-python; uv build; uv build apache-flink-libraries --sdist;
+```
+
+The sdist package of `apache-flink-libraries` will be found under `./flink-python/apache-flink-libraries/dist/`. It can be installed as follows:
+
+```bash
+uv pip install apache-flink-libraries/dist/*.tar.gz
+```
+
+The sdist and wheel packages of `apache-flink` will be found under `./flink-python/dist/`. Either of them could be used for installation, such as:
+
+```bash
+uv pip install dist/*.whl
+```
+
+##### With `python` and `pip`
+
+To build and install the `apache-flink` and `apache-flink-libraries` packages with `python` and `pip`, go to the root directory of flink source code and run the following:
 
 ```bash
 cd flink-python; python setup.py sdist bdist_wheel; cd apache-flink-libraries; python setup.py sdist; cd ..;
 ```
 
-The sdist package of `apache-flink-libraries` will be found under `./flink-python/apache-flink-libraries/dist/`. It could be installed as following:
+The sdist package of `apache-flink-libraries` will be found under `./flink-python/apache-flink-libraries/dist/`. It can be installed as follows:
 
 ```bash
 python -m pip install apache-flink-libraries/dist/*.tar.gz

docs/static/downloads/setup-pyflink-virtual-env.sh

@@ -20,7 +20,7 @@ echo "Detected OS: ${sys_os}"
 sys_machine=$(uname -m)
 echo "Detected machine: ${sys_machine}"
 
-wget "https://github.com/astral-sh/uv/releases/download/0.5.23/uv-installer.sh" -O "uv-installer.sh"
+wget "https://github.com/astral-sh/uv/releases/download/0.8.9/uv-installer.sh" -O "uv-installer.sh"
 chmod +x uv-installer.sh
 
 UV_UNMANAGED_INSTALL="uv-bin" ./uv-installer.sh
@@ -34,10 +34,10 @@ source venv/bin/activate ""
 # install PyFlink dependency
 if [[ $1 = "" ]]; then
     # install the latest version of pyflink
-    pip install apache-flink
+    uv pip install apache-flink
 else
     # install the specified version of pyflink
-    pip install "apache-flink==$1"
+    uv pip install "apache-flink==$1"
 fi
 
 # deactivate the python virtual environment

flink-end-to-end-tests/test-scripts/test_kubernetes_pyflink_application.sh

@@ -54,13 +54,9 @@ cd "${FLINK_PYTHON_DIR}"
 
 if [[ -d "dist" ]]; then rm -Rf dist; fi
 
-pushd apache-flink-libraries
+uv build apache-flink-libraries --sdist
 
-python setup.py sdist
-
-popd
-
-python setup.py sdist
+uv build
 
 cd dev
 

flink-end-to-end-tests/test-scripts/test_pyflink.sh

@@ -97,19 +97,7 @@ cd "${FLINK_PYTHON_DIR}"
 
 rm -rf dist
 
-uv pip install --group dev
-
-python setup.py sdist
-
-pushd apache-flink-libraries
-
-python setup.py sdist
-
-uv pip install dist/*
-
-popd
-
-uv pip install dist/*
+uv pip install --group dev .
 
 cd dev
 

flink-python/README.md

@@ -67,6 +67,5 @@ Then you can activate your environment and run tests, for example:
 
 ```shell
 source pyflink_39/bin/activate
-uv pip install --group dev
 ./dev/lint-python.sh
 ```

flink-python/apache-flink-libraries/setup.cfg renamed to flink-python/apache-flink-libraries/pyproject.toml

@@ -15,9 +15,30 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = [
+    "setuptools>=75.3",
+    "wheel",
+]
 
-[bdist_wheel]
-universal = 1
+[project]
+name = "apache-flink-libraries"
+description = "Apache Flink Libraries"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    {name = "Apache Software Foundation", email = "[email protected]"}
+]
+requires-python = ">=3.9"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dynamic = ["version", "dependencies"]
 
-[metadata]
-description-file = README.md
+[project.urls]
+homepage = "https://flink.apache.org"

flink-python/apache-flink-libraries/setup.py

@@ -215,27 +215,11 @@ def find_file_path(pattern):
         PACKAGE_DATA['pyflink.licenses'] = ['*']
 
     setup(
-        name='apache-flink-libraries',
         version=VERSION,
         packages=PACKAGES,
         include_package_data=True,
         package_dir=PACKAGE_DIR,
         package_data=PACKAGE_DATA,
-        url='https://flink.apache.org',
-        license='https://www.apache.org/licenses/LICENSE-2.0',
-        author='Apache Software Foundation',
-        author_email='[email protected]',
-        python_requires='>=3.9',
-        description='Apache Flink Libraries',
-        long_description=long_description,
-        long_description_content_type='text/markdown',
-        classifiers=[
-            'Development Status :: 5 - Production/Stable',
-            'License :: OSI Approved :: Apache Software License',
-            'Programming Language :: Python :: 3.9',
-            'Programming Language :: Python :: 3.10',
-            'Programming Language :: Python :: 3.11',
-            'Programming Language :: Python :: 3.12'],
     )
 finally:
     if in_flink_source:

flink-python/dev/build-wheels.sh

@@ -15,41 +15,35 @@
 # limitations under the License.
 set -e -x
 
-## 1. install python env
-dev/lint-python.sh -s py_env
-
-PY_ENV_DIR=`pwd`/dev/.uv/envs
-py_env=("3.9" "3.10" "3.11" "3.12")
-## 2. install dependency
-for ((i=0;i<${#py_env[@]};i++)) do
-    source `pwd`/dev/.uv/envs/${py_env[i]}/bin/activate
-    echo "Installing dependencies for environment: ${py_env[i]}"
-    uv pip install --group dev
-    deactivate
-done
+## 1. Setup virtual environment with uv
+dev/lint-python.sh -s basic
+
+python_versions=("3.9" "3.10" "3.11" "3.12")
+
+if [[ "$(uname)" != "Darwin" ]]; then
+    # force the linker to use the older glibc version in Linux
+    export CFLAGS="-I. -include dev/glibc_version_fix.h"
+fi
+
+source "$(pwd)"/dev/.uv/bin/activate
 
-## 3. build wheels
-for ((i=0;i<${#py_env[@]};i++)) do
-    echo "Building wheel for environment: ${py_env[i]}"
-    if [[ "$(uname)" != "Darwin" ]]; then
-        # force the linker to use the older glibc version in Linux
-        export CFLAGS="-I. -include dev/glibc_version_fix.h"
-    fi
-    ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
+## 2. Build wheels for each Python version
+for ((i=0;i<${#python_versions[@]};i++)) do
+    echo "Building wheel for Python: ${python_versions[i]}"
+    uv build --wheel --python "${python_versions[i]}"
 done
 
-## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux
+## 3. Convert linux_x86_64 wheel to manylinux1 wheel in Linux
 if [[ "$(uname)" != "Darwin" ]]; then
     echo "Converting linux_x86_64 wheel to manylinux1"
-    source `pwd`/dev/.uv/bin/activate
-    # 4.1 install patchelf and auditwheel
-    uv pip install patchelf==0.17.2.1 auditwheel==3.2.0
-    # 4.2 convert Linux wheel
     for wheel_file in dist/*.whl; do
-        auditwheel repair ${wheel_file} -w dist
-        rm -f ${wheel_file}
+        uv run --group auditwheel auditwheel repair "${wheel_file}" -w dist
+        rm -f "${wheel_file}"
     done
-    deactivate
 fi
-## see the result
+
+deactivate
+
+## 4. Output the result
+echo "Build finished - created the following wheels:"
 ls -al dist/