From a5dc20c558948db900dd0964ac23dcd2787d946c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:13:46 +0900 Subject: [PATCH] refactor(pypi): move the platform config to MODULE.bazel This splits the configuration that we have for the `rules_python` and the defaults that we set for our users from the actual unit tests where we check that the extension is working correctly. With this we will be able to dog food the API and point users into the `MODULE.bazel` as the example snippet. Work towards #2949 Work towards #2747 --- MODULE.bazel | 56 ++++++++++ python/private/pypi/extension.bzl | 60 +---------- tests/pypi/extension/extension_tests.bzl | 132 ++++++++++------------- 3 files changed, 115 insertions(+), 133 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b1d8711815..a9b51951b2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -60,6 +60,62 @@ register_toolchains("@pythons_hub//:all") # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. pip = use_extension("//python/extensions:pip.bzl", "pip") + +# NOTE @aignas 2025-07-06: we define these platforms to keep backwards compatibility with the +# current `experimental_index_url` implementation. Whilst we stabilize the API this list may be +# updated with a mention in the CHANGELOG. +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:linux", + ], + env = {"platform_version": "0"}, + os_name = "linux", + platform = "linux_{}".format(cpu), + ) + for cpu in [ + "x86_64", + "aarch64", + # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the + # `pip.default` extension. i.e. drop the below values - users will have to + # define themselves if they need them. + "arm", + "ppc", + "s390x", + ] +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:osx", + ], + # We choose the oldest non-EOL version at the time when we release `rules_python`. + # See https://endoflife.date/macos + env = {"platform_version": "14.0"}, + os_name = "osx", + platform = "osx_{}".format(cpu), + ) + for cpu in [ + "aarch64", + "x86_64", + ] +] + +pip.default( + arch_name = "x86_64", + config_settings = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_x86_64", +) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index a0095f8f15..505458008f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -393,64 +393,6 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { else: config["platforms"].pop(platform) -def _create_config(defaults): - if defaults["platforms"]: - return struct(**defaults) - - # NOTE: We have this so that it is easier to maintain unit tests assuming certain - # defaults - for cpu in [ - "x86_64", - "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", - ]: - _configure( - defaults, - arch_name = cpu, - os_name = "linux", - platform = "linux_{}".format(cpu), - config_settings = [ - "@platforms//os:linux", - "@platforms//cpu:{}".format(cpu), - ], - env = {"platform_version": "0"}, - ) - for cpu in [ - "aarch64", - "x86_64", - ]: - _configure( - defaults, - arch_name = cpu, - # We choose the oldest non-EOL version at the time when we release `rules_python`. - # See https://endoflife.date/macos - os_name = "osx", - platform = "osx_{}".format(cpu), - config_settings = [ - "@platforms//os:osx", - "@platforms//cpu:{}".format(cpu), - ], - env = {"platform_version": "14.0"}, - ) - - _configure( - defaults, - arch_name = "x86_64", - os_name = "windows", - platform = "windows_x86_64", - config_settings = [ - "@platforms//os:windows", - "@platforms//cpu:x86_64", - ], - env = {"platform_version": "0"}, - ) - return struct(**defaults) - def parse_modules( module_ctx, _fail = fail, @@ -527,7 +469,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # for what. We could also model the `cp313t` freethreaded as separate platforms. ) - config = _create_config(defaults) + config = struct(**defaults) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 146293ee8d..cf96d4005a 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -56,7 +56,23 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo parse = parse, override = override, whl_mods = whl_mods, - default = default, + default = default or [ + _default( + platform = "{}_{}".format(os, cpu), + os_name = os, + arch_name = cpu, + config_settings = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + ) + for os, cpu in [ + ("linux", "x86_64"), + ("linux", "aarch64"), + ("osx", "aarch64"), + ("windows", "aarch64"), + ] + ], ), is_root = is_root, ) @@ -235,19 +251,18 @@ def _test_simple_multiple_requirements(env): pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { "simple": { - "pypi_315_simple_osx_aarch64_osx_x86_64": [ + "pypi_315_simple_osx_aarch64": [ whl_config_setting( target_platforms = [ "cp315_osx_aarch64", - "cp315_osx_x86_64", ], version = "3.15", ), ], - "pypi_315_simple_windows_x86_64": [ + "pypi_315_simple_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_x86_64", + "cp315_windows_aarch64", ], version = "3.15", ), @@ -255,12 +270,12 @@ def _test_simple_multiple_requirements(env): }, }}) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_osx_aarch64_osx_x86_64": { + "pypi_315_simple_osx_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.2 --hash=sha256:deadb00f", }, - "pypi_315_simple_windows_x86_64": { + "pypi_315_simple_windows_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef", @@ -310,24 +325,20 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { "torch": { - "pypi_315_torch_linux_aarch64_linux_arm_linux_ppc_linux_s390x_osx_aarch64": [ + "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": [ whl_config_setting( target_platforms = [ "cp315_linux_aarch64", - "cp315_linux_arm", - "cp315_linux_ppc", - "cp315_linux_s390x", "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), ], - "pypi_315_torch_linux_x86_64_osx_x86_64_windows_x86_64": [ + "pypi_315_torch_linux_x86_64": [ whl_config_setting( target_platforms = [ "cp315_linux_x86_64", - "cp315_osx_x86_64", - "cp315_windows_x86_64", ], version = "3.15", ), @@ -335,12 +346,12 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ }, }}) pypi.whl_libraries().contains_exactly({ - "pypi_315_torch_linux_aarch64_linux_arm_linux_ppc_linux_s390x_osx_aarch64": { + "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, - "pypi_315_torch_linux_x86_64_osx_x86_64_windows_x86_64": { + "pypi_315_torch_linux_x86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -385,6 +396,23 @@ def _test_torch_experimental_index_url(env): module_ctx = _mock_mctx( _mod( name = "rules_python", + default = [ + _default( + platform = "{}_{}".format(os, cpu), + os_name = os, + arch_name = cpu, + config_settings = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + ) + for os, cpu in [ + ("linux", "aarch64"), + ("linux", "x86_64"), + ("osx", "aarch64"), + ("windows", "x86_64"), + ] + ], parse = [ _parse( hub_name = "pypi", @@ -444,34 +472,26 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.hub_whl_map().contains_exactly({"pypi": { "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", - target_platforms = None, version = "3.12", ), ], @@ -482,7 +502,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_x86_64", - "osx_x86_64", "windows_x86_64", ], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", @@ -495,9 +514,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "osx_aarch64", ], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", @@ -510,7 +526,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_x86_64", - "osx_x86_64", "windows_x86_64", ], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", @@ -523,9 +538,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "osx_aarch64", ], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", @@ -817,13 +829,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", @@ -836,13 +844,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -866,13 +870,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -884,13 +884,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "simple-0.0.1.tar.gz", @@ -903,13 +899,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -921,13 +913,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -995,26 +983,22 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_linux_aarch64_linux_arm_linux_ppc_linux_s390x_linux_x86_64": [ + "pypi_315_optimum_linux_aarch64_linux_x86_64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_linux_aarch64", - "cp315_linux_arm", - "cp315_linux_ppc", - "cp315_linux_s390x", "cp315_linux_x86_64", ], config_setting = None, filename = None, ), ], - "pypi_315_optimum_osx_aarch64_osx_x86_64": [ + "pypi_315_optimum_osx_aarch64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_osx_aarch64", - "cp315_osx_x86_64", ], config_setting = None, filename = None, @@ -1025,12 +1009,12 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_aarch64_linux_arm_linux_ppc_linux_s390x_linux_x86_64": { + "pypi_315_optimum_linux_aarch64_linux_x86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, - "pypi_315_optimum_osx_aarch64_osx_x86_64": { + "pypi_315_optimum_osx_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1",