From 2e51b61e4c8c009c29ea82b0934ba5d9d6e4f943 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:12:16 +0300 Subject: [PATCH 01/75] refactor: define internal vars for each tag class --- python/private/python.bzl | 70 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 6a265d1395..8a6285e1db 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -297,18 +297,14 @@ def _get_bazel_version_specific_kwargs(): return kwargs -python = module_extension( - doc = """Bzlmod extension that is used to register Python toolchains. -""", - implementation = _python_impl, - tag_classes = { - "rules_python_private_testing": tag_class( - attrs = { - "register_all_versions": attr.bool(default = False), - }, - ), - "toolchain": tag_class( - doc = """Tag class used to register Python toolchains. +_rules_python_private_testing = tag_class( + attrs = { + "register_all_versions": attr.bool(default = False), + }, +) + +_toolchain = tag_class( + doc = """Tag class used to register Python toolchains. Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. @@ -338,14 +334,14 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are `major` and `minor` are the Python version from the `python_version` attribute. """, - attrs = { - "configure_coverage_tool": attr.bool( - mandatory = False, - doc = "Whether or not to configure the default coverage tool for the toolchains.", - ), - "ignore_root_user_error": attr.bool( - default = False, - doc = """\ + attrs = { + "configure_coverage_tool": attr.bool( + mandatory = False, + doc = "Whether or not to configure the default coverage tool for the toolchains.", + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ If False, the Python runtime installation will be made read only. This improves the ability for Bazel to cache it, but prevents the interpreter from creating pyc files for the standard library dynamically at runtime as they are loaded. @@ -355,21 +351,29 @@ interpreter to create pyc files for the standard library, but, because they are created as needed, it adversely affects Bazel's ability to cache the runtime and can result in spurious build failures. """, - mandatory = False, - ), - "is_default": attr.bool( - mandatory = False, - doc = "Whether the toolchain is the default version", - ), - "python_version": attr.string( - mandatory = True, - doc = "The Python version, in `major.minor` format, e.g " + - "'3.12', to create a toolchain for. Patch level " + - "granularity (e.g. '3.12.1') is not supported.", - ), - }, + mandatory = False, + ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), + "python_version": attr.string( + mandatory = True, + doc = "The Python version, in `major.minor` format, e.g " + + "'3.12', to create a toolchain for. Patch level " + + "granularity (e.g. '3.12.1') is not supported.", ), }, +) + +python = module_extension( + doc = """Bzlmod extension that is used to register Python toolchains. +""", + implementation = _python_impl, + tag_classes = { + "rules_python_private_testing": _rules_python_private_testing, + "toolchain": _toolchain, + }, **_get_bazel_version_specific_kwargs() ) From e7e2cbeffea2d8cb41d05d5f1a61d4a6872e9382 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:32:25 +0300 Subject: [PATCH 02/75] refactor: rename some classes and thread the available tool versions --- python/private/BUILD.bazel | 2 +- python/private/python.bzl | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 146e934654..eec538e72c 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -131,11 +131,11 @@ bzl_library( name = "python_bzl", srcs = ["python.bzl"], deps = [ + ":python_repositories_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", ":toolchains_repo_bzl", ":util_bzl", - "//python:repositories_bzl", "@bazel_features//:features", ], ) diff --git a/python/private/python.bzl b/python/private/python.bzl index 8a6285e1db..cd42e14670 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,10 +15,10 @@ "Python toolchain module extensions for use with bzlmod" load("@bazel_features//:features.bzl", "bazel_features") -load("//python:repositories.bzl", "python_register_toolchains") load("//python:versions.bzl", "TOOL_VERSIONS") -load("//python/private:repo_utils.bzl", "repo_utils") +load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") +load(":repo_utils.bzl", "repo_utils") load(":text_util.bzl", "render") load(":toolchains_repo.bzl", "multi_toolchain_aliases") load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") @@ -28,14 +28,14 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def _python_register_toolchains(name, toolchain_attr, module, ignore_root_user_error): +def _python_register_toolchains(name, toolchain_attr, module, **kwargs): """Calls python_register_toolchains and returns a struct used to collect the toolchains. """ python_register_toolchains( name = name, python_version = toolchain_attr.python_version, register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = ignore_root_user_error, + **kwargs ) return struct( python_version = toolchain_attr.python_version, @@ -76,9 +76,9 @@ def _python_impl(module_ctx): for mod in module_ctx.modules: module_toolchain_versions = [] - toolchain_attr_structs = _create_toolchain_attr_structs(mod) + python_tools = _process_tag_classes(mod) - for toolchain_attr in toolchain_attr_structs: + for toolchain_attr in python_tools.registrations: toolchain_version = toolchain_attr.python_version toolchain_name = "python_" + toolchain_version.replace(".", "_") @@ -144,6 +144,10 @@ def _python_impl(module_ctx): toolchain_attr, module = mod, ignore_root_user_error = ignore_root_user_error, + # TODO @aignas 2024-08-08: allow to modify these values via the bzlmod extension + # distutils_content = None, + # register_toolchains = True, + tool_versions = python_tools.available_versions, ) global_toolchain_versions[toolchain_version] = toolchain_info if debug_info: @@ -252,9 +256,11 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _create_toolchain_attr_structs(mod): +def _process_tag_classes(mod): arg_structs = [] seen_versions = {} + available_versions = TOOL_VERSIONS + for tag in mod.tags.toolchain: arg_structs.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) seen_versions[tag.python_version] = True @@ -268,10 +274,14 @@ def _create_toolchain_attr_structs(mod): if register_all: arg_structs.extend([ _create_toolchain_attrs_struct(python_version = v) - for v in TOOL_VERSIONS.keys() + for v in available_versions.keys() if v not in seen_versions ]) - return arg_structs + + return struct( + registrations = arg_structs, + available_versions = available_versions, + ) def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): if tag and python_version: From 330008ad2dd424f1984fedbcd2e426154089a4f2 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:00:36 +0300 Subject: [PATCH 03/75] spike the override tag classes --- python/private/python.bzl | 61 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index cd42e14670..6a6afbf6db 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,7 +15,7 @@ "Python toolchain module extensions for use with bzlmod" load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "TOOL_VERSIONS") +load("//python:versions.bzl", "PLATFORMS", "TOOL_VERSIONS") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") @@ -266,11 +266,35 @@ def _process_tag_classes(mod): seen_versions[tag.python_version] = True if mod.is_root: + for tag in mod.tags.version_override: + sha256 = {} + for p, sha in tag.sha256.items(): + if p not in PLATFORMS: + fail("The platform must be one of {allowed} but got '{got}'".format( + allowed = sorted(PLATFORMS), + got = p, + )) + + sha256[p] = sha + + available_versions[tag.version] = { + "sha256": sha256, + "strip_prefix": tag.strip_prefix, + "url": tag.url, + } + + for tag in mod.tags.override: + available_versions = { + v: available_versions[v] + for v in tag.available_python_versions + } + register_all = False for tag in mod.tags.rules_python_private_testing: if tag.register_all_versions: register_all = True break + if register_all: arg_structs.extend([ _create_toolchain_attrs_struct(python_version = v) @@ -376,13 +400,48 @@ can result in spurious build failures. }, ) +_override = tag_class( + doc = """Tag class used to override defaults and behaviour of the module extension.""", + attrs = { + "available_python_versions": attr.string_list( + mandatory = True, + doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", + ), + }, +) + +_version_override = tag_class( + doc = """Tag class used to override single python version settings.""", + attrs = { + "sha256s": attr.string_dict( + mandatory = True, + doc = "The python platform to sha256 dict. The platform key must be present in the PLATFORMS dict.", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "url": attr.string( + mandatory = True, + doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of {python_version}, {platform} and {build} will be interpolated based on the contents in the override and the PLATFORMS dict.", + ), + "version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + }, +) + python = module_extension( doc = """Bzlmod extension that is used to register Python toolchains. """, implementation = _python_impl, tag_classes = { + "override": _override, "rules_python_private_testing": _rules_python_private_testing, "toolchain": _toolchain, + "version_override": _version_override, }, **_get_bazel_version_specific_kwargs() ) From 2d8642c9ca45b2dd3da7dc9b7727aa70cddc4703 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:03:47 +0300 Subject: [PATCH 04/75] add base_url override as well --- python/private/python.bzl | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 6a6afbf6db..4fcf066e3a 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,7 +15,7 @@ "Python toolchain module extensions for use with bzlmod" load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "PLATFORMS", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") @@ -143,11 +143,12 @@ def _python_impl(module_ctx): toolchain_name, toolchain_attr, module = mod, + # Extra kwargs to the underlying python_register_toolchains methods + base_url = python_tools.base_url, ignore_root_user_error = ignore_root_user_error, + tool_versions = python_tools.available_versions, # TODO @aignas 2024-08-08: allow to modify these values via the bzlmod extension # distutils_content = None, - # register_toolchains = True, - tool_versions = python_tools.available_versions, ) global_toolchain_versions[toolchain_version] = toolchain_info if debug_info: @@ -260,6 +261,7 @@ def _process_tag_classes(mod): arg_structs = [] seen_versions = {} available_versions = TOOL_VERSIONS + base_url = DEFAULT_RELEASE_BASE_URL for tag in mod.tags.toolchain: arg_structs.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) @@ -284,10 +286,12 @@ def _process_tag_classes(mod): } for tag in mod.tags.override: + base_url = tag.base_url available_versions = { v: available_versions[v] for v in tag.available_python_versions } + break register_all = False for tag in mod.tags.rules_python_private_testing: @@ -303,8 +307,9 @@ def _process_tag_classes(mod): ]) return struct( - registrations = arg_structs, available_versions = available_versions, + base_url = base_url, + registrations = arg_structs, ) def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): @@ -407,6 +412,11 @@ _override = tag_class( mandatory = True, doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", ), + "base_url": attr.string_list( + mandatory = False, + doc = "The base URL to be used when downloading toolchains.", + default = DEFAULT_RELEASE_BASE_URL, + ), }, ) From 12d8ec64ceaf1a51651b827d719e6b7642f7c003 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 9 Aug 2024 10:14:53 +0300 Subject: [PATCH 05/75] refactor: move the register_all_versions to python.override --- MODULE.bazel | 2 +- python/private/python.bzl | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9ac3e7a04c..100eae8569 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -81,7 +81,7 @@ dev_python = use_extension( "python", dev_dependency = True, ) -dev_python.rules_python_private_testing( +dev_python.override( register_all_versions = True, ) diff --git a/python/private/python.bzl b/python/private/python.bzl index 4fcf066e3a..b91c3d7dbe 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -285,19 +285,20 @@ def _process_tag_classes(mod): "url": tag.url, } + register_all = False for tag in mod.tags.override: base_url = tag.base_url - available_versions = { - v: available_versions[v] - for v in tag.available_python_versions - } - break - - register_all = False - for tag in mod.tags.rules_python_private_testing: - if tag.register_all_versions: + if tag.available_python_versions: + available_versions = { + v: available_versions[v] + for v in tag.available_python_versions + } + + if tag.register_all_versions and mod.name != "rules_python": + fail("This override can only be used by 'rules_python'") + elif tag.register_all_versions: register_all = True - break + break if register_all: arg_structs.extend([ @@ -409,14 +410,17 @@ _override = tag_class( doc = """Tag class used to override defaults and behaviour of the module extension.""", attrs = { "available_python_versions": attr.string_list( - mandatory = True, + mandatory = False, doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", ), - "base_url": attr.string_list( + "base_url": attr.string( mandatory = False, doc = "The base URL to be used when downloading toolchains.", default = DEFAULT_RELEASE_BASE_URL, ), + + # Internal attributes that are only usable from `rules_python` + "register_all_versions": attr.bool(default = False), }, ) From 5ec8acd52fcc3980656c76251bcdddf9b4ed153d Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 13 Aug 2024 07:53:12 +0300 Subject: [PATCH 06/75] wip --- examples/bzlmod/MODULE.bazel | 11 +++++++++++ python/private/python.bzl | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index b7b46b7dba..52b7cdb746 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -37,6 +37,17 @@ python.toolchain( python_version = "3.10", ) +# One can override the actual toolchain versions that are available +python.override( + available_python_versions = [ + "3.10.9", + "3.9.19", + ], +) +python.version_override( + version = "3.10.2", +) + # You only need to load this repositories if you are using multiple Python versions. # See the tests folder for various examples on using multiple Python versions. # The names "python_3_9" and "python_3_10" are autmatically created by the repo diff --git a/python/private/python.bzl b/python/private/python.bzl index b91c3d7dbe..e6620f7add 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -289,8 +289,12 @@ def _process_tag_classes(mod): for tag in mod.tags.override: base_url = tag.base_url if tag.available_python_versions: + all_known_versions = sorted(available_versions) available_versions = { - v: available_versions[v] + v: available_versions.get(v, fail("unknown version {}, known versions: {}".format( + v, + all_known_versions, + ))) for v in tag.available_python_versions } @@ -298,6 +302,7 @@ def _process_tag_classes(mod): fail("This override can only be used by 'rules_python'") elif tag.register_all_versions: register_all = True + break if register_all: From 35492e4238f83ae24d787a0bf552e43d4e995393 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 23 Aug 2024 12:56:31 +0300 Subject: [PATCH 07/75] Add an example on how to override the URLs --- examples/bzlmod/MODULE.bazel | 11 +++++++++++ python/private/python.bzl | 11 ++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 52b7cdb746..f9a1c2dbb1 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -42,9 +42,20 @@ python.override( available_python_versions = [ "3.10.9", "3.9.19", + # Keep the following versions which are present in MINOR_MAPPING, but + # are not present above + "3.10.14", ], ) python.version_override( + sha256s = { + "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", + "x86_64-apple-darwin": "8146ad4390710ec69b316a5649912df0247d35f4a42e2aa9615bffd87b3e235a", + "x86_64-pc-windows-msvc": "a1d9a594cd3103baa24937ad9150c1a389544b4350e859200b3e5c036ac352bd", + "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", + }, + url = "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", version = "3.10.2", ) diff --git a/python/private/python.bzl b/python/private/python.bzl index e6620f7add..7f3a28f396 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -260,7 +260,8 @@ def _fail_multiple_default_toolchains(first, second): def _process_tag_classes(mod): arg_structs = [] seen_versions = {} - available_versions = TOOL_VERSIONS + available_versions = {} + available_versions.update(TOOL_VERSIONS) base_url = DEFAULT_RELEASE_BASE_URL for tag in mod.tags.toolchain: @@ -270,7 +271,7 @@ def _process_tag_classes(mod): if mod.is_root: for tag in mod.tags.version_override: sha256 = {} - for p, sha in tag.sha256.items(): + for p, sha in tag.sha256s.items(): if p not in PLATFORMS: fail("The platform must be one of {allowed} but got '{got}'".format( allowed = sorted(PLATFORMS), @@ -291,10 +292,10 @@ def _process_tag_classes(mod): if tag.available_python_versions: all_known_versions = sorted(available_versions) available_versions = { - v: available_versions.get(v, fail("unknown version {}, known versions: {}".format( + v: available_versions[v] if v in available_versions else fail("unknown version '{}', known versions are: {}".format( v, all_known_versions, - ))) + )) for v in tag.available_python_versions } @@ -433,7 +434,7 @@ _version_override = tag_class( doc = """Tag class used to override single python version settings.""", attrs = { "sha256s": attr.string_dict( - mandatory = True, + mandatory = False, doc = "The python platform to sha256 dict. The platform key must be present in the PLATFORMS dict.", ), "strip_prefix": attr.string( From 4b5da0e506f1b3ca493a7f77231557c934c1a6fd Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:12:14 +0300 Subject: [PATCH 08/75] add a doc --- python/private/python.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 7f3a28f396..96937422cc 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -426,7 +426,7 @@ _override = tag_class( ), # Internal attributes that are only usable from `rules_python` - "register_all_versions": attr.bool(default = False), + "register_all_versions": attr.bool(default = False, doc = "rules_python internal use only"), }, ) From c765d11d24786a563c5a1233289612f14ce42cbf Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:14:45 +0300 Subject: [PATCH 09/75] minor refactoring in processing the tag classes --- python/private/python.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 96937422cc..c3b99ad7ad 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -257,15 +257,15 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _process_tag_classes(mod): - arg_structs = [] +def _process_tag_classes(mod, fail = fail): + registrations = [] seen_versions = {} available_versions = {} available_versions.update(TOOL_VERSIONS) base_url = DEFAULT_RELEASE_BASE_URL for tag in mod.tags.toolchain: - arg_structs.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) + registrations.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) seen_versions[tag.python_version] = True if mod.is_root: @@ -307,7 +307,7 @@ def _process_tag_classes(mod): break if register_all: - arg_structs.extend([ + registrations.extend([ _create_toolchain_attrs_struct(python_version = v) for v in available_versions.keys() if v not in seen_versions @@ -316,7 +316,7 @@ def _process_tag_classes(mod): return struct( available_versions = available_versions, base_url = base_url, - registrations = arg_structs, + registrations = registrations, ) def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): From 46af7f92cdfff2c10307dd277e8c30873cc7dce3 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:53:35 +0300 Subject: [PATCH 10/75] version_override -> single_version_override --- examples/bzlmod/MODULE.bazel | 2 +- python/private/python.bzl | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index f9a1c2dbb1..d0062dce39 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -47,7 +47,7 @@ python.override( "3.10.14", ], ) -python.version_override( +python.single_version_override( sha256s = { "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", diff --git a/python/private/python.bzl b/python/private/python.bzl index c3b99ad7ad..6ef5448b05 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -269,7 +269,7 @@ def _process_tag_classes(mod, fail = fail): seen_versions[tag.python_version] = True if mod.is_root: - for tag in mod.tags.version_override: + for tag in mod.tags.single_version_override: sha256 = {} for p, sha in tag.sha256s.items(): if p not in PLATFORMS: @@ -430,8 +430,8 @@ _override = tag_class( }, ) -_version_override = tag_class( - doc = """Tag class used to override single python version settings.""", +_single_version_override = tag_class( + doc = """Override single python version settings.""", attrs = { "sha256s": attr.string_dict( mandatory = False, @@ -460,8 +460,8 @@ python = module_extension( tag_classes = { "override": _override, "rules_python_private_testing": _rules_python_private_testing, + "single_version_override": _single_version_override, "toolchain": _toolchain, - "version_override": _version_override, }, **_get_bazel_version_specific_kwargs() ) From 0a88a6eeb086986a21a6463bcc035734a14ff36c Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:15:32 +0300 Subject: [PATCH 11/75] improve docs - add repositories.bzl docs and improve docs for bzlmod --- CHANGELOG.md | 7 ++- docs/BUILD.bazel | 2 + python/private/BUILD.bazel | 1 + python/private/common/py_runtime_rule.bzl | 8 +-- python/private/python.bzl | 40 +++++++++----- python/private/python_repositories.bzl | 65 ++++++++++++----------- python/repositories.bzl | 2 - python/versions.bzl | 7 +-- 8 files changed, 80 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dfbe481b..976386b0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,12 @@ A brief description of the categories of changes: [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x ### Changed -* (gazelle): Update error messages when unable to resolve a dependency to be more human-friendly. +* (gazelle): Update error messages when unable to resolve a dependency to be + more human-friendly. +* (toolchain): The toolchain patches now expose the `patch_strip` attribute + that one should use when patching toolchains. Please set it if you are + patching python interpreter. In the next release the default will be set to + `0` which better reflects the defaults used in public `bazel` APIs. ### Fixed * (whl_library): Remove `--no-index` and add `--no-build-isolation` to the diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index aa1345aa08..a45d7fcc1a 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -94,6 +94,8 @@ sphinx_stardocs( "//python/private/common:py_library_rule_bazel_bzl", "//python/private/common:py_runtime_rule_bzl", "//python/private/common:py_test_rule_bazel_bzl", + # FIXME @aignas 2024-08-26: should we have this here? + "//python:repositories_bzl", ] + ([ # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension "//python/extensions:python_bzl", diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index eec538e72c..a8dc9e70ce 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -298,6 +298,7 @@ bzl_library( srcs = ["toolchains_repo.bzl"], deps = [ ":repo_utils_bzl", + ":text_util_bzl", "//python:versions_bzl", ], ) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index e0b5fb2313..1cc8361e54 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -204,8 +204,8 @@ See @bazel_tools//tools/python:python_bootstrap_template.txt for more variables. "coverage_tool": attr.label( allow_files = False, doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. +This is a target to use for collecting code coverage information from +{rule}`py_binary` and {rule}`py_test` targets. If set, the target must either produce a single file or be an executable target. The path to the single file, or the executable if the target is executable, @@ -214,7 +214,7 @@ runfiles will be added to the runfiles when coverage is enabled. The entry point for the tool must be loadable by a Python interpreter (e.g. a `.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including +of [`coverage.py`](https://coverage.readthedocs.io), at least including the `run` and `lcov` subcommands. """, ), @@ -306,7 +306,7 @@ The template to use when two stage bootstrapping is enabled default = DEFAULT_STUB_SHEBANG, doc = """ "Shebang" expression prepended to the bootstrapping Python stub script -used when executing `py_binary` targets. +used when executing {rule}`py_binary` targets. See https://github.com/bazelbuild/bazel/issues/8685 for motivation. diff --git a/python/private/python.bzl b/python/private/python.bzl index 6ef5448b05..01218953f3 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -147,7 +147,7 @@ def _python_impl(module_ctx): base_url = python_tools.base_url, ignore_root_user_error = ignore_root_user_error, tool_versions = python_tools.available_versions, - # TODO @aignas 2024-08-08: allow to modify these values via the bzlmod extension + # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension # distutils_content = None, ) global_toolchain_versions[toolchain_version] = toolchain_info @@ -280,11 +280,16 @@ def _process_tag_classes(mod, fail = fail): sha256[p] = sha - available_versions[tag.version] = { + version = { + "patch_strip": tag.patch_strip, + # Use a list if there is only a single item the key is `*`, + # otherwise pass the dict down to the toolchain rule. + "patches": tag.patches.get("*", tag.patches) if len(tag.patches) == 1 else tag.patches, "sha256": sha256, "strip_prefix": tag.strip_prefix, "url": tag.url, } + available_versions[tag.version] = version register_all = False for tag in mod.tags.override: @@ -343,12 +348,6 @@ def _get_bazel_version_specific_kwargs(): return kwargs -_rules_python_private_testing = tag_class( - attrs = { - "register_all_versions": attr.bool(default = False), - }, -) - _toolchain = tag_class( doc = """Tag class used to register Python toolchains. Use this tag class to register one or more Python toolchains. This class @@ -413,7 +412,11 @@ can result in spurious build failures. ) _override = tag_class( - doc = """Tag class used to override defaults and behaviour of the module extension.""", + doc = """Tag class used to override defaults and behaviour of the module extension. + +:::{versionadded} 0.36.0 +::: +""", attrs = { "available_python_versions": attr.string_list( mandatory = False, @@ -426,13 +429,27 @@ _override = tag_class( ), # Internal attributes that are only usable from `rules_python` - "register_all_versions": attr.bool(default = False, doc = "rules_python internal use only"), + "register_all_versions": attr.bool(default = False, doc = "`rules_python` **internal** use only!"), }, ) _single_version_override = tag_class( - doc = """Override single python version settings.""", + doc = """Override single python version settings. + +:::{versionadded} 0.36.0 +::: +""", attrs = { + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + # TODO @aignas 2024-08-26: switch to 0 when 0.36.0 is released + default = 1, + ), + "patches": attr.string_list_dict( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for this module as values with keys as values from the PLATFORMS dict. The patch files must exist in the source tree of the top level project. They are applied in the list order. If patches have a single key '*', then the patches will be applied to all available interpreters for that version.", + ), "sha256s": attr.string_dict( mandatory = False, doc = "The python platform to sha256 dict. The platform key must be present in the PLATFORMS dict.", @@ -459,7 +476,6 @@ python = module_extension( implementation = _python_impl, tag_classes = { "override": _override, - "rules_python_private_testing": _rules_python_private_testing, "single_version_override": _single_version_override, "toolchain": _toolchain, }, diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index 25d8a96b79..3710cf945a 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -47,8 +47,10 @@ def http_archive(**kwargs): def py_repositories(): """Runtime dependencies that users must install. - This function should be loaded and called in the user's WORKSPACE. - With bzlmod enabled, this function is not needed since MODULE.bazel handles transitive deps. + This function should be loaded and called in the user's `WORKSPACE`. + + With `bzlmod` enabled, this function is not needed since `MODULE.bazel` + handles transitive deps. """ maybe( internal_config_repo, @@ -178,8 +180,7 @@ def _python_repository_impl(rctx): patches = rctx.attr.patches if patches: for patch in patches: - # Should take the strip as an attr, but this is fine for the moment - rctx.patch(patch, strip = 1) + rctx.patch(patch, strip = rctx.attr.patch_strip) # Write distutils.cfg to the Python installation. if "windows" in platform: @@ -450,6 +451,7 @@ py_exec_tools_toolchain( "ignore_root_user_error": rctx.attr.ignore_root_user_error, "name": rctx.attr.name, "netrc": rctx.attr.netrc, + "patch_strip": rctx.attr.patch_strip, "patches": rctx.attr.patches, "platform": platform, "python_version": python_version, @@ -473,27 +475,11 @@ python_repository = repository_rule( doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", ), "coverage_tool": attr.string( - # Mirrors the definition at - # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. - -If set, the target must either produce a single file or be an executable target. -The path to the single file, or the executable if the target is executable, -determines the entry point for the python coverage tool. The target and its -runfiles will be added to the runfiles when coverage is enabled. - -The entry point for the tool must be loadable by a Python interpreter (e.g. a -`.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including -the `run` and `lcov` subcommands. +This is a target to use for collecting code coverage information from {rule}`py_binary` +and {rule}`py_test` targets. -The target is accepted as a string by the python_repository and evaluated within -the context of the toolchain repository. - -For more information see the official bazel docs -(https://bazel.build/reference/be/python#py_runtime.coverage_tool). +For more information see {attr}`py_runtime.coverage_tool`. """, ), "distutils": attr.label( @@ -515,6 +501,12 @@ For more information see the official bazel docs "netrc": attr.string( doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ), + "patch_strip": attr.int( + doc = "Same as the --strip argument of Unix patch.", + # TODO @aignas 2024-08-26: switch to 0 when 0.36.0 is released + default = 1, + mandatory = False, + ), "patches": attr.label_list( doc = "A list of patch files to apply to the unpacked interpreter", mandatory = False, @@ -568,22 +560,28 @@ def python_register_toolchains( register_toolchains = True, register_coverage_tool = False, set_python_version_constraint = False, - tool_versions = TOOL_VERSIONS, + tool_versions = None, **kwargs): - """Convenience macro for users which does typical setup. + """Convenience macro for users which does typical setup in `WORKSPACE`. - Create a repository for each built-in platform like "python_linux_amd64" - this repository is lazily fetched when Python is needed for that platform. - Create a repository exposing toolchains for each platform like "python_platforms". - Register a toolchain pointing at each platform. + Users can avoid this macro and do these steps themselves, if they want more control. + + With `bzlmod` enabled, this function is not needed since `rules_python` is + handling everything. In order to override the default behaviour from the + root module one can see the docs for the {rule}`python` extension. + Args: name: base name for all created repos, like "python38". python_version: the Python version. - distutils: see the distutils attribute in the python_repository repository rule. - distutils_content: see the distutils_content attribute in the python_repository repository rule. + distutils: see the {attr}`python_repository.distutils`. + distutils_content: see the {attr}`python_repository.distutils_content`. register_toolchains: Whether or not to register the downloaded toolchains. register_coverage_tool: Whether or not to register the downloaded coverage tool to the toolchains. NOTE: Coverage support using the toolchain is only supported in Bazel 6 and higher. @@ -591,9 +589,11 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. - **kwargs: passed to each python_repositories call. + **kwargs: passed to each {rule}`python_repositories` call. """ + tool_versions = tool_versions or TOOL_VERSIONS + if BZLMOD_ENABLED: # you cannot used native.register_toolchains when using bzlmod. register_toolchains = False @@ -626,7 +626,7 @@ def python_register_toolchains( continue loaded_platforms.append(platform) - (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) + (release_filename, urls, strip_prefix, patches, patch_strip) = get_release_info(platform, python_version, base_url, tool_versions) # allow passing in a tool version coverage_tool = None @@ -651,6 +651,7 @@ def python_register_toolchains( platform = platform, ), sha256 = sha256, + patches = patch_strip, patches = patches, platform = platform, python_version = python_version, @@ -660,6 +661,10 @@ def python_register_toolchains( distutils_content = distutils_content, strip_prefix = strip_prefix, coverage_tool = coverage_tool, + # Will be one of + # * auth_patterns + # * ignore_root_user_error + # * netrc **kwargs ) if register_toolchains: @@ -713,7 +718,7 @@ def python_register_multi_toolchains( python_versions: the Python version. default_version: the default Python version. If not set, the first version in python_versions is used. - **kwargs: passed to each python_register_toolchains call. + **kwargs: passed to each {rule}`python_register_toolchains` call. """ if len(python_versions) == 0: fail("python_versions must not be empty") diff --git a/python/repositories.bzl b/python/repositories.bzl index cf8723405c..5d30125537 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -18,7 +18,6 @@ load( "//python/private:python_repositories.bzl", _STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER_FILENAME", - _http_archive = "http_archive", _is_standalone_interpreter = "is_standalone_interpreter", _py_repositories = "py_repositories", _python_register_multi_toolchains = "python_register_multi_toolchains", @@ -33,6 +32,5 @@ python_register_toolchains = _python_register_toolchains # These symbols are of questionable public visibility. They were probably # not intended to be actually public. STANDALONE_INTERPRETER_FILENAME = _STANDALONE_INTERPRETER_FILENAME -http_archive = _http_archive is_standalone_interpreter = _is_standalone_interpreter python_repository = _python_repository diff --git a/python/versions.bzl b/python/versions.bzl index 2cf9b39e96..424f0a86bd 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -636,7 +636,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U tool_versions: A dict listing the interpreter versions, their SHAs and URL Returns: - A tuple of (filename, url, and archive strip prefix) + A tuple of (filename, url, archive strip prefix, patches, patch_strip) """ url = tool_versions[python_version]["url"] @@ -669,12 +669,13 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U patches = tool_versions[python_version].get("patches", []) if type(patches) == type({}): - if platform in patches.keys(): + if platform in patches: patches = patches[platform] else: patches = [] + patch_strip = tool_versions[python_version].get("patch_strip", None) - return (release_filename, rendered_urls, strip_prefix, patches) + return (release_filename, rendered_urls, strip_prefix, patches, patch_strip) def print_toolchains_checksums(name): native.genrule( From b0e2f13d74edbae170cbc815ca572642d37cb509 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:19:02 +0300 Subject: [PATCH 12/75] clarify docs --- python/private/python.bzl | 3 +-- python/private/python_repositories.bzl | 11 ++++++++++- python/versions.bzl | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 01218953f3..d19467b2f6 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -443,8 +443,7 @@ _single_version_override = tag_class( "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", - # TODO @aignas 2024-08-26: switch to 0 when 0.36.0 is released - default = 1, + default = 0, ), "patches": attr.string_list_dict( mandatory = False, diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index 3710cf945a..d09994a42e 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -502,7 +502,16 @@ For more information see {attr}`py_runtime.coverage_tool`. doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ), "patch_strip": attr.int( - doc = "Same as the --strip argument of Unix patch.", + doc = """Same as the --strip argument of Unix patch. + +:::{note} +The default value of `1` is kept for backwards compatibility, it will be set to +`0` in a later release. +::: + +:::{versionadded} 0.36.0 +::: +""", # TODO @aignas 2024-08-26: switch to 0 when 0.36.0 is released default = 1, mandatory = False, diff --git a/python/versions.bzl b/python/versions.bzl index 424f0a86bd..5d799677e7 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -673,7 +673,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U patches = patches[platform] else: patches = [] - patch_strip = tool_versions[python_version].get("patch_strip", None) + patch_strip = tool_versions[python_version].get("patch_strip") return (release_filename, rendered_urls, strip_prefix, patches, patch_strip) From dd70a04f2bc9cab5b7cfa9f361cddfb14da2605f Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:19:06 +0300 Subject: [PATCH 13/75] cleanup the override impl --- examples/bzlmod/MODULE.bazel | 23 +++- examples/bzlmod/MODULE.bazel.lock | 4 +- python/private/python.bzl | 150 ++++++++++++++++++++----- python/private/python_repositories.bzl | 2 +- python/versions.bzl | 10 +- 5 files changed, 154 insertions(+), 35 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index d0062dce39..1526ecd24d 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -47,15 +47,34 @@ python.override( "3.10.14", ], ) + +# Or the sources that the toolchains come from for all platforms python.single_version_override( - sha256s = { + patch_strip = 1, + # The user can specify patches to be applied to all interpreters, they will + # be applied before the platform specific patches are applied. + patches = [], + sha256 = { "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", "x86_64-apple-darwin": "8146ad4390710ec69b316a5649912df0247d35f4a42e2aa9615bffd87b3e235a", "x86_64-pc-windows-msvc": "a1d9a594cd3103baa24937ad9150c1a389544b4350e859200b3e5c036ac352bd", "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", }, - url = "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], + version = "3.10.2", +) + +# Or a single platform. This can be used in combination with the +# `single_version_override` and `single_version_platform_override` will be +# applied after `single_version_override`. +# TODO @aignas 2024-08-27: implement that +python.single_version_platform_override( + patch_strip = 1, + patches = [], + platform = "aarch64-apple-darwin", + sha256 = "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], version = "3.10.2", ) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 8d02256e95..f573eaab15 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "9hiLCuWaaaU7Q+l2ONVr1A0NcG1JfSihv1UYeA1SpNY=", + "bzlTransitiveDigest": "4xxemDQ9FFPaWE1Gg3znSsYvxN1GQfil48+VzVFxhV0=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "VoK/T0JkBdcomCHnDIYkX+stkywdxrh1MVM16e8D4sE=", + "bzlTransitiveDigest": "P3LWVNuHO3sawrf2c6eTZ9dfg8nY3vajtjCV5JjwuLM=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/python.bzl b/python/private/python.bzl index d19467b2f6..666e407193 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -260,8 +260,22 @@ def _fail_multiple_default_toolchains(first, second): def _process_tag_classes(mod, fail = fail): registrations = [] seen_versions = {} - available_versions = {} - available_versions.update(TOOL_VERSIONS) + available_versions = { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() + } base_url = DEFAULT_RELEASE_BASE_URL for tag in mod.tags.toolchain: @@ -270,26 +284,54 @@ def _process_tag_classes(mod, fail = fail): if mod.is_root: for tag in mod.tags.single_version_override: - sha256 = {} - for p, sha in tag.sha256s.items(): - if p not in PLATFORMS: + for platform in tag.sha256 or []: + if platform not in PLATFORMS: fail("The platform must be one of {allowed} but got '{got}'".format( allowed = sorted(PLATFORMS), - got = p, + got = platform, )) - sha256[p] = sha - - version = { - "patch_strip": tag.patch_strip, - # Use a list if there is only a single item the key is `*`, - # otherwise pass the dict down to the toolchain rule. - "patches": tag.patches.get("*", tag.patches) if len(tag.patches) == 1 else tag.patches, - "sha256": sha256, - "strip_prefix": tag.strip_prefix, - "url": tag.url, + override = { + "patch_strip": { + platform: tag.patch_strip + for platform in tag.sha256 + }, + "patches": { + platform: list(tag.patches) + for platform in tag.sha256 + }, + "sha256": dict(tag.sha256), + "strip_prefix": { + platform: tag.strip_prefix + for platform in tag.sha256 + }, + "url": { + platform: list(tag.urls) + for platform in tag.sha256 + }, } - available_versions[tag.version] = version + available_versions[tag.version] = override + + for tag in mod.tags.single_version_platform_override: + if tag.version not in available_versions: + if not tag.urls or not tag.sha256 or not tag.strip_prefix: + fail("When introducing a new version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.version)) + available_versions[tag.version] = { + "sha256": {tag.platform: tag.sha256}, + "strip_prefix": {tag.platform: tag.strip_prefix}, + "url": {tag.platform: tag.urls}, + } + + if tag.sha256: + available_versions[tag.version]["sha256"][tag.platform] = tag.sha256 + if tag.urls: + available_versions[tag.version]["url"][tag.platform] = tag.urls + if tag.strip_prefix: + available_versions[tag.version]["strip_prefix"][tag.platform] = tag.strip_prefix + if tag.patch_strip: + available_versions[tag.version]["patch_strip"][tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.version]["patches"].setdefault(tag.platform, []).extend(tag.patches) register_all = False for tag in mod.tags.override: @@ -434,7 +476,15 @@ _override = tag_class( ) _single_version_override = tag_class( - doc = """Override single python version settings. + doc = """Override single python version URLs and patches for all platforms. + +:::{note} +This will replace any existing configuration for the given python version. + +If you would like to modify the configuration for a specific `(version, +platform), please use the {rule}`python.single_version_platform_override` tag +class. +::: :::{versionadded} 0.36.0 ::: @@ -445,22 +495,71 @@ _single_version_override = tag_class( doc = "Same as the --strip argument of Unix patch.", default = 0, ), - "patches": attr.string_list_dict( + "patches": attr.label_list( mandatory = False, - doc = "A list of labels pointing to patch files to apply for this module as values with keys as values from the PLATFORMS dict. The patch files must exist in the source tree of the top level project. They are applied in the list order. If patches have a single key '*', then the patches will be applied to all available interpreters for that version.", + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied before any platform-specific patches are applied.", ), - "sha256s": attr.string_dict( - mandatory = False, - doc = "The python platform to sha256 dict. The platform key must be present in the PLATFORMS dict.", + "sha256": attr.string_dict( + mandatory = True, + doc = "The python platform to sha256 dict. See {attr}`python.single_version_platform_override.platform` for allowed key values.", ), "strip_prefix": attr.string( mandatory = False, doc = "The 'strip_prefix' for the archive, defaults to 'python'.", default = "python", ), - "url": attr.string( + "urls": attr.string_list( + mandatory = True, + doc = "The URL template to fetch releases for this Python version. See {attr}`python.single_version_platform_override.urls` for documentation.", + ), + "version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + }, +) + +_single_version_platform_override = tag_class( + doc = """Override single python version for a single existing platform. + +If the `(version, platform)` is new, we will add it to the existing versions and will +use the same `url` template. + +:::{note} +If you would like to add or remove platforms to a single python version toolchain +configuration, please use {rule}`python.single_version_override`. +::: + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + default = 0, + ), + "patches": attr.label_list( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied after the common patches are applied.", + ), + "platform": attr.string( mandatory = True, - doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of {python_version}, {platform} and {build} will be interpolated based on the contents in the override and the PLATFORMS dict.", + values = PLATFORMS.keys(), + doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), + ), + "sha256": attr.string( + mandatory = False, + doc = "The sha256 for the archive", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "urls": attr.string_list( + mandatory = False, + doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", ), "version": attr.string( mandatory = True, @@ -476,6 +575,7 @@ python = module_extension( tag_classes = { "override": _override, "single_version_override": _single_version_override, + "single_version_platform_override": _single_version_platform_override, "toolchain": _toolchain, }, **_get_bazel_version_specific_kwargs() diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index d09994a42e..ed6dbba52a 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -660,7 +660,7 @@ def python_register_toolchains( platform = platform, ), sha256 = sha256, - patches = patch_strip, + patch_strip = patch_strip, patches = patches, platform = platform, python_version = python_version, diff --git a/python/versions.bzl b/python/versions.bzl index 5d799677e7..62616653c4 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -669,11 +669,11 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U patches = tool_versions[python_version].get("patches", []) if type(patches) == type({}): - if platform in patches: - patches = patches[platform] - else: - patches = [] - patch_strip = tool_versions[python_version].get("patch_strip") + patches = patches.get(platform, []) + + patch_strip = tool_versions[python_version].get("patch_strip", None) + if type(patch_strip) == type({}): + patch_strip = patch_strip.get(platform, None) return (release_filename, rendered_urls, strip_prefix, patches, patch_strip) From 53105dcb1ec0a707aa2783e146e06d2aa4617e91 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:20:53 +0300 Subject: [PATCH 14/75] make patch_strip handling identical to strip_prefix --- python/versions.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/versions.bzl b/python/versions.bzl index 62616653c4..e1787b7b4a 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -673,7 +673,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U patch_strip = tool_versions[python_version].get("patch_strip", None) if type(patch_strip) == type({}): - patch_strip = patch_strip.get(platform, None) + patch_strip = patch_strip[platform] return (release_filename, rendered_urls, strip_prefix, patches, patch_strip) From 30e922d5ef394b9ea76ba052308139c87421bf4b Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:23:56 +0300 Subject: [PATCH 15/75] add docs --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 976386b0bf..8ad5abba9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,11 @@ A brief description of the categories of changes: * (bzlmod) Properly handle relative path URLs in parse_simpleapi_html.bzl ### Added -* Nothing yet +* (bzlmod): Toolchain overrides can now be done to fully override anything from + within the TOOL_VERSIONS dict using the new + {rule}`python.override`, + {rule}`python.single_version_override` and + {rule}`python.single_version_platform_override`. ### Removed * Nothing yet From 1f28f071bdd3102cdc858d7889511bcb16bed5f3 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Wed, 28 Aug 2024 21:51:09 +0300 Subject: [PATCH 16/75] add some thoughts --- python/private/python.bzl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 666e407193..a72d8fe576 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -28,17 +28,16 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def _python_register_toolchains(name, toolchain_attr, module, **kwargs): +def _python_register_toolchains(*, name, python_version, module, **kwargs): """Calls python_register_toolchains and returns a struct used to collect the toolchains. """ python_register_toolchains( name = name, - python_version = toolchain_attr.python_version, - register_coverage_tool = toolchain_attr.configure_coverage_tool, + python_version = python_version, **kwargs ) return struct( - python_version = toolchain_attr.python_version, + python_version = python_version, name = name, module = struct(name = module.name, is_root = module.is_root), ) @@ -140,12 +139,17 @@ def _python_impl(module_ctx): toolchain_info = None else: toolchain_info = _python_register_toolchains( - toolchain_name, - toolchain_attr, + name = toolchain_name, module = mod, + python_version = toolchain_attr.python_version, + register_coverage_tool = toolchain_attr.configure_coverage_tool, # Extra kwargs to the underlying python_register_toolchains methods - base_url = python_tools.base_url, ignore_root_user_error = ignore_root_user_error, + # TODO @aignas 2024-08-28: the `python_tools` values need + # to be taken from the root module. As it is implemented + # right now, it can race and we may get different results + # based on the module iteration order. + base_url = python_tools.base_url, tool_versions = python_tools.available_versions, # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension # distutils_content = None, From 82ae48f91f0e9e916f27089ff7cb3ae68b033d5f Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:00:25 +0300 Subject: [PATCH 17/75] refactor a little --- python/private/python.bzl | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index a72d8fe576..6eaabab56a 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -28,20 +28,6 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def _python_register_toolchains(*, name, python_version, module, **kwargs): - """Calls python_register_toolchains and returns a struct used to collect the toolchains. - """ - python_register_toolchains( - name = name, - python_version = python_version, - **kwargs - ) - return struct( - python_version = python_version, - name = name, - module = struct(name = module.name, is_root = module.is_root), - ) - def _python_impl(module_ctx): if module_ctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": debug_info = { @@ -138,12 +124,16 @@ def _python_impl(module_ctx): ) toolchain_info = None else: - toolchain_info = _python_register_toolchains( + toolchain_info = struct( + python_version = toolchain_attr.python_version, + name = toolchain_name, + module = struct(name = mod.name, is_root = mod.is_root), + ) + python_register_toolchains( name = toolchain_name, module = mod, python_version = toolchain_attr.python_version, register_coverage_tool = toolchain_attr.configure_coverage_tool, - # Extra kwargs to the underlying python_register_toolchains methods ignore_root_user_error = ignore_root_user_error, # TODO @aignas 2024-08-28: the `python_tools` values need # to be taken from the root module. As it is implemented From 2a30710c06eae683a0e01799b4c5ab3885cef417 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:35:22 +0900 Subject: [PATCH 18/75] ensure that only the root module can do overrides --- examples/bzlmod/MODULE.bazel | 3 +++ examples/bzlmod/MODULE.bazel.lock | 4 ++-- python/private/python.bzl | 37 ++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 1526ecd24d..3656ca3fc9 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -45,6 +45,9 @@ python.override( # Keep the following versions which are present in MINOR_MAPPING, but # are not present above "3.10.14", + # The following is used by the `other_module` and we need to include it here + # as well. + "3.11.9", ], ) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index f573eaab15..8a49759f05 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "4xxemDQ9FFPaWE1Gg3znSsYvxN1GQfil48+VzVFxhV0=", + "bzlTransitiveDigest": "Jx1tjBgdNEMjC49qORvO8TdK8/LwvQbiIXvqzGaY3a8=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "P3LWVNuHO3sawrf2c6eTZ9dfg8nY3vajtjCV5JjwuLM=", + "bzlTransitiveDigest": "3lfSUsZfXRR6537l9mhDz8B8+TeWA1CZ6snjg2nGo2Q=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/python.bzl b/python/private/python.bzl index 6eaabab56a..fbb34c4001 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -58,10 +58,16 @@ def _python_impl(module_ctx): if not module_ctx.modules[0].tags.toolchain: ignore_root_user_error = False + registrations = [] + base_url = None + available_versions = None + for mod in module_ctx.modules: module_toolchain_versions = [] python_tools = _process_tag_classes(mod) + base_url = base_url or python_tools.base_url + available_versions = available_versions or python_tools.available_versions for toolchain_attr in python_tools.registrations: toolchain_version = toolchain_attr.python_version @@ -129,21 +135,17 @@ def _python_impl(module_ctx): name = toolchain_name, module = struct(name = mod.name, is_root = mod.is_root), ) - python_register_toolchains( + + # Register the toolchains outside the main loop so that we can ensure that the + # overrides are correctly applied globally + registrations.append(dict( name = toolchain_name, - module = mod, python_version = toolchain_attr.python_version, + # TODO @aignas 2024-08-30: what about allowing overriding + # the coverage tool registration? register_coverage_tool = toolchain_attr.configure_coverage_tool, ignore_root_user_error = ignore_root_user_error, - # TODO @aignas 2024-08-28: the `python_tools` values need - # to be taken from the root module. As it is implemented - # right now, it can race and we may get different results - # based on the module iteration order. - base_url = python_tools.base_url, - tool_versions = python_tools.available_versions, - # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension - # distutils_content = None, - ) + )) global_toolchain_versions[toolchain_version] = toolchain_info if debug_info: debug_info["toolchains_registered"].append({ @@ -165,6 +167,15 @@ def _python_impl(module_ctx): elif toolchain_info: toolchains.append(toolchain_info) + for kwargs in registrations: + python_register_toolchains( + base_url = base_url, + tool_versions = available_versions, + # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension + # distutils_content = None, + **kwargs + ) + # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -355,8 +366,8 @@ def _process_tag_classes(mod, fail = fail): ]) return struct( - available_versions = available_versions, - base_url = base_url, + available_versions = available_versions if mod.is_root else None, + base_url = base_url if mod.is_root else None, registrations = registrations, ) From 260d4584263f3f19ea8a51acd5850ab644038c9e Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:38:01 +0900 Subject: [PATCH 19/75] chore: move full_version usage to the extension --- python/private/BUILD.bazel | 2 +- python/private/python.bzl | 3 ++- python/private/pythons_hub.bzl | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index a8dc9e70ce..9d1da68d77 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -131,6 +131,7 @@ bzl_library( name = "python_bzl", srcs = ["python.bzl"], deps = [ + ":full_version_bzl", ":python_repositories_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", @@ -161,7 +162,6 @@ bzl_library( name = "pythons_hub_bzl", srcs = ["pythons_hub.bzl"], deps = [ - ":full_version_bzl", ":py_toolchain_suite_bzl", "//python:versions_bzl", ], diff --git a/python/private/python.bzl b/python/private/python.bzl index fbb34c4001..6983012b17 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -16,6 +16,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") +load(":full_version.bzl", "full_version") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") @@ -203,7 +204,7 @@ def _python_impl(module_ctx): render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) for index, toolchain in enumerate(toolchains) ], - toolchain_python_versions = [t.python_version for t in toolchains], + toolchain_python_versions = [full_version(t.python_version) for t in toolchains], # The last toolchain is the default; it can't have version constraints # Despite the implication of the arg name, the values are strs, not bools toolchain_set_python_version_constraints = [ diff --git a/python/private/pythons_hub.bzl b/python/private/pythons_hub.bzl index 7a8c874ed8..d919d05c4f 100644 --- a/python/private/pythons_hub.bzl +++ b/python/private/pythons_hub.bzl @@ -14,7 +14,6 @@ "Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" -load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "python_toolchain_build_file_content", @@ -59,7 +58,7 @@ def _hub_build_file_content( [ python_toolchain_build_file_content( prefix = prefixes[i], - python_version = full_version(python_versions[i]), + python_version = python_versions[i], set_python_version_constraint = set_python_version_constraints[i], user_repository_name = user_repository_names[i], ) From eba99bda23f9c6feca2b9715b5cd16d5ed9abe7d Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:52:14 +0900 Subject: [PATCH 20/75] allow overriding the minor_mapping used to register the toolchains --- examples/bzlmod/MODULE.bazel | 15 ++++++---- examples/bzlmod/MODULE.bazel.lock | 4 +-- python/private/full_version.bzl | 9 +++--- python/private/python.bzl | 39 ++++++++++++++++++++++++-- python/private/python_repositories.bzl | 7 ++++- 5 files changed, 60 insertions(+), 14 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 3656ca3fc9..3ffc4a11ea 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -37,18 +37,23 @@ python.toolchain( python_version = "3.10", ) -# One can override the actual toolchain versions that are available +# One can override the actual toolchain versions that are available, which can be useful +# to when optimizing what gets downloaded and when. python.override( available_python_versions = [ "3.10.9", "3.9.19", - # Keep the following versions which are present in MINOR_MAPPING, but - # are not present above - "3.10.14", # The following is used by the `other_module` and we need to include it here # as well. - "3.11.9", + "3.11.8", ], + # Also override the `minor_mapping` so that when the modules specify a particular + # `3.X` version, we decide what gets used. + minor_mapping = { + "3.10": "3.10.9", + "3.11": "3.11.8", + "3.9": "3.9.19", + }, ) # Or the sources that the toolchains come from for all platforms diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 8a49759f05..bf576cf4cb 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "Jx1tjBgdNEMjC49qORvO8TdK8/LwvQbiIXvqzGaY3a8=", + "bzlTransitiveDigest": "qfrVBAJ6stEo3vj9JDc6g6nWK5QA+jrAFpVV+PEUc7M=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "3lfSUsZfXRR6537l9mhDz8B8+TeWA1CZ6snjg2nGo2Q=", + "bzlTransitiveDigest": "Ubi0PncztQ/Q/s0crkxD5jGgeyVGseprCFPM34NVa6A=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl index 98eeee59a1..5377d7f4f5 100644 --- a/python/private/full_version.bzl +++ b/python/private/full_version.bzl @@ -16,18 +16,19 @@ load("//python:versions.bzl", "MINOR_MAPPING") -def full_version(version): +def full_version(version, minor_mapping = MINOR_MAPPING): """Return a full version. Args: version: the version in `X.Y` or `X.Y.Z` format. + minor_mapping: the mapping from `X.Y` to `X.Y.Z`. Returns: a full version given the version string. If the string is already a major version then we return it as is. """ - if version in MINOR_MAPPING: - return MINOR_MAPPING[version] + if version in minor_mapping: + return minor_mapping[version] parts = version.split(".") if len(parts) == 3: @@ -36,7 +37,7 @@ def full_version(version): fail( "Unknown Python version '{}', available values are: {}".format( version, - ",".join(MINOR_MAPPING.keys()), + ",".join(minor_mapping.keys()), ), ) else: diff --git a/python/private/python.bzl b/python/private/python.bzl index 6983012b17..812b4a3515 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,7 +15,7 @@ "Python toolchain module extensions for use with bzlmod" load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS") load(":full_version.bzl", "full_version") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") @@ -29,6 +29,20 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) +def _parse_version(version): + major, _, version = version.partition(".") + minor, _, version = version.partition(".") + patch, _, version = version.partition(".") + build, _, version = version.partition(".") + + return struct( + # use semver vocabulary here + major = major, + minor = minor, + patch = patch, # this is called `micro` in the Python interpreter versioning scheme + build = build, + ) + def _python_impl(module_ctx): if module_ctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": debug_info = { @@ -62,6 +76,7 @@ def _python_impl(module_ctx): registrations = [] base_url = None available_versions = None + minor_mapping = None for mod in module_ctx.modules: module_toolchain_versions = [] @@ -69,6 +84,7 @@ def _python_impl(module_ctx): python_tools = _process_tag_classes(mod) base_url = base_url or python_tools.base_url available_versions = available_versions or python_tools.available_versions + minor_mapping = minor_mapping or python_tools.minor_mapping for toolchain_attr in python_tools.registrations: toolchain_version = toolchain_attr.python_version @@ -172,6 +188,7 @@ def _python_impl(module_ctx): python_register_toolchains( base_url = base_url, tool_versions = available_versions, + minor_mapping = minor_mapping, # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension # distutils_content = None, **kwargs @@ -204,7 +221,7 @@ def _python_impl(module_ctx): render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) for index, toolchain in enumerate(toolchains) ], - toolchain_python_versions = [full_version(t.python_version) for t in toolchains], + toolchain_python_versions = [full_version(t.python_version, minor_mapping) for t in toolchains], # The last toolchain is the default; it can't have version constraints # Despite the implication of the arg name, the values are strs, not bools toolchain_set_python_version_constraints = [ @@ -283,6 +300,7 @@ def _process_tag_classes(mod, fail = fail): for version, item in TOOL_VERSIONS.items() } base_url = DEFAULT_RELEASE_BASE_URL + minor_mapping = MINOR_MAPPING for tag in mod.tags.toolchain: registrations.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) @@ -357,6 +375,17 @@ def _process_tag_classes(mod, fail = fail): elif tag.register_all_versions: register_all = True + if tag.minor_mapping: + for minor_version, full_version in tag.minor_mapping.items(): + parsed = _parse_version(minor_version) + if parsed.patch or parsed.build: + fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) + parsed = _parse_version(full_version) + if not parsed.patch: + fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) + + minor_mapping = dict(tag.minor_mapping) + break if register_all: @@ -370,6 +399,7 @@ def _process_tag_classes(mod, fail = fail): available_versions = available_versions if mod.is_root else None, base_url = base_url if mod.is_root else None, registrations = registrations, + minor_mapping = minor_mapping, ) def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): @@ -475,6 +505,11 @@ _override = tag_class( doc = "The base URL to be used when downloading toolchains.", default = DEFAULT_RELEASE_BASE_URL, ), + "minor_mapping": attr.string_dict( + mandatory = False, + doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", + default = MINOR_MAPPING, + ), # Internal attributes that are only usable from `rules_python` "register_all_versions": attr.bool(default = False, doc = "`rules_python` **internal** use only!"), diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index ed6dbba52a..1fafa3c937 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -22,6 +22,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load( "//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", + "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS", "get_release_info", @@ -570,6 +571,7 @@ def python_register_toolchains( register_coverage_tool = False, set_python_version_constraint = False, tool_versions = None, + minor_mapping = None, **kwargs): """Convenience macro for users which does typical setup in `WORKSPACE`. @@ -598,10 +600,13 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. + minor_mapping: a dict containing mapping from `X.Y` to `X.Y.Z`. If not supplied, the defaults + in python/versions.bzl will be used. **kwargs: passed to each {rule}`python_repositories` call. """ tool_versions = tool_versions or TOOL_VERSIONS + minor_mapping = minor_mapping or MINOR_MAPPING if BZLMOD_ENABLED: # you cannot used native.register_toolchains when using bzlmod. @@ -609,7 +614,7 @@ def python_register_toolchains( base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) - python_version = full_version(python_version) + python_version = full_version(python_version, minor_mapping) toolchain_repo_name = "{name}_toolchains".format(name = name) From dc039be81c599646cac1c9294961076c917e9fc6 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:55:06 +0900 Subject: [PATCH 21/75] reduce the API a bit --- examples/bzlmod/MODULE.bazel.lock | 4 ++-- python/private/python.bzl | 3 ++- python/private/python_repositories.bzl | 7 +------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index bf576cf4cb..a55563581c 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "qfrVBAJ6stEo3vj9JDc6g6nWK5QA+jrAFpVV+PEUc7M=", + "bzlTransitiveDigest": "N47Uke4zf6WMDk8astNQLWJhjvR2ROdqh6By5mEt0Kc=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "Ubi0PncztQ/Q/s0crkxD5jGgeyVGseprCFPM34NVa6A=", + "bzlTransitiveDigest": "Wv19qqsCkecRdUGJHsUirsfOVjbD+Ya/wZVrt+ki/7M=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/python.bzl b/python/private/python.bzl index 812b4a3515..873209bd0c 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -188,7 +188,8 @@ def _python_impl(module_ctx): python_register_toolchains( base_url = base_url, tool_versions = available_versions, - minor_mapping = minor_mapping, + # Ensure that we pass the full version here. + python_version = full_version(kwargs.pop("python_version"), minor_mapping), # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension # distutils_content = None, **kwargs diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index 1fafa3c937..ed6dbba52a 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -22,7 +22,6 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load( "//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", - "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS", "get_release_info", @@ -571,7 +570,6 @@ def python_register_toolchains( register_coverage_tool = False, set_python_version_constraint = False, tool_versions = None, - minor_mapping = None, **kwargs): """Convenience macro for users which does typical setup in `WORKSPACE`. @@ -600,13 +598,10 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. - minor_mapping: a dict containing mapping from `X.Y` to `X.Y.Z`. If not supplied, the defaults - in python/versions.bzl will be used. **kwargs: passed to each {rule}`python_repositories` call. """ tool_versions = tool_versions or TOOL_VERSIONS - minor_mapping = minor_mapping or MINOR_MAPPING if BZLMOD_ENABLED: # you cannot used native.register_toolchains when using bzlmod. @@ -614,7 +609,7 @@ def python_register_toolchains( base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) - python_version = full_version(python_version, minor_mapping) + python_version = full_version(python_version) toolchain_repo_name = "{name}_toolchains".format(name = name) From f2433f93decd2f2d5efdee0a6354275c28fe1d0e Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:57:43 +0900 Subject: [PATCH 22/75] correctly set the overrides --- python/private/python.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 873209bd0c..9c57b99642 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -74,17 +74,17 @@ def _python_impl(module_ctx): ignore_root_user_error = False registrations = [] - base_url = None - available_versions = None - minor_mapping = None + available_versions = TOOL_VERSIONS + base_url = DEFAULT_RELEASE_BASE_URL + minor_mapping = MINOR_MAPPING for mod in module_ctx.modules: module_toolchain_versions = [] python_tools = _process_tag_classes(mod) - base_url = base_url or python_tools.base_url - available_versions = available_versions or python_tools.available_versions - minor_mapping = minor_mapping or python_tools.minor_mapping + base_url = python_tools.base_url or base_url + available_versions = python_tools.available_versions or available_versions + minor_mapping = python_tools.minor_mapping or minor_mapping for toolchain_attr in python_tools.registrations: toolchain_version = toolchain_attr.python_version @@ -399,8 +399,8 @@ def _process_tag_classes(mod, fail = fail): return struct( available_versions = available_versions if mod.is_root else None, base_url = base_url if mod.is_root else None, + minor_mapping = minor_mapping if mod.is_root else None, registrations = registrations, - minor_mapping = minor_mapping, ) def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): From c262084805e744c90cee3a9e8ce7c7a6b51d124b Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:06:53 +0900 Subject: [PATCH 23/75] allow overriding just about everything --- examples/bzlmod/MODULE.bazel.lock | 4 +- python/private/python.bzl | 350 +++++++++++++++++------------- 2 files changed, 207 insertions(+), 147 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index a55563581c..d45c6a2300 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "N47Uke4zf6WMDk8astNQLWJhjvR2ROdqh6By5mEt0Kc=", + "bzlTransitiveDigest": "pRCr7LSWdBgrFvB+qQ9IabLPuxA/nEMgCMxZLjC3S48=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "Wv19qqsCkecRdUGJHsUirsfOVjbD+Ya/wZVrt+ki/7M=", + "bzlTransitiveDigest": "5jsCtckcJY3sx6Qc3tD82EHZH3Aj9pFO7BtoUBLfUfw=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/python.bzl b/python/private/python.bzl index 9c57b99642..4e7a7c6807 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -16,6 +16,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS") +load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") @@ -74,19 +75,43 @@ def _python_impl(module_ctx): ignore_root_user_error = False registrations = [] - available_versions = TOOL_VERSIONS - base_url = DEFAULT_RELEASE_BASE_URL - minor_mapping = MINOR_MAPPING + seen_versions = {} + + # overrides that can be changed by the root module + overrides = struct( + kwargs = {}, + minor_mapping = dict(MINOR_MAPPING), + default = { + "base_url": DEFAULT_RELEASE_BASE_URL, + "tool_versions": { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() + }, + }, + ) for mod in module_ctx.modules: module_toolchain_versions = [] - python_tools = _process_tag_classes(mod) - base_url = python_tools.base_url or base_url - available_versions = python_tools.available_versions or available_versions - minor_mapping = python_tools.minor_mapping or minor_mapping + requested_toolchains = _process_tag_classes( + mod, + seen_versions = seen_versions, + overrides = overrides, + ) - for toolchain_attr in python_tools.registrations: + for toolchain_attr in requested_toolchains: toolchain_version = toolchain_attr.python_version toolchain_name = "python_" + toolchain_version.replace(".", "_") @@ -155,11 +180,9 @@ def _python_impl(module_ctx): # Register the toolchains outside the main loop so that we can ensure that the # overrides are correctly applied globally - registrations.append(dict( + registrations.append(struct( name = toolchain_name, python_version = toolchain_attr.python_version, - # TODO @aignas 2024-08-30: what about allowing overriding - # the coverage tool registration? register_coverage_tool = toolchain_attr.configure_coverage_tool, ignore_root_user_error = ignore_root_user_error, )) @@ -184,17 +207,21 @@ def _python_impl(module_ctx): elif toolchain_info: toolchains.append(toolchain_info) - for kwargs in registrations: - python_register_toolchains( - base_url = base_url, - tool_versions = available_versions, - # Ensure that we pass the full version here. - python_version = full_version(kwargs.pop("python_version"), minor_mapping), - # TODO @aignas 2024-08-08: allow modifying these values via the bzlmod extension - # distutils_content = None, - **kwargs + for r in registrations: + # Ensure that we pass the full version here. + full_python_version = full_version(r.python_version, overrides.minor_mapping) + kwargs = dict( + python_version = full_python_version, + ignore_root_user_error = r.ignore_root_user_error, + register_coverage_tool = r.register_coverage_tool, ) + # Allow overrides per python version + kwargs.update(overrides.kwargs.get(r.python_version, {})) + kwargs.update(overrides.kwargs.get(full_python_version, {})) + kwargs.update(overrides.default) + python_register_toolchains(name = r.name, **kwargs) + # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -222,7 +249,7 @@ def _python_impl(module_ctx): render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) for index, toolchain in enumerate(toolchains) ], - toolchain_python_versions = [full_version(t.python_version, minor_mapping) for t in toolchains], + toolchain_python_versions = [full_version(t.python_version, overrides.minor_mapping) for t in toolchains], # The last toolchain is the default; it can't have version constraints # Despite the implication of the arg name, the values are strs, not bools toolchain_set_python_version_constraints = [ @@ -281,34 +308,26 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _process_tag_classes(mod, fail = fail): +def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): registrations = [] - seen_versions = {} - available_versions = { - version: { - # Use a dicts straight away so that we could do URL overrides for a - # single version. - "sha256": dict(item["sha256"]), - "strip_prefix": { - platform: item["strip_prefix"] - for platform in item["sha256"] - }, - "url": { - platform: [item["url"]] - for platform in item["sha256"] - }, - } - for version, item in TOOL_VERSIONS.items() - } - base_url = DEFAULT_RELEASE_BASE_URL - minor_mapping = MINOR_MAPPING for tag in mod.tags.toolchain: - registrations.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) + registrations.append(_create_toolchain_attrs_struct( + tag = tag, + toolchain_tag_count = len(mod.tags.toolchain), + )) seen_versions[tag.python_version] = True - if mod.is_root: - for tag in mod.tags.single_version_override: + if not mod.is_root: + return registrations + + available_versions = overrides.default["tool_versions"] + + for tag in mod.tags.single_version_override: + if tag.sha256 or tag.urls: + if not (tag.sha256 and tag.urls): + fail("Both `sha256` and `urls` overrides need to be provided together") + for platform in tag.sha256 or []: if platform not in PLATFORMS: fail("The platform must be one of {allowed} but got '{got}'".format( @@ -316,92 +335,105 @@ def _process_tag_classes(mod, fail = fail): got = platform, )) - override = { - "patch_strip": { - platform: tag.patch_strip - for platform in tag.sha256 - }, - "patches": { - platform: list(tag.patches) - for platform in tag.sha256 - }, - "sha256": dict(tag.sha256), - "strip_prefix": { - platform: tag.strip_prefix - for platform in tag.sha256 - }, - "url": { - platform: list(tag.urls) - for platform in tag.sha256 - }, + sha256 = dict(tag.sha256) or available_versions[tag.version]["sha256"] + override = { + "patch_strip": { + platform: tag.patch_strip + for platform in sha256 + }, + "patches": { + platform: list(tag.patches) + for platform in sha256 + }, + "sha256": sha256, + "strip_prefix": { + platform: tag.strip_prefix + for platform in sha256 + }, + "url": { + platform: list(tag.urls) + for platform in tag.sha256 + } or available_versions[tag.version]["url"], + } + available_versions[tag.version] = {k: v for k, v in override.items() if v} + + if tag.enable_coverage == "no": + overrides.kwargs.setdefault(tag.version, {})["register_coverage_tool"] = False + elif tag.enable_coverage == "yes": + overrides.kwargs.setdefault(tag.version, {})["register_coverage_tool"] = True + + if tag.distutils_content: + overrides.kwargs.setdefault(tag.version, {})["distutils_content"] = tag.distutils_content + if tag.distutils: + overrides.kwargs.setdefault(tag.version, {})["distutils"] = tag.distutils + + for tag in mod.tags.single_version_platform_override: + if tag.version not in available_versions: + if not tag.urls or not tag.sha256 or not tag.strip_prefix: + fail("When introducing a new version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.version)) + available_versions[tag.version] = { + "sha256": {tag.platform: tag.sha256}, + "strip_prefix": {tag.platform: tag.strip_prefix}, + "url": {tag.platform: tag.urls}, } - available_versions[tag.version] = override - - for tag in mod.tags.single_version_platform_override: - if tag.version not in available_versions: - if not tag.urls or not tag.sha256 or not tag.strip_prefix: - fail("When introducing a new version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.version)) - available_versions[tag.version] = { - "sha256": {tag.platform: tag.sha256}, - "strip_prefix": {tag.platform: tag.strip_prefix}, - "url": {tag.platform: tag.urls}, - } - if tag.sha256: - available_versions[tag.version]["sha256"][tag.platform] = tag.sha256 - if tag.urls: - available_versions[tag.version]["url"][tag.platform] = tag.urls - if tag.strip_prefix: - available_versions[tag.version]["strip_prefix"][tag.platform] = tag.strip_prefix - if tag.patch_strip: - available_versions[tag.version]["patch_strip"][tag.platform] = tag.patch_strip - if tag.patches: - available_versions[tag.version]["patches"].setdefault(tag.platform, []).extend(tag.patches) - - register_all = False - for tag in mod.tags.override: - base_url = tag.base_url - if tag.available_python_versions: - all_known_versions = sorted(available_versions) - available_versions = { - v: available_versions[v] if v in available_versions else fail("unknown version '{}', known versions are: {}".format( - v, - all_known_versions, - )) - for v in tag.available_python_versions - } + if tag.sha256: + available_versions[tag.version]["sha256"][tag.platform] = tag.sha256 + if tag.urls: + available_versions[tag.version]["url"][tag.platform] = tag.urls + if tag.strip_prefix: + available_versions[tag.version]["strip_prefix"][tag.platform] = tag.strip_prefix + if tag.patch_strip: + available_versions[tag.version]["patch_strip"][tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.version]["patches"].setdefault(tag.platform, []).extend(tag.patches) + if tag.coverage_tool: + available_versions[tag.version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + + register_all = False + for tag in mod.tags.override: + overrides.kwargs["base_url"] = tag.base_url + if tag.available_python_versions: + all_known_versions = sorted(available_versions) + available_versions = { + v: available_versions[v] if v in available_versions else fail("unknown version '{}', known versions are: {}".format( + v, + all_known_versions, + )) + for v in tag.available_python_versions + } - if tag.register_all_versions and mod.name != "rules_python": - fail("This override can only be used by 'rules_python'") - elif tag.register_all_versions: - register_all = True + if tag.register_all_versions and mod.name != "rules_python": + fail("This override can only be used by 'rules_python'") + elif tag.register_all_versions: + register_all = True - if tag.minor_mapping: - for minor_version, full_version in tag.minor_mapping.items(): - parsed = _parse_version(minor_version) - if parsed.patch or parsed.build: - fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) - parsed = _parse_version(full_version) - if not parsed.patch: - fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) + if tag.minor_mapping: + for minor_version, full_version in tag.minor_mapping.items(): + parsed = _parse_version(minor_version) + if parsed.patch or parsed.build: + fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) + parsed = _parse_version(full_version) + if not parsed.patch: + fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) - minor_mapping = dict(tag.minor_mapping) + overrides.minor_mapping.clear() + overrides.minor_mapping.update(tag.minor_mapping) - break + for key in AUTH_ATTRS: + if getattr(tag, key): + overrides.defaults[key] = getattr(tag, key) - if register_all: - registrations.extend([ - _create_toolchain_attrs_struct(python_version = v) - for v in available_versions.keys() - if v not in seen_versions - ]) + break - return struct( - available_versions = available_versions if mod.is_root else None, - base_url = base_url if mod.is_root else None, - minor_mapping = minor_mapping if mod.is_root else None, - registrations = registrations, - ) + if register_all: + registrations.extend([ + _create_toolchain_attrs_struct(python_version = v) + for v in available_versions.keys() + if v not in seen_versions + ]) + + return registrations def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): if tag and python_version: @@ -496,25 +528,26 @@ _override = tag_class( :::{versionadded} 0.36.0 ::: """, - attrs = { - "available_python_versions": attr.string_list( - mandatory = False, - doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", - ), - "base_url": attr.string( - mandatory = False, - doc = "The base URL to be used when downloading toolchains.", - default = DEFAULT_RELEASE_BASE_URL, - ), - "minor_mapping": attr.string_dict( - mandatory = False, - doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", - default = MINOR_MAPPING, - ), - - # Internal attributes that are only usable from `rules_python` - "register_all_versions": attr.bool(default = False, doc = "`rules_python` **internal** use only!"), - }, + attrs = dict( + { + "available_python_versions": attr.string_list( + mandatory = False, + doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", + ), + "base_url": attr.string( + mandatory = False, + doc = "The base URL to be used when downloading toolchains.", + default = DEFAULT_RELEASE_BASE_URL, + ), + "minor_mapping": attr.string_dict( + mandatory = False, + doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", + default = MINOR_MAPPING, + ), + "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + }, + **AUTH_ATTRS + ), ) _single_version_override = tag_class( @@ -524,7 +557,7 @@ _single_version_override = tag_class( This will replace any existing configuration for the given python version. If you would like to modify the configuration for a specific `(version, -platform), please use the {rule}`python.single_version_platform_override` tag +platform), please use the {obj}`python.single_version_platform_override` tag class. ::: @@ -532,6 +565,27 @@ class. ::: """, attrs = { + "distutils": attr.label( + allow_single_file = True, + doc = "A distutils.cfg file to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "distutils_content": attr.string( + doc = "A distutils.cfg file content to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "enable_coverage": attr.string( + default = "auto", + values = ["auto", "yes", "no"], + doc = """\ +Override coverage registration for a particular version. 'auto' means it will +be left untouched and will work as the module owner who used `python.toolchain` +call intended. `no` and `yes` will force-toggle the coverage tooling for the +given {attr}`version`. +""", + ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", @@ -542,7 +596,7 @@ class. doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied before any platform-specific patches are applied.", ), "sha256": attr.string_dict( - mandatory = True, + mandatory = False, doc = "The python platform to sha256 dict. See {attr}`python.single_version_platform_override.platform` for allowed key values.", ), "strip_prefix": attr.string( @@ -551,7 +605,7 @@ class. default = "python", ), "urls": attr.string_list( - mandatory = True, + mandatory = False, doc = "The URL template to fetch releases for this Python version. See {attr}`python.single_version_platform_override.urls` for documentation.", ), "version": attr.string( @@ -569,13 +623,19 @@ use the same `url` template. :::{note} If you would like to add or remove platforms to a single python version toolchain -configuration, please use {rule}`python.single_version_override`. +configuration, please use {obj}`python.single_version_override`. ::: :::{versionadded} 0.36.0 ::: """, attrs = { + "coverage_tool": attr.label( + doc = """\ +The coverage tool to be used for a particular Python interpreter. This can override +`rules_python` defaults. +""", + ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", From de1f69d9703001b05d97b5bbc52851804775c43a Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:19:57 +0900 Subject: [PATCH 24/75] wip --- python/private/python.bzl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 4e7a7c6807..9049d38f48 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -427,6 +427,7 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): break if register_all: + # FIXME @aignas 2024-08-30: this is technically not correct registrations.extend([ _create_toolchain_attrs_struct(python_version = v) for v in available_versions.keys() @@ -542,7 +543,7 @@ _override = tag_class( "minor_mapping": attr.string_dict( mandatory = False, doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", - default = MINOR_MAPPING, + default = {}, ), "register_all_versions": attr.bool(default = False, doc = "Add all versions"), }, @@ -584,6 +585,12 @@ Override coverage registration for a particular version. 'auto' means it will be left untouched and will work as the module owner who used `python.toolchain` call intended. `no` and `yes` will force-toggle the coverage tooling for the given {attr}`version`. + +TODO: should be able to: +- disable coverage for a version. +- enable coverage for a version with rules_python coverage.py. +- enable coverage for a specific (version, platform) in addition to that. +- disable coverage for a specific (version, platform). """, ), "patch_strip": attr.int( From 1cddfe9c9d1b547180d70f0912564a1bf9a0e780 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:26:33 +0900 Subject: [PATCH 25/75] docs --- python/private/python.bzl | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 9049d38f48..88868e9116 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -466,18 +466,18 @@ Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. -Toolchains in the Root Module +**Toolchains in the Root Module** This class registers all toolchains in the root module. -Toolchains in Sub Modules +**Toolchains in Sub Modules** It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and toolchains in sub modules. You cannot configure more than one toolchain as the default toolchain. -Toolchain set as the default version +**Toolchain set as the default version** This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have @@ -485,11 +485,14 @@ more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain it is set as the default toolchain. -Toolchain repository name +**Toolchain repository name** A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are `major` and `minor` are the Python version from the `python_version` attribute. + +If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will +be `python_{major}_{minor}_{patch}`. """, attrs = { "configure_coverage_tool": attr.bool( @@ -499,12 +502,12 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. "ignore_root_user_error": attr.bool( default = False, doc = """\ -If False, the Python runtime installation will be made read only. This improves +If `False`, the Python runtime installation will be made read only. This improves the ability for Bazel to cache it, but prevents the interpreter from creating -pyc files for the standard library dynamically at runtime as they are loaded. +`.pyc` files for the standard library dynamically at runtime as they are loaded. -If True, the Python runtime installation is read-write. This allows the -interpreter to create pyc files for the standard library, but, because they are +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are created as needed, it adversely affects Bazel's ability to cache the runtime and can result in spurious build failures. """, @@ -516,9 +519,10 @@ can result in spurious build failures. ), "python_version": attr.string( mandatory = True, - doc = "The Python version, in `major.minor` format, e.g " + - "'3.12', to create a toolchain for. Patch level " + - "granularity (e.g. '3.12.1') is not supported.", + doc = """\ +The Python version, in `major.minor` or `major.minor.patch` format, e.g +`3.12` (or `3.12.3`), to create a toolchain for. +""", ), }, ) From f4ac7c26f2f54c90a55a13710b1b31c513a477c9 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:27:09 +0900 Subject: [PATCH 26/75] docs --- python/private/python.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 88868e9116..7d2d51ceb7 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -492,7 +492,7 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `major` and `minor` are the Python version from the `python_version` attribute. If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will -be `python_{major}_{minor}_{patch}`. +be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. """, attrs = { "configure_coverage_tool": attr.bool( From d0493ec5638c5c765690b2770bd4c644b9d8dbe0 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:32:38 +0900 Subject: [PATCH 27/75] docs --- python/private/python.bzl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 7d2d51ceb7..64d8a2b817 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -466,26 +466,29 @@ Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. -**Toolchains in the Root Module** +:::{topic}Toolchains in the Root Module This class registers all toolchains in the root module. +::: -**Toolchains in Sub Modules** +:::{topic}Toolchains in Sub Modules It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and toolchains in sub modules. You cannot configure more than one toolchain as the default toolchain. +::: -**Toolchain set as the default version** +:::{topic}Toolchain set as the default version This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain it is set as the default toolchain. +::: -**Toolchain repository name** +:::{tip} Toolchain repository name A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are @@ -493,6 +496,7 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. +::: """, attrs = { "configure_coverage_tool": attr.bool( From 0508e2bf2fbf10f73bd0eb5819bba94ef0c0bc25 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:50:44 +0900 Subject: [PATCH 28/75] docs --- python/extensions/python.bzl | 25 ++++++++++++++++++++++++- python/private/python.bzl | 32 ++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 4148d90877..9b50ff5c2e 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -12,7 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"""Python toolchain module extensions for use with bzlmod. + +## Basic usage + +The simplest way to configure the toolchain with `rules_python` is as follows. + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) +use_repo(python, "python_3_11") +``` + +For more in-depth documentation see the {rule}`python.toolchain`. + +## Overrides + +Overrides can be done at 3 different levels: +* Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. +* Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. +* Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. +""" load("//python/private:python.bzl", _python = "python") diff --git a/python/private/python.bzl b/python/private/python.bzl index 64d8a2b817..8c6bf6b453 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"Python toolchain module extensions for use with bzlmod." load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS") @@ -466,12 +466,12 @@ Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. -:::{topic}Toolchains in the Root Module +:::{topic} Toolchains in the Root Module This class registers all toolchains in the root module. ::: -:::{topic}Toolchains in Sub Modules +:::{topic} Toolchains in Sub Modules It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name @@ -479,7 +479,7 @@ clashing between toolchains in the root module and toolchains in sub modules. You cannot configure more than one toolchain as the default toolchain. ::: -:::{topic}Toolchain set as the default version +:::{topic} Toolchain set as the default version This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have @@ -488,7 +488,7 @@ toolchains as the default version. If there is only one toolchain it is set as the default toolchain. ::: -:::{tip} Toolchain repository name +:::{topic} Toolchain repository name A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are @@ -497,6 +497,22 @@ A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. ::: + +:::{tip} +In order to use a different name than the above, you can use the following `MODULE.bazel` +syntax: +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) + +use_repo(python, my_python_name = "python_3_11") +``` + +Then the python interpreter will be available as `my_python_name`. +::: """, attrs = { "configure_coverage_tool": attr.bool( @@ -566,7 +582,7 @@ _single_version_override = tag_class( This will replace any existing configuration for the given python version. If you would like to modify the configuration for a specific `(version, -platform), please use the {obj}`python.single_version_platform_override` tag +platform)`, please use the {rule}`python.single_version_platform_override` tag class. ::: @@ -636,9 +652,9 @@ _single_version_platform_override = tag_class( If the `(version, platform)` is new, we will add it to the existing versions and will use the same `url` template. -:::{note} +:::{tip} If you would like to add or remove platforms to a single python version toolchain -configuration, please use {obj}`python.single_version_override`. +configuration, please use {rule}`python.single_version_override`. ::: :::{versionadded} 0.36.0 From 73793730373f1cbd3cfa87dfdbfaa41d74c2122d Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:50:56 +0900 Subject: [PATCH 29/75] docs --- python/private/python.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 8c6bf6b453..da58842ea4 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -582,7 +582,7 @@ _single_version_override = tag_class( This will replace any existing configuration for the given python version. If you would like to modify the configuration for a specific `(version, -platform)`, please use the {rule}`python.single_version_platform_override` tag +platform)`, please use the {obj}`python.single_version_platform_override` tag class. ::: @@ -654,7 +654,7 @@ use the same `url` template. :::{tip} If you would like to add or remove platforms to a single python version toolchain -configuration, please use {rule}`python.single_version_override`. +configuration, please use {obj}`python.single_version_override`. ::: :::{versionadded} 0.36.0 From 42c3f23be2585c35dad4eb93e40460ee517c26a3 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:01:40 +0900 Subject: [PATCH 30/75] docs --- docs/toolchains.md | 12 ++++++++++++ python/extensions/python.bzl | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/docs/toolchains.md b/docs/toolchains.md index fac1bfc6b0..47b80933e9 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -164,6 +164,18 @@ Remember to call `use_repo()` to make repos visible to your module: Python toolchains can be utilized in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the {gh-path}`test_current_py_toolchain ` target for an example. +### Overriding toolchains + +Overrides can be done at 3 different levels: +* Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. +* Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. +* Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. + +The overrides cover the following use-cases: +* Limiting the available toolchains for the entire `bzlmod` transitive graph via {attr}`python.override.available_python_versions`. +* Setting particular `X.Y.Z` python versions when modules request `X.Y` version via {attr}`python.override.minor_mapping`. +* Adding custom {attr}`python.single_version_platform_override.coverage_tool`. +* Adding new python versions via {obj}`python.single_version_override` or {obj}`python.single_version_platform_override`. ## Workspace configuration diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 9b50ff5c2e..ce98dc3fec 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -35,6 +35,10 @@ Overrides can be done at 3 different levels: * Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. * Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. * Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. + +:::{seealso} +The main documentation page on registering [toolchains](/toolchains). +::: """ load("//python/private:python.bzl", _python = "python") From e3bb92de361b3d6b3cad7a8e74b3771915963d66 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 31 Aug 2024 22:09:32 +0900 Subject: [PATCH 31/75] fix docs --- CHANGELOG.md | 6 +++--- docs/toolchains.md | 10 +++++----- python/private/python.bzl | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24b36ee9d..35df542493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,9 +51,9 @@ A brief description of the categories of changes: ### Added * (bzlmod): Toolchain overrides can now be done to fully override anything from within the TOOL_VERSIONS dict using the new - {rule}`python.override`, - {rule}`python.single_version_override` and - {rule}`python.single_version_platform_override`. + {bzl:obj}`@rules_python//python/extensions:python.bzl%python.override`, + {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override` and + {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. ### Removed * Nothing yet diff --git a/docs/toolchains.md b/docs/toolchains.md index 47b80933e9..6743318084 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -167,15 +167,15 @@ Python toolchains can be utilized in other bazel rules, such as `genrule()`, by ### Overriding toolchains Overrides can be done at 3 different levels: -* Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. -* Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. -* Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. +* Overrides affecting all python toolchain versions on all platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.override`. +* Overrides affecting a single toolchain versions on all platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override`. +* Overrides affecting a single toolchain versions on a single platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. The overrides cover the following use-cases: * Limiting the available toolchains for the entire `bzlmod` transitive graph via {attr}`python.override.available_python_versions`. * Setting particular `X.Y.Z` python versions when modules request `X.Y` version via {attr}`python.override.minor_mapping`. * Adding custom {attr}`python.single_version_platform_override.coverage_tool`. -* Adding new python versions via {obj}`python.single_version_override` or {obj}`python.single_version_platform_override`. +* Adding new python versions via {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override` or {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. ## Workspace configuration @@ -252,5 +252,5 @@ automatically registers a higher-priority toolchain; it won't be used unless there is a toolchain misconfiguration somewhere. To aid migration off the Bazel-builtin toolchain, rules_python provides -{obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent +{bzl:obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. diff --git a/python/private/python.bzl b/python/private/python.bzl index da58842ea4..3217803981 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -517,7 +517,7 @@ Then the python interpreter will be available as `my_python_name`. attrs = { "configure_coverage_tool": attr.bool( mandatory = False, - doc = "Whether or not to configure the default coverage tool for the toolchains.", + doc = "Whether or not to configure the default coverage tool provided by `rules_python` for the compatible toolchains.", ), "ignore_root_user_error": attr.bool( default = False, @@ -580,9 +580,11 @@ _single_version_override = tag_class( :::{note} This will replace any existing configuration for the given python version. +::: +:::{tip} If you would like to modify the configuration for a specific `(version, -platform)`, please use the {obj}`python.single_version_platform_override` tag +platform)`, please use the {obj}`single_version_platform_override` tag class. ::: @@ -654,7 +656,7 @@ use the same `url` template. :::{tip} If you would like to add or remove platforms to a single python version toolchain -configuration, please use {obj}`python.single_version_override`. +configuration, please use {obj}`single_version_override`. ::: :::{versionadded} 0.36.0 From 915a6ff14965f440b1be1a6ca7de46ffec146be6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 12:11:41 +0900 Subject: [PATCH 32/75] spike a unit test --- .bazelrc | 4 +- python/private/python.bzl | 95 ++++++++++++++++++++++------------- tests/python/BUILD.bazel | 3 ++ tests/python/python_tests.bzl | 71 ++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 tests/python/BUILD.bazel create mode 100644 tests/python/python_tests.bzl diff --git a/.bazelrc b/.bazelrc index b484751c3c..1ca469cd75 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/python/private/python.bzl b/python/private/python.bzl index 3217803981..8318e8f0e0 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -44,8 +44,18 @@ def _parse_version(version): build = build, ) -def _python_impl(module_ctx): - if module_ctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": +def parse_mods(mctx, *, logger, fail = fail): + """parse_mods returns a struct with parsed tag class content. + + Args: + mctx: module_ctx. + logger: logger. + fail: fail. + + Returns: + a struct with attributes + """ + if mctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": debug_info = { "toolchains_registered": [], } @@ -57,21 +67,19 @@ def _python_impl(module_ctx): # so there is special handling to ensure the last entry is the correct one. toolchains = [] - # We store the default toolchain separately to ensure it is the last - # toolchain added to toolchains. - # This is a toolchain_info struct. - default_toolchain = None - # Map of string Major.Minor to the toolchain_info struct global_toolchain_versions = {} ignore_root_user_error = None - logger = repo_utils.logger(module_ctx, "python") + # We store the default toolchain separately to ensure it is the last + # toolchain added to toolchains. + # This is a toolchain_info struct. + default_toolchain = None # if the root module does not register any toolchain then the # ignore_root_user_error takes its default value: False - if not module_ctx.modules[0].tags.toolchain: + if not mctx.modules[0].tags.toolchain: ignore_root_user_error = False registrations = [] @@ -102,7 +110,7 @@ def _python_impl(module_ctx): }, ) - for mod in module_ctx.modules: + for mod in mctx.modules: module_toolchain_versions = [] requested_toolchains = _process_tag_classes( @@ -207,21 +215,6 @@ def _python_impl(module_ctx): elif toolchain_info: toolchains.append(toolchain_info) - for r in registrations: - # Ensure that we pass the full version here. - full_python_version = full_version(r.python_version, overrides.minor_mapping) - kwargs = dict( - python_version = full_python_version, - ignore_root_user_error = r.ignore_root_user_error, - register_coverage_tool = r.register_coverage_tool, - ) - - # Allow overrides per python version - kwargs.update(overrides.kwargs.get(r.python_version, {})) - kwargs.update(overrides.kwargs.get(full_python_version, {})) - kwargs.update(overrides.default) - python_register_toolchains(name = r.name, **kwargs) - # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -240,23 +233,55 @@ def _python_impl(module_ctx): if len(toolchains) > _MAX_NUM_TOOLCHAINS: fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) + return struct( + debug_info = debug_info, + default_toolchain = default_toolchain, + global_toolchain_versions = global_toolchain_versions, + overrides = overrides, + registrations = registrations, + toolchains = toolchains, + ) + +def _python_impl(mctx): + logger = repo_utils.logger(mctx, "python") + + py = parse_mods( + mctx, + logger = logger, + ) + + for r in py.registrations: + # Ensure that we pass the full version here. + full_python_version = full_version(r.python_version, py.overrides.minor_mapping) + kwargs = dict( + python_version = full_python_version, + ignore_root_user_error = r.ignore_root_user_error, + register_coverage_tool = r.register_coverage_tool, + ) + + # Allow overrides per python version + kwargs.update(py.overrides.kwargs.get(r.python_version, {})) + kwargs.update(py.overrides.kwargs.get(full_python_version, {})) + kwargs.update(py.overrides.default) + python_register_toolchains(name = r.name, **kwargs) + # Create the pythons_hub repo for the interpreter meta data and the # the various toolchains. hub_repo( name = "pythons_hub", - default_python_version = default_toolchain.python_version, + default_python_version = py.default_toolchain.python_version, toolchain_prefixes = [ render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) - for index, toolchain in enumerate(toolchains) + for index, toolchain in enumerate(py.toolchains) ], - toolchain_python_versions = [full_version(t.python_version, overrides.minor_mapping) for t in toolchains], + toolchain_python_versions = [full_version(t.python_version, py.overrides.minor_mapping) for t in py.toolchains], # The last toolchain is the default; it can't have version constraints # Despite the implication of the arg name, the values are strs, not bools toolchain_set_python_version_constraints = [ - "True" if i != len(toolchains) - 1 else "False" - for i in range(len(toolchains)) + "True" if i != len(py.toolchains) - 1 else "False" + for i in range(len(py.toolchains)) ], - toolchain_user_repository_names = [t.name for t in toolchains], + toolchain_user_repository_names = [t.name for t in py.toolchains], ) # This is require in order to support multiple version py_test @@ -265,18 +290,18 @@ def _python_impl(module_ctx): name = "python_versions", python_versions = { version: toolchain.name - for version, toolchain in global_toolchain_versions.items() + for version, toolchain in py.global_toolchain_versions.items() }, ) - if debug_info != None: + if py.debug_info != None: _debug_repo( name = "rules_python_bzlmod_debug", - debug_info = json.encode_indent(debug_info), + debug_info = json.encode_indent(py.debug_info), ) if bazel_features.external_deps.extension_metadata_has_reproducible: - return module_ctx.extension_metadata(reproducible = True) + return mctx.extension_metadata(reproducible = True) else: return None diff --git a/tests/python/BUILD.bazel b/tests/python/BUILD.bazel new file mode 100644 index 0000000000..df2b18101d --- /dev/null +++ b/tests/python/BUILD.bazel @@ -0,0 +1,3 @@ +load(":python_tests.bzl", "python_test_suite") + +python_test_suite(name = "python_tests") diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl new file mode 100644 index 0000000000..02dfd07e1f --- /dev/null +++ b/tests/python/python_tests.bzl @@ -0,0 +1,71 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:python.bzl", "parse_mods") # buildifier: disable=bzl-visibility + +_tests = [] + +def _mock_mctx(root_module, *modules): + return struct( + os = struct(environ = {}), + modules = [ + struct( + name = root_module.name, + tags = root_module.tags, + is_root = True, + ), + ] + list(modules), + ) + +def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = []): + return struct( + name = name, + tags = struct( + toolchain = toolchain, + override = override, + single_version_override = single_version_override, + single_version_platform_override = single_version_platform_override, + ), + ) + +def _toolchain(python_version, *, is_default = False): + return struct( + is_default = is_default, + python_version = python_version, + ) + +def test_default(env): + got = parse_mods( + mctx = _mock_mctx( + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") + +_tests.append(_test_simple) + +def python_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From cd5b1013de305b01ec736cfa7a90f4379ad7d0e9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 16:33:05 +0900 Subject: [PATCH 33/75] move the ignore_root_user_error to a different place --- examples/bzlmod/MODULE.bazel | 2 + python/private/python.bzl | 52 ++++---- tests/python/python_tests.bzl | 241 +++++++++++++++++++++++++++++++++- 3 files changed, 266 insertions(+), 29 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 3ffc4a11ea..ff74790075 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -49,6 +49,8 @@ python.override( ], # Also override the `minor_mapping` so that when the modules specify a particular # `3.X` version, we decide what gets used. + # + # TODO @aignas 2024-09-01: calculate the default based on available python versions. minor_mapping = { "3.10": "3.10.9", "3.11": "3.11.8", diff --git a/python/private/python.bzl b/python/private/python.bzl index 8318e8f0e0..b20836b727 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -55,19 +55,13 @@ def parse_mods(mctx, *, logger, fail = fail): Returns: a struct with attributes """ - if mctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": - debug_info = { - "toolchains_registered": [], - } - else: - debug_info = None # The toolchain_info structs to register, in the order to register them in. # NOTE: The last element is special: it is treated as the default toolchain, # so there is special handling to ensure the last entry is the correct one. toolchains = [] - # Map of string Major.Minor to the toolchain_info struct + # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} ignore_root_user_error = None @@ -192,14 +186,8 @@ def parse_mods(mctx, *, logger, fail = fail): name = toolchain_name, python_version = toolchain_attr.python_version, register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = ignore_root_user_error, )) global_toolchain_versions[toolchain_version] = toolchain_info - if debug_info: - debug_info["toolchains_registered"].append({ - "ignore_root_user_error": ignore_root_user_error, - "name": toolchain_name, - }) if is_default: # This toolchain is setting the default, but the actual @@ -215,6 +203,11 @@ def parse_mods(mctx, *, logger, fail = fail): elif toolchain_info: toolchains.append(toolchain_info) + # TODO @aignas 2024-09-01: deprecate the ignore_root_user_error + # specification via the `python.toolchain` call and instead use + # `python.override`. + overrides.default["ignore_root_user_error"] = ignore_root_user_error + # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -234,12 +227,11 @@ def parse_mods(mctx, *, logger, fail = fail): fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) return struct( - debug_info = debug_info, default_toolchain = default_toolchain, global_toolchain_versions = global_toolchain_versions, - overrides = overrides, - registrations = registrations, toolchains = toolchains, + registrations = registrations, + overrides = overrides, ) def _python_impl(mctx): @@ -250,20 +242,31 @@ def _python_impl(mctx): logger = logger, ) + if mctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": + debug_info = { + "toolchains_registered": [], + } + else: + debug_info = None + for r in py.registrations: # Ensure that we pass the full version here. full_python_version = full_version(r.python_version, py.overrides.minor_mapping) - kwargs = dict( - python_version = full_python_version, - ignore_root_user_error = r.ignore_root_user_error, - register_coverage_tool = r.register_coverage_tool, - ) + kwargs = { + "python_version": full_python_version, + "register_coverage_tool": r.register_coverage_tool, + } # Allow overrides per python version kwargs.update(py.overrides.kwargs.get(r.python_version, {})) kwargs.update(py.overrides.kwargs.get(full_python_version, {})) kwargs.update(py.overrides.default) python_register_toolchains(name = r.name, **kwargs) + if debug_info: + debug_info["default"] = py.overrides.default + debug_info["toolchains_registered"].append({ + "name": r.name, + }) # Create the pythons_hub repo for the interpreter meta data and the # the various toolchains. @@ -294,10 +297,10 @@ def _python_impl(mctx): }, ) - if py.debug_info != None: + if debug_info != None: _debug_repo( name = "rules_python_bzlmod_debug", - debug_info = json.encode_indent(py.debug_info), + debug_info = json.encode_indent(debug_info), ) if bazel_features.external_deps.extension_metadata_has_reproducible: @@ -313,6 +316,9 @@ def _fail_duplicate_module_toolchain_version(version, module): )) def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name, logger): + if not logger: + return + logger.info(lambda: ( "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + "Toolchain '{first_toolchain}' from module '{first_module}' " + diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 02dfd07e1f..c40baed7af 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -28,7 +28,14 @@ def _mock_mctx(root_module, *modules): tags = root_module.tags, is_root = True, ), - ] + list(modules), + ] + [ + struct( + name = mod.name, + tags = mod.tags, + is_root = False, + ) + for mod in modules + ], ) def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = []): @@ -42,15 +49,97 @@ def _mod(*, name, toolchain = [], override = [], single_version_override = [], s ), ) -def _toolchain(python_version, *, is_default = False): +def _toolchain(python_version, *, is_default = False, **kwargs): return struct( is_default = is_default, python_version = python_version, + **kwargs + ) + +def _test_default(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + env.expect.that_collection(py.overrides.minor_mapping.keys()).contains_exactly([ + "3.10", + "3.11", + "3.12", + "3.8", + "3.9", + ]) + env.expect.that_collection(py.overrides.kwargs).has_size(0) + env.expect.that_collection(py.overrides.default.keys()).contains_exactly([ + "base_url", + "ignore_root_user_error", + "tool_versions", + ]) + env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) + env.expect.that_str(py.default_toolchain.python_version).equals("3.11") + want_toolchain = struct( + module = struct(is_root = True, name = "rules_python"), + name = "python_3_11", + python_version = "3.11", + ) + + env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) + env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ + want_toolchain.python_version: want_toolchain, + }) + env.expect.that_collection(py.registrations).contains_exactly([ + struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_default) + +def _test_default_with_patch(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11.2")], + ), + ), + logger = None, + ) + + env.expect.that_str(py.default_toolchain.python_version).equals("3.11.2") + want_toolchain = struct( + module = struct(is_root = True, name = "rules_python"), + name = "python_3_11_2", + python_version = "3.11.2", ) -def test_default(env): - got = parse_mods( + env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) + env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ + want_toolchain.python_version: want_toolchain, + }) + env.expect.that_collection(py.registrations).contains_exactly([ + struct( + name = want_toolchain.name, + python_version = want_toolchain.python_version, + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_default_with_patch) + +def _test_default_non_rules_python(env): + py = parse_mods( mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + ), _mod( name = "rules_python", toolchain = [_toolchain("3.11")], @@ -58,9 +147,149 @@ def test_default(env): ), logger = None, ) - env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") -_tests.append(_test_simple) + env.expect.that_str(py.default_toolchain.python_version).equals("3.12") + my_module_toolchain = struct( + module = struct(is_root = True, name = "my_module"), + name = "python_3_12", + python_version = "3.12", + ) + rules_python_toolchain = struct( + module = struct(is_root = False, name = "rules_python"), + name = "python_3_11", + python_version = "3.11", + ) + + env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) + env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ + my_module_toolchain.python_version: my_module_toolchain, + rules_python_toolchain.python_version: rules_python_toolchain, + }) + env.expect.that_collection(py.registrations).contains_exactly([ + struct( + name = my_module_toolchain.name, + python_version = my_module_toolchain.python_version, + register_coverage_tool = False, + ), + struct( + name = rules_python_toolchain.name, + python_version = rules_python_toolchain.python_version, + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_default_non_rules_python) + +def _test_default_non_rules_python_ignore_root_user_error(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12", ignore_root_user_error = True)], + ), + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + + env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) + env.expect.that_str(py.default_toolchain.python_version).equals("3.12") + my_module_toolchain = struct( + module = struct(is_root = True, name = "my_module"), + name = "python_3_12", + python_version = "3.12", + ) + rules_python_toolchain = struct( + module = struct(is_root = False, name = "rules_python"), + name = "python_3_11", + python_version = "3.11", + ) + + env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) + env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ + my_module_toolchain.python_version: my_module_toolchain, + rules_python_toolchain.python_version: rules_python_toolchain, + }) + env.expect.that_collection(py.registrations).contains_exactly([ + struct( + name = my_module_toolchain.name, + python_version = my_module_toolchain.python_version, + register_coverage_tool = False, + ), + struct( + name = rules_python_toolchain.name, + python_version = rules_python_toolchain.python_version, + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_default_non_rules_python_ignore_root_user_error) + +def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + ), + _mod( + name = "some_module", + toolchain = [_toolchain("3.12", ignore_root_user_error = True)], + ), + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + + env.expect.that_str(py.default_toolchain.python_version).equals("3.13") + env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) + my_module_toolchain = struct( + module = struct(is_root = True, name = "my_module"), + name = "python_3_13", + python_version = "3.13", + ) + some_module_toolchain = struct( + module = struct(is_root = False, name = "some_module"), + name = "python_3_12", + python_version = "3.12", + ) + rules_python_toolchain = struct( + module = struct(is_root = False, name = "rules_python"), + name = "python_3_11", + python_version = "3.11", + ) + + env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain, some_module_toolchain]) + env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ + my_module_toolchain.python_version: my_module_toolchain, + some_module_toolchain.python_version: some_module_toolchain, + rules_python_toolchain.python_version: rules_python_toolchain, + }) + env.expect.that_collection(py.registrations).contains_exactly([ + struct( + name = my_module_toolchain.name, + python_version = my_module_toolchain.python_version, + register_coverage_tool = False, + ), + struct( + name = some_module_toolchain.name, + python_version = some_module_toolchain.python_version, + register_coverage_tool = False, + ), + struct( + name = rules_python_toolchain.name, + python_version = rules_python_toolchain.python_version, + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) def python_test_suite(name): """Create the test suite. From 3655ab0f400d6753219da884d1dd87d66c8fed27 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 16:38:49 +0900 Subject: [PATCH 34/75] move coverage registration tool so that we can reuse structs --- python/private/python.bzl | 7 +--- tests/python/python_tests.bzl | 69 +++++++---------------------------- 2 files changed, 16 insertions(+), 60 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index b20836b727..af7e69b7a6 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -178,15 +178,12 @@ def parse_mods(mctx, *, logger, fail = fail): python_version = toolchain_attr.python_version, name = toolchain_name, module = struct(name = mod.name, is_root = mod.is_root), + register_coverage_tool = toolchain_attr.configure_coverage_tool, ) # Register the toolchains outside the main loop so that we can ensure that the # overrides are correctly applied globally - registrations.append(struct( - name = toolchain_name, - python_version = toolchain_attr.python_version, - register_coverage_tool = toolchain_attr.configure_coverage_tool, - )) + registrations.append(toolchain_info) global_toolchain_versions[toolchain_version] = toolchain_info if is_default: diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index c40baed7af..e129bb0e6e 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -85,19 +85,14 @@ def _test_default(env): module = struct(is_root = True, name = "rules_python"), name = "python_3_11", python_version = "3.11", + register_coverage_tool = False, ) env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ want_toolchain.python_version: want_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([ - struct( - name = "python_3_11", - python_version = "3.11", - register_coverage_tool = False, - ), - ]) + env.expect.that_collection(py.registrations).contains_exactly([want_toolchain]) _tests.append(_test_default) @@ -117,19 +112,14 @@ def _test_default_with_patch(env): module = struct(is_root = True, name = "rules_python"), name = "python_3_11_2", python_version = "3.11.2", + register_coverage_tool = False, ) env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ want_toolchain.python_version: want_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([ - struct( - name = want_toolchain.name, - python_version = want_toolchain.python_version, - register_coverage_tool = False, - ), - ]) + env.expect.that_collection(py.registrations).contains_exactly([want_toolchain]) _tests.append(_test_default_with_patch) @@ -153,11 +143,13 @@ def _test_default_non_rules_python(env): module = struct(is_root = True, name = "my_module"), name = "python_3_12", python_version = "3.12", + register_coverage_tool = False, ) rules_python_toolchain = struct( module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", + register_coverage_tool = False, ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) @@ -165,18 +157,7 @@ def _test_default_non_rules_python(env): my_module_toolchain.python_version: my_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([ - struct( - name = my_module_toolchain.name, - python_version = my_module_toolchain.python_version, - register_coverage_tool = False, - ), - struct( - name = rules_python_toolchain.name, - python_version = rules_python_toolchain.python_version, - register_coverage_tool = False, - ), - ]) + env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python) @@ -201,11 +182,13 @@ def _test_default_non_rules_python_ignore_root_user_error(env): module = struct(is_root = True, name = "my_module"), name = "python_3_12", python_version = "3.12", + register_coverage_tool = False, ) rules_python_toolchain = struct( module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", + register_coverage_tool = False, ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) @@ -213,18 +196,7 @@ def _test_default_non_rules_python_ignore_root_user_error(env): my_module_toolchain.python_version: my_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([ - struct( - name = my_module_toolchain.name, - python_version = my_module_toolchain.python_version, - register_coverage_tool = False, - ), - struct( - name = rules_python_toolchain.name, - python_version = rules_python_toolchain.python_version, - register_coverage_tool = False, - ), - ]) + env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python_ignore_root_user_error) @@ -253,16 +225,19 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): module = struct(is_root = True, name = "my_module"), name = "python_3_13", python_version = "3.13", + register_coverage_tool = False, ) some_module_toolchain = struct( module = struct(is_root = False, name = "some_module"), name = "python_3_12", python_version = "3.12", + register_coverage_tool = False, ) rules_python_toolchain = struct( module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", + register_coverage_tool = False, ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain, some_module_toolchain]) @@ -271,23 +246,7 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): some_module_toolchain.python_version: some_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([ - struct( - name = my_module_toolchain.name, - python_version = my_module_toolchain.python_version, - register_coverage_tool = False, - ), - struct( - name = some_module_toolchain.name, - python_version = some_module_toolchain.python_version, - register_coverage_tool = False, - ), - struct( - name = rules_python_toolchain.name, - python_version = rules_python_toolchain.python_version, - register_coverage_tool = False, - ), - ]) + env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, some_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) From 3ff9618a987018f996d4b27266a0c3cd026d5a31 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 16:40:45 +0900 Subject: [PATCH 35/75] remove a duplicated var --- python/private/python.bzl | 18 ++++++------------ tests/python/python_tests.bzl | 5 ----- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index af7e69b7a6..1524cea893 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -76,7 +76,6 @@ def parse_mods(mctx, *, logger, fail = fail): if not mctx.modules[0].tags.toolchain: ignore_root_user_error = False - registrations = [] seen_versions = {} # overrides that can be changed by the root module @@ -180,10 +179,6 @@ def parse_mods(mctx, *, logger, fail = fail): module = struct(name = mod.name, is_root = mod.is_root), register_coverage_tool = toolchain_attr.configure_coverage_tool, ) - - # Register the toolchains outside the main loop so that we can ensure that the - # overrides are correctly applied globally - registrations.append(toolchain_info) global_toolchain_versions[toolchain_version] = toolchain_info if is_default: @@ -227,7 +222,6 @@ def parse_mods(mctx, *, logger, fail = fail): default_toolchain = default_toolchain, global_toolchain_versions = global_toolchain_versions, toolchains = toolchains, - registrations = registrations, overrides = overrides, ) @@ -246,23 +240,23 @@ def _python_impl(mctx): else: debug_info = None - for r in py.registrations: + for toolchain in py.toolchains: # Ensure that we pass the full version here. - full_python_version = full_version(r.python_version, py.overrides.minor_mapping) + full_python_version = full_version(toolchain.python_version, py.overrides.minor_mapping) kwargs = { "python_version": full_python_version, - "register_coverage_tool": r.register_coverage_tool, + "register_coverage_tool": toolchain.register_coverage_tool, } # Allow overrides per python version - kwargs.update(py.overrides.kwargs.get(r.python_version, {})) + kwargs.update(py.overrides.kwargs.get(toolchain.python_version, {})) kwargs.update(py.overrides.kwargs.get(full_python_version, {})) kwargs.update(py.overrides.default) - python_register_toolchains(name = r.name, **kwargs) + python_register_toolchains(name = toolchain.name, **kwargs) if debug_info: debug_info["default"] = py.overrides.default debug_info["toolchains_registered"].append({ - "name": r.name, + "name": toolchain.name, }) # Create the pythons_hub repo for the interpreter meta data and the diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index e129bb0e6e..63dffebfcb 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -92,7 +92,6 @@ def _test_default(env): env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ want_toolchain.python_version: want_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([want_toolchain]) _tests.append(_test_default) @@ -119,7 +118,6 @@ def _test_default_with_patch(env): env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ want_toolchain.python_version: want_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([want_toolchain]) _tests.append(_test_default_with_patch) @@ -157,7 +155,6 @@ def _test_default_non_rules_python(env): my_module_toolchain.python_version: my_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python) @@ -196,7 +193,6 @@ def _test_default_non_rules_python_ignore_root_user_error(env): my_module_toolchain.python_version: my_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python_ignore_root_user_error) @@ -246,7 +242,6 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): some_module_toolchain.python_version: some_module_toolchain, rules_python_toolchain.python_version: rules_python_toolchain, }) - env.expect.that_collection(py.registrations).contains_exactly([my_module_toolchain, some_module_toolchain, rules_python_toolchain]) _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) From 7beb84a14b81b91608df9e0e44fda4ea5e71e1b5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 17:42:14 +0900 Subject: [PATCH 36/75] remove the default_toolchain from the struct --- python/private/python.bzl | 4 ++-- tests/python/python_tests.bzl | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 1524cea893..14289223ca 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -219,7 +219,7 @@ def parse_mods(mctx, *, logger, fail = fail): fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) return struct( - default_toolchain = default_toolchain, + default_python_version = default_toolchain.python_version, global_toolchain_versions = global_toolchain_versions, toolchains = toolchains, overrides = overrides, @@ -263,7 +263,7 @@ def _python_impl(mctx): # the various toolchains. hub_repo( name = "pythons_hub", - default_python_version = py.default_toolchain.python_version, + default_python_version = py.default_python_version, toolchain_prefixes = [ render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) for index, toolchain in enumerate(py.toolchains) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 63dffebfcb..fd682c9b78 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -80,7 +80,7 @@ def _test_default(env): "tool_versions", ]) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) - env.expect.that_str(py.default_toolchain.python_version).equals("3.11") + env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( module = struct(is_root = True, name = "rules_python"), name = "python_3_11", @@ -106,7 +106,7 @@ def _test_default_with_patch(env): logger = None, ) - env.expect.that_str(py.default_toolchain.python_version).equals("3.11.2") + env.expect.that_str(py.default_python_version).equals("3.11.2") want_toolchain = struct( module = struct(is_root = True, name = "rules_python"), name = "python_3_11_2", @@ -136,7 +136,7 @@ def _test_default_non_rules_python(env): logger = None, ) - env.expect.that_str(py.default_toolchain.python_version).equals("3.12") + env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( module = struct(is_root = True, name = "my_module"), name = "python_3_12", @@ -174,7 +174,7 @@ def _test_default_non_rules_python_ignore_root_user_error(env): ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) - env.expect.that_str(py.default_toolchain.python_version).equals("3.12") + env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( module = struct(is_root = True, name = "my_module"), name = "python_3_12", @@ -215,7 +215,7 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): logger = None, ) - env.expect.that_str(py.default_toolchain.python_version).equals("3.13") + env.expect.that_str(py.default_python_version).equals("3.13") env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) my_module_toolchain = struct( module = struct(is_root = True, name = "my_module"), From 933e6f875d9f94d5bf4714c5a6d9598103797c2c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 17:45:54 +0900 Subject: [PATCH 37/75] make global_toolchain_versions an implementation detail --- python/private/python.bzl | 7 +++---- tests/python/python_tests.bzl | 19 ------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 14289223ca..3b419c3a04 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -174,7 +174,7 @@ def parse_mods(mctx, *, logger, fail = fail): toolchain_info = None else: toolchain_info = struct( - python_version = toolchain_attr.python_version, + python_version = toolchain_version, name = toolchain_name, module = struct(name = mod.name, is_root = mod.is_root), register_coverage_tool = toolchain_attr.configure_coverage_tool, @@ -220,7 +220,6 @@ def parse_mods(mctx, *, logger, fail = fail): return struct( default_python_version = default_toolchain.python_version, - global_toolchain_versions = global_toolchain_versions, toolchains = toolchains, overrides = overrides, ) @@ -283,8 +282,8 @@ def _python_impl(mctx): multi_toolchain_aliases( name = "python_versions", python_versions = { - version: toolchain.name - for version, toolchain in py.global_toolchain_versions.items() + toolchain.python_version: toolchain.name + for toolchain in py.toolchains }, ) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index fd682c9b78..813d30213a 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -89,9 +89,6 @@ def _test_default(env): ) env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) - env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ - want_toolchain.python_version: want_toolchain, - }) _tests.append(_test_default) @@ -115,9 +112,6 @@ def _test_default_with_patch(env): ) env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) - env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ - want_toolchain.python_version: want_toolchain, - }) _tests.append(_test_default_with_patch) @@ -151,10 +145,6 @@ def _test_default_non_rules_python(env): ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) - env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ - my_module_toolchain.python_version: my_module_toolchain, - rules_python_toolchain.python_version: rules_python_toolchain, - }) _tests.append(_test_default_non_rules_python) @@ -189,10 +179,6 @@ def _test_default_non_rules_python_ignore_root_user_error(env): ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) - env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ - my_module_toolchain.python_version: my_module_toolchain, - rules_python_toolchain.python_version: rules_python_toolchain, - }) _tests.append(_test_default_non_rules_python_ignore_root_user_error) @@ -237,11 +223,6 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): ) env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain, some_module_toolchain]) - env.expect.that_dict(py.global_toolchain_versions).contains_exactly({ - my_module_toolchain.python_version: my_module_toolchain, - some_module_toolchain.python_version: some_module_toolchain, - rules_python_toolchain.python_version: rules_python_toolchain, - }) _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) From ad6edbaad72d1556940a5400a4d74afd75fef254 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 17:48:23 +0900 Subject: [PATCH 38/75] refactor tests and ensure we have an ordering test --- tests/python/python_tests.bzl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 813d30213a..3f6c67a43b 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -144,7 +144,10 @@ def _test_default_non_rules_python(env): register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() _tests.append(_test_default_non_rules_python) @@ -178,7 +181,10 @@ def _test_default_non_rules_python_ignore_root_user_error(env): register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain]) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() _tests.append(_test_default_non_rules_python_ignore_root_user_error) @@ -222,7 +228,11 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain, my_module_toolchain, some_module_toolchain]) + env.expect.that_collection(py.toolchains).contains_exactly([ + some_module_toolchain, + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) From 3a459c844047a8476a90a5a507866c01c5d9f816 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 17:49:28 +0900 Subject: [PATCH 39/75] refactor: expose from the module processing function only the bare minimum --- python/private/python.bzl | 34 ++++++++++++++++++++++++---------- tests/python/python_tests.bzl | 9 --------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 3b419c3a04..9c5fb1bbb4 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -44,12 +44,13 @@ def _parse_version(version): build = build, ) -def parse_mods(mctx, *, logger, fail = fail): +def parse_mods(mctx, *, logger, debug = False, fail = fail): """parse_mods returns a struct with parsed tag class content. Args: mctx: module_ctx. logger: logger. + debug: whether to add extra diagnostic information. fail: fail. Returns: @@ -220,18 +221,25 @@ def parse_mods(mctx, *, logger, fail = fail): return struct( default_python_version = default_toolchain.python_version, - toolchains = toolchains, + toolchains = [ + struct( + name = t.name, + python_version = t.python_version, + register_coverage_tool = t.register_coverage_tool, + ) if not debug else struct( + name = t.name, + python_version = t.python_version, + register_coverage_tool = t.register_coverage_tool, + debug = {"module": t.module} if debug else None, + ) + for t in toolchains + ], overrides = overrides, ) def _python_impl(mctx): logger = repo_utils.logger(mctx, "python") - py = parse_mods( - mctx, - logger = logger, - ) - if mctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": debug_info = { "toolchains_registered": [], @@ -239,6 +247,11 @@ def _python_impl(mctx): else: debug_info = None + py = parse_mods( + mctx, + logger = logger, + ) + for toolchain in py.toolchains: # Ensure that we pass the full version here. full_python_version = full_version(toolchain.python_version, py.overrides.minor_mapping) @@ -254,9 +267,10 @@ def _python_impl(mctx): python_register_toolchains(name = toolchain.name, **kwargs) if debug_info: debug_info["default"] = py.overrides.default - debug_info["toolchains_registered"].append({ - "name": toolchain.name, - }) + debug_info["toolchains_registered"].append(dict( + name = toolchain.name, + **toolchain.debug + )) # Create the pythons_hub repo for the interpreter meta data and the # the various toolchains. diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 3f6c67a43b..3f005462b5 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -82,7 +82,6 @@ def _test_default(env): env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( - module = struct(is_root = True, name = "rules_python"), name = "python_3_11", python_version = "3.11", register_coverage_tool = False, @@ -105,7 +104,6 @@ def _test_default_with_patch(env): env.expect.that_str(py.default_python_version).equals("3.11.2") want_toolchain = struct( - module = struct(is_root = True, name = "rules_python"), name = "python_3_11_2", python_version = "3.11.2", register_coverage_tool = False, @@ -132,13 +130,11 @@ def _test_default_non_rules_python(env): env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( - module = struct(is_root = True, name = "my_module"), name = "python_3_12", python_version = "3.12", register_coverage_tool = False, ) rules_python_toolchain = struct( - module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", register_coverage_tool = False, @@ -169,13 +165,11 @@ def _test_default_non_rules_python_ignore_root_user_error(env): env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( - module = struct(is_root = True, name = "my_module"), name = "python_3_12", python_version = "3.12", register_coverage_tool = False, ) rules_python_toolchain = struct( - module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", register_coverage_tool = False, @@ -210,19 +204,16 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): env.expect.that_str(py.default_python_version).equals("3.13") env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) my_module_toolchain = struct( - module = struct(is_root = True, name = "my_module"), name = "python_3_13", python_version = "3.13", register_coverage_tool = False, ) some_module_toolchain = struct( - module = struct(is_root = False, name = "some_module"), name = "python_3_12", python_version = "3.12", register_coverage_tool = False, ) rules_python_toolchain = struct( - module = struct(is_root = False, name = "rules_python"), name = "python_3_11", python_version = "3.11", register_coverage_tool = False, From 760d0eb3a8ddee8685fac82724709d0c420e16ec Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 18:00:56 +0900 Subject: [PATCH 40/75] add a test to check what happens if multiple toolchain calls are made --- tests/python/python_tests.bzl | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 3f005462b5..bb9bebc6cb 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -227,6 +227,53 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) +def _test_first_occurance_of_the_toolchain_wins(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + ), + _mod( + name = "some_module", + toolchain = [_toolchain("3.12", configure_coverage_tool = True)], + ), + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + debug = True, + ) + + env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + # NOTE: coverage stays disabled even though `some_module` was + # configuring something else. + register_coverage_tool = False, + debug = { + "module": struct(is_root = True, name = "my_module"), + }, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + debug = { + "module": struct(is_root = False, name = "rules_python"), + }, + ) + + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() + +_tests.append(_test_first_occurance_of_the_toolchain_wins) + def python_test_suite(name): """Create the test suite. From fab7a23ca10c3d03019d70a2f3ba7425154b6238 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 18:08:10 +0900 Subject: [PATCH 41/75] add todo notes --- python/private/python.bzl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/private/python.bzl b/python/private/python.bzl index 9c5fb1bbb4..2523452468 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -582,6 +582,7 @@ The Python version, in `major.minor` or `major.minor.patch` format, e.g }, ) +# TODO @aignas 2024-09-01: Is the name good enough, should this be something like `python.configure`? _override = tag_class( doc = """Tag class used to override defaults and behaviour of the module extension. @@ -610,6 +611,9 @@ _override = tag_class( ), ) +# TODO @aignas 2024-09-01: should the attributes within this be incorporated into the `python.toolchain` call? +# It does sound like the override being done in a separate method is not something that is super ergonomic, because +# then the ordering needs to be decided and we may need to also think about what to do with the coverage attribute. _single_version_override = tag_class( doc = """Override single python version URLs and patches for all platforms. @@ -683,6 +687,7 @@ TODO: should be able to: }, ) +# Is the name good enough? _single_version_platform_override = tag_class( doc = """Override single python version for a single existing platform. From 061b760ecc130a079ca369584d372e521142be62 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 18:13:27 +0900 Subject: [PATCH 42/75] remove todo --- python/private/python.bzl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 2523452468..0043658055 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -533,6 +533,14 @@ If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. ::: +:::{topic} Toolchain detection +The definition of the first toolchain wins, which means that the root module +can override settings for any python toolchain available. This relies on the +documented module traversal from the {attr}`module_ctx.modules`. + +TODO: add tests for this. +::: + :::{tip} In order to use a different name than the above, you can use the following `MODULE.bazel` syntax: @@ -650,12 +658,6 @@ Override coverage registration for a particular version. 'auto' means it will be left untouched and will work as the module owner who used `python.toolchain` call intended. `no` and `yes` will force-toggle the coverage tooling for the given {attr}`version`. - -TODO: should be able to: -- disable coverage for a version. -- enable coverage for a version with rules_python coverage.py. -- enable coverage for a specific (version, platform) in addition to that. -- disable coverage for a specific (version, platform). """, ), "patch_strip": attr.int( @@ -687,7 +689,9 @@ TODO: should be able to: }, ) -# Is the name good enough? +# TODO @aignas 2024-09-01: Is the name good enough, maybe if the +# `python.override` is renamed to `python.configure` then this can become +# `python.override`? _single_version_platform_override = tag_class( doc = """Override single python version for a single existing platform. From c32b259a8b82cb38a90e8012b3ec7b4601a24606 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 18:31:37 +0900 Subject: [PATCH 43/75] docs: attempt to add `module_ctx` inventory --- python/private/python.bzl | 6 ++++-- sphinxdocs/inventories/bazel_inventory.txt | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 0043658055..6673fcd813 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -536,9 +536,11 @@ be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. :::{topic} Toolchain detection The definition of the first toolchain wins, which means that the root module can override settings for any python toolchain available. This relies on the -documented module traversal from the {attr}`module_ctx.modules`. +documented module traversal from the {obj}`module_ctx.modules`. +::: -TODO: add tests for this. +:::{todo} +fix the `module_ctx.modules` reference so that we can link back to the bazel page. ::: :::{tip} diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index 445f0f71f4..49bb4d32e8 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -69,3 +69,20 @@ Name bzl:type 1 concepts/labels#target-names - CcInfo bzl:provider 1 rules/lib/providers/CcInfo - CcInfo.linking_context bzl:provider-field 1 rules/lib/providers/CcInfo#linking_context - ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - +module_ctx bzl:obj 1 rules/lib/builtins/module_ctx - +module_ctx.download bzl:function 1 rules/lib/builtins/module_ctx#download - +module_ctx.download_and_extract bzl:function 1 rules/lib/builtins/module_ctx#download_and_extract - +module_ctx.execute bzl:function 1 rules/lib/builtins/module_ctx#execute - +module_ctx.extension_metadata bzl:function 1 rules/lib/builtins/module_ctx#extension_metadata - +module_ctx.extract bzl:function 1 rules/lib/builtins/module_ctx#extract - +module_ctx.file bzl:function 1 rules/lib/builtins/module_ctx#file - +module_ctx.getenv bzl:function 1 rules/lib/builtins/module_ctx#getenv - +module_ctx.is_dev_dependency bzl:obj 1 rules/lib/builtins/module_ctx#is_dev_dependency - +module_ctx.modules bzl:obj 1 rules/lib/builtins/module_ctx#modules - +module_ctx.os bzl:obj 1 rules/lib/builtins/module_ctx#os - +module_ctx.path bzl:function 1 rules/lib/builtins/module_ctx#path - +module_ctx.read bzl:function 1 rules/lib/builtins/module_ctx#read - +module_ctx.report_progress bzl:function 1 rules/lib/builtins/module_ctx#report_progress - +module_ctx.root_module_has_non_dev_dependency bzl:function 1 rules/lib/builtins/module_ctx#root_module_has_non_dev_dependency - +module_ctx.watch bzl:function 1 rules/lib/builtins/module_ctx#watch - +module_ctx.which bzl:function 1 rules/lib/builtins/module_ctx#which - From 671d0d93cd402cc50436d3bf6c53c9c049cc0604 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 1 Sep 2024 23:23:22 +0900 Subject: [PATCH 44/75] wip --- python/private/python.bzl | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 6673fcd813..613d7ad90c 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -281,7 +281,10 @@ def _python_impl(mctx): render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH) for index, toolchain in enumerate(py.toolchains) ], - toolchain_python_versions = [full_version(t.python_version, py.overrides.minor_mapping) for t in py.toolchains], + toolchain_python_versions = [ + full_version(t.python_version, py.overrides.minor_mapping) + for t in py.toolchains + ], # The last toolchain is the default; it can't have version constraints # Despite the implication of the arg name, the values are strs, not bools toolchain_set_python_version_constraints = [ @@ -392,11 +395,6 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): } available_versions[tag.version] = {k: v for k, v in override.items() if v} - if tag.enable_coverage == "no": - overrides.kwargs.setdefault(tag.version, {})["register_coverage_tool"] = False - elif tag.enable_coverage == "yes": - overrides.kwargs.setdefault(tag.version, {})["register_coverage_tool"] = True - if tag.distutils_content: overrides.kwargs.setdefault(tag.version, {})["distutils_content"] = tag.distutils_content if tag.distutils: @@ -610,6 +608,8 @@ _override = tag_class( doc = "The base URL to be used when downloading toolchains.", default = DEFAULT_RELEASE_BASE_URL, ), + # TODO @aignas 2024-09-01: could be replaced by an attribute on the `toolchain` with + # a boolean attribute `is_default_minor`. "minor_mapping": attr.string_dict( mandatory = False, doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", @@ -641,6 +641,18 @@ class. ::: """, attrs = { + # NOTE @aignas 2024-09-01: all of the attributes except for `version` + # can be part of the `python.toolchain` call. That would make it more + # ergonomic to define new toolchains and to override values for old + # toolchains. The same semantics of the `first one wins` would apply, + # so technically there is no need for any overrides? + # + # Although these attributes would override the code that is used by the + # code in non-root modules, so technically this could be thought as + # being overridden. + # + # rules_go has a single download call: + # https://github.com/bazelbuild/rules_go/blob/master/go/private/extensions.bzl#L38 "distutils": attr.label( allow_single_file = True, doc = "A distutils.cfg file to be included in the Python installation. " + @@ -652,16 +664,6 @@ class. "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", mandatory = False, ), - "enable_coverage": attr.string( - default = "auto", - values = ["auto", "yes", "no"], - doc = """\ -Override coverage registration for a particular version. 'auto' means it will -be left untouched and will work as the module owner who used `python.toolchain` -call intended. `no` and `yes` will force-toggle the coverage tooling for the -given {attr}`version`. -""", - ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", From 5e5b8f68a9be8eeccd0fbede8e0ab812f0dbba0b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 2 Sep 2024 22:30:17 +0900 Subject: [PATCH 45/75] Allow overriding ignore_root_user_error via override API --- python/private/python.bzl | 31 ++++++++++++++++-------- tests/python/python_tests.bzl | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 613d7ad90c..dcfbf6d93d 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -196,10 +196,7 @@ def parse_mods(mctx, *, logger, debug = False, fail = fail): elif toolchain_info: toolchains.append(toolchain_info) - # TODO @aignas 2024-09-01: deprecate the ignore_root_user_error - # specification via the `python.toolchain` call and instead use - # `python.override`. - overrides.default["ignore_root_user_error"] = ignore_root_user_error + overrides.default.setdefault("ignore_root_user_error", ignore_root_user_error) # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. @@ -453,9 +450,9 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): overrides.minor_mapping.clear() overrides.minor_mapping.update(tag.minor_mapping) - for key in AUTH_ATTRS: - if getattr(tag, key): - overrides.defaults[key] = getattr(tag, key) + for key in sorted(AUTH_ATTRS) + ["ignore_root_user_error"]: + if getattr(tag, key, None): + overrides.default[key] = getattr(tag, key) break @@ -608,6 +605,20 @@ _override = tag_class( doc = "The base URL to be used when downloading toolchains.", default = DEFAULT_RELEASE_BASE_URL, ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ +If `False`, the Python runtime installation will be made read only. This improves +the ability for Bazel to cache it, but prevents the interpreter from creating +`.pyc` files for the standard library dynamically at runtime as they are loaded. + +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are +created as needed, it adversely affects Bazel's ability to cache the runtime and +can result in spurious build failures. +""", + mandatory = False, + ), # TODO @aignas 2024-09-01: could be replaced by an attribute on the `toolchain` with # a boolean attribute `is_default_minor`. "minor_mapping": attr.string_dict( @@ -653,6 +664,9 @@ class. # # rules_go has a single download call: # https://github.com/bazelbuild/rules_go/blob/master/go/private/extensions.bzl#L38 + # + # However, we need to understand how to accommodate the fact that + # {attr}`single_version_override.version` only allows patch versions. "distutils": attr.label( allow_single_file = True, doc = "A distutils.cfg file to be included in the Python installation. " + @@ -693,9 +707,6 @@ class. }, ) -# TODO @aignas 2024-09-01: Is the name good enough, maybe if the -# `python.override` is renamed to `python.configure` then this can become -# `python.override`? _single_version_platform_override = tag_class( doc = """Override single python version for a single existing platform. diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index bb9bebc6cb..9a23d413a0 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -56,6 +56,15 @@ def _toolchain(python_version, *, is_default = False, **kwargs): **kwargs ) +def _override(**kwargs): + return struct( + base_url = kwargs.get("base_url", ""), + available_python_versions = kwargs.get("available_python_versions", []), + register_all_versions = kwargs.get("register_all_versions", False), + ignore_root_user_error = kwargs.get("ignore_root_user_error", False), + minor_mapping = kwargs.get("minor_mapping", {}), + ) + def _test_default(env): py = parse_mods( mctx = _mock_mctx( @@ -182,6 +191,42 @@ def _test_default_non_rules_python_ignore_root_user_error(env): _tests.append(_test_default_non_rules_python_ignore_root_user_error) +def _test_default_non_rules_python_ignore_root_user_error_override(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [_override(ignore_root_user_error = True)], + ), + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + + env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) + env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() + +_tests.append(_test_default_non_rules_python_ignore_root_user_error_override) + def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): py = parse_mods( mctx = _mock_mctx( From e17f2086f331dea527df94d46e6349fcc6a41b56 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 2 Sep 2024 22:35:29 +0900 Subject: [PATCH 46/75] test: add more override tests --- tests/python/python_tests.bzl | 64 +++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 9a23d413a0..3c2073c870 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -56,13 +56,22 @@ def _toolchain(python_version, *, is_default = False, **kwargs): **kwargs ) -def _override(**kwargs): +def _override( + auth_patterns = {}, + available_python_versions = [], + base_url = "", + ignore_root_user_error = False, + minor_mapping = {}, + netrc = "", + register_all_versions = False): return struct( - base_url = kwargs.get("base_url", ""), - available_python_versions = kwargs.get("available_python_versions", []), - register_all_versions = kwargs.get("register_all_versions", False), - ignore_root_user_error = kwargs.get("ignore_root_user_error", False), - minor_mapping = kwargs.get("minor_mapping", {}), + auth_patterns = auth_patterns, + available_python_versions = available_python_versions, + base_url = base_url, + ignore_root_user_error = ignore_root_user_error, + minor_mapping = minor_mapping, + netrc = netrc, + register_all_versions = register_all_versions, ) def _test_default(env): @@ -319,6 +328,49 @@ def _test_first_occurance_of_the_toolchain_wins(env): _tests.append(_test_first_occurance_of_the_toolchain_wins) +def _test_auth_overrides(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [ + _override( + netrc = "/my/netrc", + auth_patterns = {"foo": "bar"}, + ), + ], + ), + _mod( + name = "rules_python", + toolchain = [_toolchain("3.11")], + ), + ), + logger = None, + ) + + env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) + env.expect.that_str(py.overrides.default["netrc"]).equals("/my/netrc") + env.expect.that_dict(py.overrides.default["auth_patterns"]).contains_exactly({"foo": "bar"}) + env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, # default toolchain is last + ]).in_order() + +_tests.append(_test_auth_overrides) + def python_test_suite(name): """Create the test suite. From 75314e5e54f18ea4a610bead9217a3014959b04b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:19:50 +0900 Subject: [PATCH 47/75] refactor: dryer tests --- tests/python/python_tests.bzl | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 3c2073c870..f1f2ce673a 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -15,17 +15,20 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:python.bzl", "parse_mods") # buildifier: disable=bzl-visibility +load("//python/private:python.bzl", _parse_mods = "parse_mods") # buildifier: disable=bzl-visibility _tests = [] -def _mock_mctx(root_module, *modules): +def parse_mods(*, mctx, **kwargs): + return _parse_mods(mctx = mctx, logger = None, **kwargs) + +def _mock_mctx(*modules): return struct( os = struct(environ = {}), modules = [ struct( - name = root_module.name, - tags = root_module.tags, + name = modules[0].name, + tags = modules[0].tags, is_root = True, ), ] + [ @@ -34,7 +37,7 @@ def _mock_mctx(root_module, *modules): tags = mod.tags, is_root = False, ) - for mod in modules + for mod in modules[1:] ], ) @@ -82,8 +85,8 @@ def _test_default(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) + env.expect.that_collection(py.overrides.minor_mapping.keys()).contains_exactly([ "3.10", "3.11", @@ -117,7 +120,6 @@ def _test_default_with_patch(env): toolchain = [_toolchain("3.11.2")], ), ), - logger = None, ) env.expect.that_str(py.default_python_version).equals("3.11.2") @@ -126,7 +128,6 @@ def _test_default_with_patch(env): python_version = "3.11.2", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) _tests.append(_test_default_with_patch) @@ -143,7 +144,6 @@ def _test_default_non_rules_python(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) env.expect.that_str(py.default_python_version).equals("3.12") @@ -177,7 +177,6 @@ def _test_default_non_rules_python_ignore_root_user_error(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) @@ -213,7 +212,6 @@ def _test_default_non_rules_python_ignore_root_user_error_override(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) @@ -252,7 +250,6 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) env.expect.that_str(py.default_python_version).equals("3.13") @@ -297,7 +294,6 @@ def _test_first_occurance_of_the_toolchain_wins(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, debug = True, ) @@ -346,7 +342,6 @@ def _test_auth_overrides(env): toolchain = [_toolchain("3.11")], ), ), - logger = None, ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) From 7a173e482cb52a4fa4a468f98f575029bdf66027 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:27:11 +0900 Subject: [PATCH 48/75] refactor: dryer tests --- tests/python/python_tests.bzl | 100 +++++++++++----------------------- 1 file changed, 32 insertions(+), 68 deletions(-) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index f1f2ce673a..f41a5f44af 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -22,9 +22,9 @@ _tests = [] def parse_mods(*, mctx, **kwargs): return _parse_mods(mctx = mctx, logger = None, **kwargs) -def _mock_mctx(*modules): +def _mock_mctx(*modules, environ = {}): return struct( - os = struct(environ = {}), + os = struct(environ = environ), modules = [ struct( name = modules[0].name, @@ -80,10 +80,7 @@ def _override( def _test_default(env): py = parse_mods( mctx = _mock_mctx( - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) @@ -102,12 +99,12 @@ def _test_default(env): ]) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) env.expect.that_str(py.default_python_version).equals("3.11") + want_toolchain = struct( name = "python_3_11", python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) _tests.append(_test_default) @@ -115,14 +112,12 @@ _tests.append(_test_default) def _test_default_with_patch(env): py = parse_mods( mctx = _mock_mctx( - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11.2")], - ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11.2")]), ), ) env.expect.that_str(py.default_python_version).equals("3.11.2") + want_toolchain = struct( name = "python_3_11_2", python_version = "3.11.2", @@ -135,18 +130,13 @@ _tests.append(_test_default_with_patch) def _test_default_non_rules_python(env): py = parse_mods( mctx = _mock_mctx( - _mod( - name = "my_module", - toolchain = [_toolchain("3.12")], - ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "my_module", toolchain = [_toolchain("3.12")]), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( name = "python_3_12", python_version = "3.12", @@ -157,7 +147,6 @@ def _test_default_non_rules_python(env): python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ rules_python_toolchain, my_module_toolchain, # default toolchain is last @@ -172,15 +161,13 @@ def _test_default_non_rules_python_ignore_root_user_error(env): name = "my_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)], ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( name = "python_3_12", python_version = "3.12", @@ -191,10 +178,9 @@ def _test_default_non_rules_python_ignore_root_user_error(env): python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ rules_python_toolchain, - my_module_toolchain, # default toolchain is last + my_module_toolchain, ]).in_order() _tests.append(_test_default_non_rules_python_ignore_root_user_error) @@ -207,15 +193,13 @@ def _test_default_non_rules_python_ignore_root_user_error_override(env): toolchain = [_toolchain("3.12")], override = [_override(ignore_root_user_error = True)], ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(True) env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( name = "python_3_12", python_version = "3.12", @@ -226,10 +210,9 @@ def _test_default_non_rules_python_ignore_root_user_error_override(env): python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ rules_python_toolchain, - my_module_toolchain, # default toolchain is last + my_module_toolchain, ]).in_order() _tests.append(_test_default_non_rules_python_ignore_root_user_error_override) @@ -237,23 +220,15 @@ _tests.append(_test_default_non_rules_python_ignore_root_user_error_override) def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): py = parse_mods( mctx = _mock_mctx( - _mod( - name = "my_module", - toolchain = [_toolchain("3.13")], - ), - _mod( - name = "some_module", - toolchain = [_toolchain("3.12", ignore_root_user_error = True)], - ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "my_module", toolchain = [_toolchain("3.13")]), + _mod(name = "some_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)]), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) env.expect.that_str(py.default_python_version).equals("3.13") env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) + my_module_toolchain = struct( name = "python_3_13", python_version = "3.13", @@ -269,11 +244,10 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ some_module_toolchain, rules_python_toolchain, - my_module_toolchain, # default toolchain is last + my_module_toolchain, ]).in_order() _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_module) @@ -281,23 +255,15 @@ _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_mod def _test_first_occurance_of_the_toolchain_wins(env): py = parse_mods( mctx = _mock_mctx( - _mod( - name = "my_module", - toolchain = [_toolchain("3.12")], - ), - _mod( - name = "some_module", - toolchain = [_toolchain("3.12", configure_coverage_tool = True)], - ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "my_module", toolchain = [_toolchain("3.12")]), + _mod(name = "some_module", toolchain = [_toolchain("3.12", configure_coverage_tool = True)]), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), debug = True, ) env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( name = "python_3_12", python_version = "3.12", @@ -316,7 +282,6 @@ def _test_first_occurance_of_the_toolchain_wins(env): "module": struct(is_root = False, name = "rules_python"), }, ) - env.expect.that_collection(py.toolchains).contains_exactly([ rules_python_toolchain, my_module_toolchain, # default toolchain is last @@ -337,17 +302,17 @@ def _test_auth_overrides(env): ), ], ), - _mod( - name = "rules_python", - toolchain = [_toolchain("3.11")], - ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) - env.expect.that_bool(py.overrides.default["ignore_root_user_error"]).equals(False) - env.expect.that_str(py.overrides.default["netrc"]).equals("/my/netrc") - env.expect.that_dict(py.overrides.default["auth_patterns"]).contains_exactly({"foo": "bar"}) + env.expect.that_dict(py.overrides.default).contains_at_least({ + "auth_patterns": {"foo": "bar"}, + "ignore_root_user_error": False, + "netrc": "/my/netrc", + }) env.expect.that_str(py.default_python_version).equals("3.12") + my_module_toolchain = struct( name = "python_3_12", python_version = "3.12", @@ -358,10 +323,9 @@ def _test_auth_overrides(env): python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ rules_python_toolchain, - my_module_toolchain, # default toolchain is last + my_module_toolchain, ]).in_order() _tests.append(_test_auth_overrides) From 43e016855c767d94584460a63eacf4b8044deb7a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:51:27 +0900 Subject: [PATCH 49/75] test that we can set patches --- python/private/python.bzl | 74 ++++++++++++++++++----------------- tests/python/python_tests.bzl | 62 +++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 35 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index dcfbf6d93d..cf42e6d0fb 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -370,16 +370,8 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): got = platform, )) - sha256 = dict(tag.sha256) or available_versions[tag.version]["sha256"] + sha256 = dict(tag.sha256) or available_versions[tag.python_version]["sha256"] override = { - "patch_strip": { - platform: tag.patch_strip - for platform in sha256 - }, - "patches": { - platform: list(tag.patches) - for platform in sha256 - }, "sha256": sha256, "strip_prefix": { platform: tag.strip_prefix @@ -388,50 +380,62 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): "url": { platform: list(tag.urls) for platform in tag.sha256 - } or available_versions[tag.version]["url"], + } or available_versions[tag.python_version]["url"], } - available_versions[tag.version] = {k: v for k, v in override.items() if v} + + if tag.patches: + override["patch_strip"] = { + platform: tag.patch_strip + for platform in sha256 + } + override["patches"] = { + platform: list(tag.patches) + for platform in sha256 + } + + available_versions[tag.python_version] = {k: v for k, v in override.items() if v} if tag.distutils_content: - overrides.kwargs.setdefault(tag.version, {})["distutils_content"] = tag.distutils_content + overrides.kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content if tag.distutils: - overrides.kwargs.setdefault(tag.version, {})["distutils"] = tag.distutils + overrides.kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils for tag in mod.tags.single_version_platform_override: - if tag.version not in available_versions: + if tag.python_version not in available_versions: if not tag.urls or not tag.sha256 or not tag.strip_prefix: - fail("When introducing a new version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.version)) - available_versions[tag.version] = { + fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) + available_versions[tag.python_version] = { "sha256": {tag.platform: tag.sha256}, "strip_prefix": {tag.platform: tag.strip_prefix}, "url": {tag.platform: tag.urls}, } if tag.sha256: - available_versions[tag.version]["sha256"][tag.platform] = tag.sha256 + available_versions[tag.python_version]["sha256"][tag.platform] = tag.sha256 if tag.urls: - available_versions[tag.version]["url"][tag.platform] = tag.urls + available_versions[tag.python_version]["url"][tag.platform] = tag.urls if tag.strip_prefix: - available_versions[tag.version]["strip_prefix"][tag.platform] = tag.strip_prefix + available_versions[tag.python_version]["strip_prefix"][tag.platform] = tag.strip_prefix if tag.patch_strip: - available_versions[tag.version]["patch_strip"][tag.platform] = tag.patch_strip + available_versions[tag.python_version]["patch_strip"][tag.platform] = tag.patch_strip if tag.patches: - available_versions[tag.version]["patches"].setdefault(tag.platform, []).extend(tag.patches) + available_versions[tag.python_version]["patches"].setdefault(tag.platform, []).extend(tag.patches) if tag.coverage_tool: - available_versions[tag.version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool register_all = False for tag in mod.tags.override: overrides.kwargs["base_url"] = tag.base_url if tag.available_python_versions: - all_known_versions = sorted(available_versions) - available_versions = { - v: available_versions[v] if v in available_versions else fail("unknown version '{}', known versions are: {}".format( + all_versions = dict(available_versions) + available_versions.clear() + available_versions.update({ + v: all_versions[v] if v in all_versions else fail("unknown version '{}', known versions are: {}".format( v, - all_known_versions, + sorted(all_versions), )) for v in tag.available_python_versions - } + }) if tag.register_all_versions and mod.name != "rules_python": fail("This override can only be used by 'rules_python'") @@ -687,6 +691,10 @@ class. mandatory = False, doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied before any platform-specific patches are applied.", ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), "sha256": attr.string_dict( mandatory = False, doc = "The python platform to sha256 dict. See {attr}`python.single_version_platform_override.platform` for allowed key values.", @@ -700,10 +708,6 @@ class. mandatory = False, doc = "The URL template to fetch releases for this Python version. See {attr}`python.single_version_platform_override.urls` for documentation.", ), - "version": attr.string( - mandatory = True, - doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", - ), }, ) @@ -742,6 +746,10 @@ The coverage tool to be used for a particular Python interpreter. This can overr values = PLATFORMS.keys(), doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), "sha256": attr.string( mandatory = False, doc = "The sha256 for the archive", @@ -755,10 +763,6 @@ The coverage tool to be used for a particular Python interpreter. This can overr mandatory = False, doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", ), - "version": attr.string( - mandatory = True, - doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", - ), }, ) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index f41a5f44af..d79ffe5b67 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -77,6 +77,12 @@ def _override( register_all_versions = register_all_versions, ) +def _single_version_override( + **kwargs): + return struct( + **kwargs + ) + def _test_default(env): py = parse_mods( mctx = _mock_mctx( @@ -330,6 +336,62 @@ def _test_auth_overrides(env): _tests.append(_test_auth_overrides) +def _test_add_new_version(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 0, + patches = [], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.overrides.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + ]) + env.expect.that_dict(py.overrides.default["tool_versions"]["3.13.0"]).contains_exactly({ + "sha256": {"aarch64-unknown-linux-gnu": "deadbeef"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "prefix"}, + "url": {"aarch64-unknown-linux-gnu": ["example.org"]}, + }) + env.expect.that_dict(py.overrides.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_new_version) + def python_test_suite(name): """Create the test suite. From a12751c1bd0145c0d9fee6791e3f9625f9f22ce3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:05:06 +0900 Subject: [PATCH 50/75] add a test for single version platform override --- python/private/python.bzl | 16 +++---- tests/python/python_tests.bzl | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index cf42e6d0fb..c98787f029 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -410,18 +410,18 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): "url": {tag.platform: tag.urls}, } + if tag.coverage_tool: + available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + if tag.patch_strip: + available_versions[tag.python_version]["patch_strip"][tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.python_version]["patches"][tag.platform] = list(tag.patches) if tag.sha256: available_versions[tag.python_version]["sha256"][tag.platform] = tag.sha256 - if tag.urls: - available_versions[tag.python_version]["url"][tag.platform] = tag.urls if tag.strip_prefix: available_versions[tag.python_version]["strip_prefix"][tag.platform] = tag.strip_prefix - if tag.patch_strip: - available_versions[tag.python_version]["patch_strip"][tag.platform] = tag.patch_strip - if tag.patches: - available_versions[tag.python_version]["patches"].setdefault(tag.platform, []).extend(tag.patches) - if tag.coverage_tool: - available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + if tag.urls: + available_versions[tag.python_version]["url"][tag.platform] = tag.urls register_all = False for tag in mod.tags.override: diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index d79ffe5b67..afbb5dc20a 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -83,6 +83,12 @@ def _single_version_override( **kwargs ) +def _single_version_platform_override( + **kwargs): + return struct( + **kwargs + ) + def _test_default(env): py = parse_mods( mctx = _mock_mctx( @@ -392,6 +398,82 @@ def _test_add_new_version(env): _tests.append(_test_add_new_version) +def _test_add_patches(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-apple-darwin": "deadbeef", + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 1, + patches = ["common.txt"], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.0", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.13.0"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_dict(py.overrides.default["tool_versions"]).contains_exactly({ + "3.13.0": { + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-apple-darwin": 1, "aarch64-unknown-linux-gnu": 2}, + "patches": { + "aarch64-apple-darwin": ["common.txt"], + "aarch64-unknown-linux-gnu": ["specific-patch.txt"], + }, + "sha256": {"aarch64-apple-darwin": "deadbeef", "aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-apple-darwin": "prefix", "aarch64-unknown-linux-gnu": "python"}, + "url": { + "aarch64-apple-darwin": ["example.org"], + "aarch64-unknown-linux-gnu": ["something.org", "else.org"], + }, + }, + }) + env.expect.that_dict(py.overrides.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_patches) + def python_test_suite(name): """Create the test suite. From 4e2bb3e4da8d76fc5469f6f5a7e463b88c9bda26 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:12:46 +0900 Subject: [PATCH 51/75] docs --- CHANGELOG.md | 8 +++----- docs/toolchains.md | 16 +++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf9e9cb08..5d56454b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,11 +52,9 @@ A brief description of the categories of changes: * (gazelle) Correctly resolve deps that have top-level module overlap with a gazelle_python.yaml dep module ### Added -* (bzlmod): Toolchain overrides can now be done to fully override anything from - within the TOOL_VERSIONS dict using the new - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.override`, - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override` and - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. +* (bzlmod): Toolchain overrides can now be done using the new + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override` and + {bzl:obj}`python.single_version_platform_override` tag classes. * (rules) Executables provide {obj}`PyExecutableInfo`, which contains executable-specific information useful for packaging an executable or or deriving a new one from the original. diff --git a/docs/toolchains.md b/docs/toolchains.md index 6743318084..b744a9c4f8 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -166,16 +166,14 @@ Python toolchains can be utilized in other bazel rules, such as `genrule()`, by ### Overriding toolchains -Overrides can be done at 3 different levels: -* Overrides affecting all python toolchain versions on all platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.override`. -* Overrides affecting a single toolchain versions on all platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override`. -* Overrides affecting a single toolchain versions on a single platforms - {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. - -The overrides cover the following use-cases: -* Limiting the available toolchains for the entire `bzlmod` transitive graph via {attr}`python.override.available_python_versions`. -* Setting particular `X.Y.Z` python versions when modules request `X.Y` version via {attr}`python.override.minor_mapping`. +One can perform various overrides for the registered toolchains from the root module. For example, the following usecases would be supported using the existing attributes: +* Limiting the available toolchains for the entire `bzlmod` transitive graph + via {attr}`python.override.available_python_versions`. +* Setting particular `X.Y.Z` python versions when modules request `X.Y` version + via {attr}`python.override.minor_mapping`. * Adding custom {attr}`python.single_version_platform_override.coverage_tool`. -* Adding new python versions via {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_override` or {bzl:obj}`@rules_python//python/extensions:python.bzl%python.single_version_platform_override`. +* Adding new python versions via {bzl:obj}`python.single_version_override` or + {bzl:obj}`python.single_version_platform_override`. ## Workspace configuration From f60febdafff18390bafa420ec4fceed89297bf49 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:22:53 +0900 Subject: [PATCH 52/75] ensure that the new python version can be also added with a different override --- CHANGELOG.md | 2 -- docs/toolchains.md | 3 ++- python/private/python.bzl | 16 ++++++---------- tests/python/python_tests.bzl | 23 ++++++++++++++++++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d56454b2a..7c3d6cf982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,6 @@ A brief description of the categories of changes: [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x ### Changed -* (gazelle): Update error messages when unable to resolve a dependency to be - more human-friendly. * (gazelle): Update error messages when unable to resolve a dependency to be more human-friendly. * (flags) The {obj}`--python_version` flag now also returns {obj}`config_common.FeatureFlagInfo`. diff --git a/docs/toolchains.md b/docs/toolchains.md index b744a9c4f8..c31585079e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -164,9 +164,10 @@ Remember to call `use_repo()` to make repos visible to your module: Python toolchains can be utilized in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the {gh-path}`test_current_py_toolchain ` target for an example. -### Overriding toolchains +### Overriding toolchain defaults and adding more toolchains One can perform various overrides for the registered toolchains from the root module. For example, the following usecases would be supported using the existing attributes: + * Limiting the available toolchains for the entire `bzlmod` transitive graph via {attr}`python.override.available_python_versions`. * Setting particular `X.Y.Z` python versions when modules request `X.Y` version diff --git a/python/private/python.bzl b/python/private/python.bzl index c98787f029..5c27a27af9 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -404,24 +404,20 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): if tag.python_version not in available_versions: if not tag.urls or not tag.sha256 or not tag.strip_prefix: fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) - available_versions[tag.python_version] = { - "sha256": {tag.platform: tag.sha256}, - "strip_prefix": {tag.platform: tag.strip_prefix}, - "url": {tag.platform: tag.urls}, - } + available_versions[tag.python_version] = {} if tag.coverage_tool: available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool if tag.patch_strip: - available_versions[tag.python_version]["patch_strip"][tag.platform] = tag.patch_strip + available_versions[tag.python_version].setdefault("patch_strip", {})[tag.platform] = tag.patch_strip if tag.patches: - available_versions[tag.python_version]["patches"][tag.platform] = list(tag.patches) + available_versions[tag.python_version].setdefault("patches", {})[tag.platform] = list(tag.patches) if tag.sha256: - available_versions[tag.python_version]["sha256"][tag.platform] = tag.sha256 + available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 if tag.strip_prefix: - available_versions[tag.python_version]["strip_prefix"][tag.platform] = tag.strip_prefix + available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix if tag.urls: - available_versions[tag.python_version]["url"][tag.platform] = tag.urls + available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls register_all = False for tag in mod.tags.override: diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index afbb5dc20a..988ab2fd2c 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -362,10 +362,22 @@ def _test_add_new_version(env): distutils = None, ), ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.1", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], override = [ _override( base_url = "", - available_python_versions = ["3.12.4", "3.13.0"], + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], minor_mapping = { "3.13": "3.13.0", }, @@ -379,12 +391,21 @@ def _test_add_new_version(env): env.expect.that_collection(py.overrides.default["tool_versions"].keys()).contains_exactly([ "3.12.4", "3.13.0", + "3.13.1", ]) env.expect.that_dict(py.overrides.default["tool_versions"]["3.13.0"]).contains_exactly({ "sha256": {"aarch64-unknown-linux-gnu": "deadbeef"}, "strip_prefix": {"aarch64-unknown-linux-gnu": "prefix"}, "url": {"aarch64-unknown-linux-gnu": ["example.org"]}, }) + env.expect.that_dict(py.overrides.default["tool_versions"]["3.13.1"]).contains_exactly({ + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-unknown-linux-gnu": 2}, + "patches": {"aarch64-unknown-linux-gnu": ["specific-patch.txt"]}, + "sha256": {"aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "python"}, + "url": {"aarch64-unknown-linux-gnu": ["something.org", "else.org"]}, + }) env.expect.that_dict(py.overrides.minor_mapping).contains_exactly({ "3.13": "3.13.0", }) From 0a40f885e57032d0212f185c82e5d9219990eb44 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:25:01 +0900 Subject: [PATCH 53/75] add a reference to a ticket --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3d6cf982..90f8106211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ A brief description of the categories of changes: * (bzlmod): Toolchain overrides can now be done using the new {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override` and {bzl:obj}`python.single_version_platform_override` tag classes. + See [#2081](https://github.com/bazelbuild/rules_python/issues/2081). * (rules) Executables provide {obj}`PyExecutableInfo`, which contains executable-specific information useful for packaging an executable or or deriving a new one from the original. From 83d2c64e4a9b730fb51b26a9d1f9d808b0a3f584 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:26:05 +0900 Subject: [PATCH 54/75] fix the examples/bzlmod --- examples/bzlmod/MODULE.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index ff74790075..39c842643d 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -64,6 +64,7 @@ python.single_version_override( # The user can specify patches to be applied to all interpreters, they will # be applied before the platform specific patches are applied. patches = [], + python_version = "3.10.2", sha256 = { "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", @@ -72,7 +73,6 @@ python.single_version_override( "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", }, urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], - version = "3.10.2", ) # Or a single platform. This can be used in combination with the @@ -83,9 +83,9 @@ python.single_version_platform_override( patch_strip = 1, patches = [], platform = "aarch64-apple-darwin", + python_version = "3.10.2", sha256 = "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], - version = "3.10.2", ) # You only need to load this repositories if you are using multiple Python versions. From bd06e6dfec5f652dd7155554548788f9f1fac8c5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:28:32 +0900 Subject: [PATCH 55/75] improve example docs --- examples/bzlmod/MODULE.bazel | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 39c842643d..502c6fa712 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -22,7 +22,7 @@ bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf" python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( configure_coverage_tool = True, - # Only set when you have mulitple toolchain versions. + # Only set when you have multiple toolchain versions. is_default = True, python_version = "3.9", ) @@ -49,8 +49,6 @@ python.override( ], # Also override the `minor_mapping` so that when the modules specify a particular # `3.X` version, we decide what gets used. - # - # TODO @aignas 2024-09-01: calculate the default based on available python versions. minor_mapping = { "3.10": "3.10.9", "3.11": "3.11.8", @@ -61,8 +59,7 @@ python.override( # Or the sources that the toolchains come from for all platforms python.single_version_override( patch_strip = 1, - # The user can specify patches to be applied to all interpreters, they will - # be applied before the platform specific patches are applied. + # The user can specify patches to be applied to all interpreters. patches = [], python_version = "3.10.2", sha256 = { @@ -77,8 +74,8 @@ python.single_version_override( # Or a single platform. This can be used in combination with the # `single_version_override` and `single_version_platform_override` will be -# applied after `single_version_override`. -# TODO @aignas 2024-08-27: implement that +# applied after `single_version_override`. Any values present in this override +# will overwrite the values set by the `single_version_override` python.single_version_platform_override( patch_strip = 1, patches = [], From 88a1d5e33ce669f83f4638744803045c83082eea Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 00:35:00 +0900 Subject: [PATCH 56/75] cleanup --- CHANGELOG.md | 3 ++- docs/BUILD.bazel | 3 +-- python/private/python.bzl | 7 ++----- tests/python/python_tests.bzl | 5 +++++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f8106211..f7a9d21dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,8 @@ A brief description of the categories of changes: have it installed. ### Removed -* Nothing yet +* (toolchains): Removed accidentally exposed `http_archive` symbol from + `python/repositories.bzl`. ## [0.35.0] - 2024-08-15 diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 92a389059a..e211c13344 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -89,6 +89,7 @@ sphinx_stardocs( "//python:py_runtime_bzl", "//python:py_runtime_info_bzl", "//python:py_test_bzl", + "//python:repositories_bzl", "//python/cc:py_cc_toolchain_info_bzl", "//python/entry_points:py_console_script_binary_bzl", "//python/private:py_exec_tools_info_bzl", @@ -97,8 +98,6 @@ sphinx_stardocs( "//python/private/common:py_library_rule_bazel_bzl", "//python/private/common:py_runtime_rule_bzl", "//python/private/common:py_test_rule_bazel_bzl", - # FIXME @aignas 2024-08-26: should we have this here? - "//python:repositories_bzl", ] + ([ # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension "//python/extensions:python_bzl", diff --git a/python/private/python.bzl b/python/private/python.bzl index 5c27a27af9..b193c828df 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -44,7 +44,7 @@ def _parse_version(version): build = build, ) -def parse_mods(mctx, *, logger, debug = False, fail = fail): +def parse_mods(*, mctx, logger, debug = False, fail = fail): """parse_mods returns a struct with parsed tag class content. Args: @@ -244,10 +244,7 @@ def _python_impl(mctx): else: debug_info = None - py = parse_mods( - mctx, - logger = logger, - ) + py = parse_mods(mctx = mctx, logger = logger, debug = debug_info != None) for toolchain in py.toolchains: # Ensure that we pass the full version here. diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 988ab2fd2c..a26037428c 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -495,6 +495,11 @@ def _test_add_patches(env): _tests.append(_test_add_patches) +# TODO @aignas 2024-09-03: add failure tests: +# * validate the python_version in overrides +# * incorrect platform failure +# * missing python_version failure + def python_test_suite(name): """Create the test suite. From 581191d4c99c38fb29e6699f408661cdb0a2ac05 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:47:12 +0900 Subject: [PATCH 57/75] remove TODOs --- python/private/python.bzl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index b193c828df..12ca0946a8 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -584,7 +584,6 @@ The Python version, in `major.minor` or `major.minor.patch` format, e.g }, ) -# TODO @aignas 2024-09-01: Is the name good enough, should this be something like `python.configure`? _override = tag_class( doc = """Tag class used to override defaults and behaviour of the module extension. @@ -616,8 +615,6 @@ can result in spurious build failures. """, mandatory = False, ), - # TODO @aignas 2024-09-01: could be replaced by an attribute on the `toolchain` with - # a boolean attribute `is_default_minor`. "minor_mapping": attr.string_dict( mandatory = False, doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", @@ -629,9 +626,6 @@ can result in spurious build failures. ), ) -# TODO @aignas 2024-09-01: should the attributes within this be incorporated into the `python.toolchain` call? -# It does sound like the override being done in a separate method is not something that is super ergonomic, because -# then the ordering needs to be decided and we may need to also think about what to do with the coverage attribute. _single_version_override = tag_class( doc = """Override single python version URLs and patches for all platforms. From c8b0833cc9a4207690e18b04331a5ac6a94a1bd7 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:59:13 +0900 Subject: [PATCH 58/75] refactor: add a semver function to consolidate code --- python/private/BUILD.bazel | 7 ++++++ python/private/pypi/BUILD.bazel | 1 + python/private/pypi/extension.bzl | 17 ++------------ python/private/python.bzl | 19 +++------------- python/private/semver.bzl | 37 +++++++++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 python/private/semver.bzl diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index c755bd9604..7cbd057c0c 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -135,6 +135,7 @@ bzl_library( ":python_repositories_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", + ":semver_bzl", ":toolchains_repo_bzl", ":util_bzl", "@bazel_features//:features", @@ -294,6 +295,12 @@ bzl_library( srcs = ["repo_utils.bzl"], ) +bzl_library( + name = "semver_bzl", + srcs = ["semver.bzl"], + visibility = ["//:__subpackages__"], +) + bzl_library( name = "stamp_bzl", srcs = ["stamp.bzl"], diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 3b11dbe7f8..65bdc4a011 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -70,6 +70,7 @@ bzl_library( "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_label_bzl", + "//python/private:semver_bzl", "@bazel_features//:features", ] + [ "@pythons_hub//:interpreters_bzl", diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1bc8f15149..77a477899e 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -19,6 +19,7 @@ load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_L load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private:semver.bzl", "semver") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") @@ -32,22 +33,8 @@ load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_library.bzl", "whl_library") load(":whl_repo_name.bzl", "whl_repo_name") -def _parse_version(version): - major, _, version = version.partition(".") - minor, _, version = version.partition(".") - patch, _, version = version.partition(".") - build, _, version = version.partition(".") - - return struct( - # use semver vocabulary here - major = major, - minor = minor, - patch = patch, # this is called `micro` in the Python interpreter versioning scheme - build = build, - ) - def _major_minor_version(version): - version = _parse_version(version) + version = semver(version) return "{}.{}".format(version.major, version.minor) def _whl_mods_impl(mctx): diff --git a/python/private/python.bzl b/python/private/python.bzl index 12ca0946a8..e2be10a93b 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -21,6 +21,7 @@ load(":full_version.bzl", "full_version") load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") +load(":semver.bzl", "semver") load(":text_util.bzl", "render") load(":toolchains_repo.bzl", "multi_toolchain_aliases") load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") @@ -30,20 +31,6 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def _parse_version(version): - major, _, version = version.partition(".") - minor, _, version = version.partition(".") - patch, _, version = version.partition(".") - build, _, version = version.partition(".") - - return struct( - # use semver vocabulary here - major = major, - minor = minor, - patch = patch, # this is called `micro` in the Python interpreter versioning scheme - build = build, - ) - def parse_mods(*, mctx, logger, debug = False, fail = fail): """parse_mods returns a struct with parsed tag class content. @@ -437,10 +424,10 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): if tag.minor_mapping: for minor_version, full_version in tag.minor_mapping.items(): - parsed = _parse_version(minor_version) + parsed = semver(minor_version) if parsed.patch or parsed.build: fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) - parsed = _parse_version(full_version) + parsed = semver(full_version) if not parsed.patch: fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) diff --git a/python/private/semver.bzl b/python/private/semver.bzl new file mode 100644 index 0000000000..65a608c949 --- /dev/null +++ b/python/private/semver.bzl @@ -0,0 +1,37 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"A semver version parser" + +def semver(version): + """Parse the semver version and return the values as a struct. + + Args: + version: {type}`str` the version string + + Returns: + A {type}`struct` with `major`, `minor`, `patch` and `build` attributes. + """ + major, _, version = version.partition(".") + minor, _, version = version.partition(".") + patch, _, version = version.partition(".") + build, _, version = version.partition("+") + + return struct( + # use semver vocabulary here + major = major, + minor = minor, + patch = patch, # this is called `micro` in the Python interpreter versioning scheme + build = build, + ) From 0aac2001fd588fb8bdbb2d86f629da0e8da8796a Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:05:29 +0900 Subject: [PATCH 59/75] improve the docs --- python/private/python.bzl | 8 ++-- python/private/python_repositories.bzl | 44 +++++++++++----------- sphinxdocs/inventories/bazel_inventory.txt | 3 +- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index e2be10a93b..78610cdddb 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -35,10 +35,10 @@ def parse_mods(*, mctx, logger, debug = False, fail = fail): """parse_mods returns a struct with parsed tag class content. Args: - mctx: module_ctx. - logger: logger. - debug: whether to add extra diagnostic information. - fail: fail. + mctx: {type}`module_ctx`. + logger: logger for diagnostic output. + debug: whether to add extra diagnostic information about the configured toolchains. + fail: {type}`function` the fail for failure handling. Returns: a struct with attributes diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index ed6dbba52a..f9c4480dd7 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -82,12 +82,12 @@ def is_standalone_interpreter(rctx, python_interpreter_path, *, logger = None): """Query a python interpreter target for whether or not it's a rules_rust provided toolchain Args: - rctx (repository_ctx): The repository rule's context object. - python_interpreter_path (path): A path representing the interpreter. - logger: Optional logger to use for operations. + rctx: {type}`repository_ctx` The repository rule's context object. + python_interpreter_path: {type}`path` A path representing the interpreter. + logger: {type}`struct` Optional logger to use for operations. Returns: - bool: Whether or not the target is from a rules_python generated toolchain. + {type}`bool`: Whether or not the target is from a rules_python generated toolchain. """ # Only update the location when using a hermetic toolchain. @@ -587,18 +587,20 @@ def python_register_toolchains( root module one can see the docs for the {rule}`python` extension. Args: - name: base name for all created repos, like "python38". - python_version: the Python version. - distutils: see the {attr}`python_repository.distutils`. - distutils_content: see the {attr}`python_repository.distutils_content`. - register_toolchains: Whether or not to register the downloaded toolchains. - register_coverage_tool: Whether or not to register the downloaded coverage tool to the toolchains. - NOTE: Coverage support using the toolchain is only supported in Bazel 6 and higher. - - set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. - tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults - in python/versions.bzl will be used. - **kwargs: passed to each {rule}`python_repositories` call. + name: {type}`str` base name for all created repos, like "python38". + python_version: {type}`str` the Python version. + distutils: {type}`Label` see the {attr}`python_repository.distutils`. + distutils_content: {type}`str` see the {attr}`python_repository.distutils_content`. + register_toolchains: {type}`bool` Whether or not to register the downloaded toolchains. + register_coverage_tool: {type}`bool` Whether or not to register the + downloaded coverage tool to the toolchains. + set_python_version_constraint: {type}`bool` When set to true, + target_compatible_with for the toolchains will include a version + constraint. + tool_versions: {type}`dict[str, dict[str, dict[str]]]` contains a + mapping of version with SHASUM and platform info. If not supplied, + the defaults in `python/versions.bzl` will be used. + **kwargs: passed to each {obj}`python_repository` call. """ tool_versions = tool_versions or TOOL_VERSIONS @@ -723,11 +725,11 @@ def python_register_multi_toolchains( """Convenience macro for registering multiple Python toolchains. Args: - name: base name for each name in python_register_toolchains call. - python_versions: the Python version. - default_version: the default Python version. If not set, the first version in - python_versions is used. - **kwargs: passed to each {rule}`python_register_toolchains` call. + name: {type}`str` base name for each name in python_register_toolchains call. + python_versions: {type}`list[str]` the Python version. + default_version: {type}`str` the default Python version. If not set, + the first version in python_versions is used. + **kwargs: passed to each {obj}`python_register_toolchains` call. """ if len(python_versions) == 0: fail("python_versions must not be empty") diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index 4441b6f24f..8aa76e6981 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -80,7 +80,8 @@ Name bzl:type 1 concepts/labels#target-names - CcInfo bzl:provider 1 rules/lib/providers/CcInfo - CcInfo.linking_context bzl:provider-field 1 rules/lib/providers/CcInfo#linking_context - ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - -module_ctx bzl:obj 1 rules/lib/builtins/module_ctx - +repository_ctx bzl:type 1 rules/lib/builtins/repository_ctx - +module_ctx bzl:type 1 rules/lib/builtins/module_ctx - module_ctx.download bzl:function 1 rules/lib/builtins/module_ctx#download - module_ctx.download_and_extract bzl:function 1 rules/lib/builtins/module_ctx#download_and_extract - module_ctx.execute bzl:function 1 rules/lib/builtins/module_ctx#execute - From 246febee5491fe7ef055fbce1f1953dfd970c8f9 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:08:06 +0900 Subject: [PATCH 60/75] update the lock file --- examples/bzlmod/MODULE.bazel.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index d45c6a2300..a89b4504ae 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "pRCr7LSWdBgrFvB+qQ9IabLPuxA/nEMgCMxZLjC3S48=", + "bzlTransitiveDigest": "DZkhdC8EeYU4Q6NdWrZRLPThiDghH834VDrPw5srRxo=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "5jsCtckcJY3sx6Qc3tD82EHZH3Aj9pFO7BtoUBLfUfw=", + "bzlTransitiveDigest": "F+IySUOJQOeR/+3CX+k7vbGmuAoyh2gXToh/7Q9+LM4=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", From 266e9ab3f9aa758f2e7e9a6efd33af0b77914edb Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:40:35 +0900 Subject: [PATCH 61/75] add path to inventory --- python/private/python.bzl | 4 ---- sphinxdocs/inventories/bazel_inventory.txt | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 78610cdddb..c9df5df818 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -518,10 +518,6 @@ can override settings for any python toolchain available. This relies on the documented module traversal from the {obj}`module_ctx.modules`. ::: -:::{todo} -fix the `module_ctx.modules` reference so that we can link back to the bazel page. -::: - :::{tip} In order to use a different name than the above, you can use the following `MODULE.bazel` syntax: diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index 8aa76e6981..5c53dcd863 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -98,3 +98,4 @@ module_ctx.report_progress bzl:function 1 rules/lib/builtins/module_ctx#report_p module_ctx.root_module_has_non_dev_dependency bzl:function 1 rules/lib/builtins/module_ctx#root_module_has_non_dev_dependency - module_ctx.watch bzl:function 1 rules/lib/builtins/module_ctx#watch - module_ctx.which bzl:function 1 rules/lib/builtins/module_ctx#which - +path bzl:type 1 rules/lib/builtins/path - From 09b9cd6d23dc272db0f356e71cd874a6fbd820a9 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:54:43 +0900 Subject: [PATCH 62/75] fix integration tests --- python/private/python.bzl | 5 ++++- .../ignore_root_user_error/bzlmod_test.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index c9df5df818..0a6636fa7d 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -214,7 +214,10 @@ def parse_mods(*, mctx, logger, debug = False, fail = fail): name = t.name, python_version = t.python_version, register_coverage_tool = t.register_coverage_tool, - debug = {"module": t.module} if debug else None, + debug = { + "ignore_root_user_error": ignore_root_user_error, + "module": t.module, + } if debug else None, ) for t in toolchains ], diff --git a/tests/integration/ignore_root_user_error/bzlmod_test.py b/tests/integration/ignore_root_user_error/bzlmod_test.py index 98715b32ec..6c0862df45 100644 --- a/tests/integration/ignore_root_user_error/bzlmod_test.py +++ b/tests/integration/ignore_root_user_error/bzlmod_test.py @@ -28,8 +28,16 @@ def test_toolchains(self): debug_info = json.loads(debug_path.read_bytes()) expected = [ - {"ignore_root_user_error": True, "name": "python_3_11"}, - {"ignore_root_user_error": True, "name": "python_3_10"}, + { + "ignore_root_user_error": True, + "module": {"is_root": False, "name": "submodule"}, + "name": "python_3_11", + }, + { + "ignore_root_user_error": True, + "module": {"is_root": True, "name": "ignore_root_user_error"}, + "name": "python_3_10", + }, ] self.assertCountEqual(debug_info["toolchains_registered"], expected) From 93684eb5ed585290992a9524a117a79cbbfd4cc7 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:12:00 +0900 Subject: [PATCH 63/75] register_all_versions fix and fixup tests --- python/private/python.bzl | 6 +- tests/python/python_tests.bzl | 106 ++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 0a6636fa7d..3fefe4ae7f 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -420,9 +420,7 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): for v in tag.available_python_versions }) - if tag.register_all_versions and mod.name != "rules_python": - fail("This override can only be used by 'rules_python'") - elif tag.register_all_versions: + if tag.register_all_versions: register_all = True if tag.minor_mapping: @@ -447,7 +445,7 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): # FIXME @aignas 2024-08-30: this is technically not correct registrations.extend([ _create_toolchain_attrs_struct(python_version = v) - for v in available_versions.keys() + for v in available_versions.keys() + overrides.minor_mapping.keys() if v not in seen_versions ]) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index a26037428c..9bfa02af21 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -78,15 +78,49 @@ def _override( ) def _single_version_override( - **kwargs): + python_version = "", + sha256 = {}, + urls = [], + patch_strip = 0, + patches = [], + strip_prefix = "python", + distutils_content = "", + distutils = None): + if not python_version: + fail("missing mandatory args: python_version ({})".format(python_version)) + return struct( - **kwargs + python_version = python_version, + sha256 = sha256, + urls = urls, + patch_strip = patch_strip, + patches = patches, + strip_prefix = strip_prefix, + distutils_content = distutils_content, + distutils = distutils, ) def _single_version_platform_override( - **kwargs): + coverage_tool = None, + patch_strip = 0, + patches = [], + platform = "", + python_version = "", + sha256 = "", + strip_prefix = "python", + urls = []): + if not platform or not python_version: + fail("missing mandatory args: platform ({}) and python_version ({})".format(platform, python_version)) + return struct( - **kwargs + sha256 = sha256, + urls = urls, + strip_prefix = strip_prefix, + platform = platform, + coverage_tool = coverage_tool, + python_version = python_version, + patch_strip = patch_strip, + patches = patches, ) def _test_default(env): @@ -283,6 +317,7 @@ def _test_first_occurance_of_the_toolchain_wins(env): # configuring something else. register_coverage_tool = False, debug = { + "ignore_root_user_error": False, "module": struct(is_root = True, name = "my_module"), }, ) @@ -291,6 +326,7 @@ def _test_first_occurance_of_the_toolchain_wins(env): python_version = "3.11", register_coverage_tool = False, debug = { + "ignore_root_user_error": False, "module": struct(is_root = False, name = "rules_python"), }, ) @@ -419,6 +455,68 @@ def _test_add_new_version(env): _tests.append(_test_add_new_version) +def _test_register_all_versions(env): + py = parse_mods( + mctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org"], + platform = "aarch64-unknown-linux-gnu", + python_version = "3.13.1", + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + minor_mapping = {"3.12": "3.12.4", "3.13": "3.13.0"}, + register_all_versions = True, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.overrides.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + "3.13.1", + ]) + env.expect.that_dict(py.overrides.minor_mapping).contains_exactly({ + "3.12": "3.12.4", + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = name, + python_version = version, + register_coverage_tool = False, + ) + for name, version in { + "python_3_12": "3.12", + "python_3_12_4": "3.12.4", + "python_3_13": "3.13", + "python_3_13_0": "3.13.0", + "python_3_13_1": "3.13.1", + }.items() + ]) + +_tests.append(_test_register_all_versions) + def _test_add_patches(env): py = parse_mods( mctx = _mock_mctx( From 2f892345f3309ec304493599c3722e9c1bf6af39 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:14:33 +0900 Subject: [PATCH 64/75] improve the docs --- python/private/python.bzl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 3fefe4ae7f..b8df4c97cc 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -578,7 +578,17 @@ _override = tag_class( { "available_python_versions": attr.string_list( mandatory = False, - doc = "The list of available python tool versions to use. Must be in `X.Y.Z` format.", + doc = """\ +The list of available python tool versions to use. Must be in `X.Y.Z` format. +If the unknown version given the processing of the extension will fail - all of +the versions in the list have to be defined with +{obj}`python.single_version_override` or +{obj}`python.single_version_platform_override` before they are used in this +list. + +This attribute is usually used in order to ensure that no unexpected transitive +dependencies are introduced. +""", ), "base_url": attr.string( mandatory = False, From 6cfd4835ebd9c860974bc8f70dea980eb3f2cc30 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:05:43 +0900 Subject: [PATCH 65/75] fixup tests --- tests/integration/ignore_root_user_error/bzlmod_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/ignore_root_user_error/bzlmod_test.py b/tests/integration/ignore_root_user_error/bzlmod_test.py index 6c0862df45..1283415987 100644 --- a/tests/integration/ignore_root_user_error/bzlmod_test.py +++ b/tests/integration/ignore_root_user_error/bzlmod_test.py @@ -31,12 +31,12 @@ def test_toolchains(self): { "ignore_root_user_error": True, "module": {"is_root": False, "name": "submodule"}, - "name": "python_3_11", + "name": "python_3_10", }, { "ignore_root_user_error": True, "module": {"is_root": True, "name": "ignore_root_user_error"}, - "name": "python_3_10", + "name": "python_3_11", }, ] self.assertCountEqual(debug_info["toolchains_registered"], expected) From def721a5d044256b4f0321e64d1d7db841cd192f Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:23:41 +0900 Subject: [PATCH 66/75] fix merge conflicts --- sphinxdocs/inventories/bazel_inventory.txt | 97 +++++++++++++++------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index 5c53dcd863..ee507d1ccc 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -3,9 +3,21 @@ # Version: 7.3.0 # The remainder of this file is compressed using zlib Action bzl:type 1 rules/lib/Action - +CcInfo bzl:provider 1 rules/lib/providers/CcInfo - +CcInfo.linking_context bzl:provider-field 1 rules/lib/providers/CcInfo#linking_context - +ExecutionInfo bzl:type 1 rules/lib/providers/ExecutionInfo - File bzl:type 1 rules/lib/File - Label bzl:type 1 rules/lib/Label - +Name bzl:type 1 concepts/labels#target-names - +RunEnvironmentInfo bzl:type 1 rules/lib/providers/RunEnvironmentInfo - Target bzl:type 1 rules/lib/builtins/Target - +ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - +attr.bool bzl:type 1 rules/lib/toplevel/attr#bool - +attr.int bzl:type 1 rules/lib/toplevel/attr#int - +attr.label bzl:type 1 rules/lib/toplevel/attr#label - +attr.label_list bzl:type 1 rules/lib/toplevel/attr#label_list - +attr.string bzl:type 1 rules/lib/toplevel/attr#string - +attr.string_list bzl:type 1 rules/lib/toplevel/attr#string_list - bool bzl:type 1 rules/lib/bool - callable bzl:type 1 rules/lib/core/function - config_common.FeatureFlagInfo bzl:type 1 rules/lib/toplevel/config_common#FeatureFlagInfo - @@ -44,17 +56,28 @@ ctx.toolchains bzl:obj 1 rules/lib/builtins/ctx#toolchains - ctx.var bzl:obj 1 rules/lib/builtins/ctx#var - ctx.version_file bzl:obj 1 rules/lib/builtins/ctx#version_file - ctx.workspace_name bzl:obj 1 rules/lib/builtins/ctx#workspace_name - -int bzl:type 1 rules/lib/int - depset bzl:type 1 rules/lib/depset - dict bzl:type 1 rules/lib/dict - +int bzl:type 1 rules/lib/int - label bzl:type 1 concepts/labels - -attr.bool bzl:type 1 rules/lib/toplevel/attr#bool - -attr.int bzl:type 1 rules/lib/toplevel/attr#int - -attr.label bzl:type 1 rules/lib/toplevel/attr#label - -attr.label_list bzl:type 1 rules/lib/toplevel/attr#label_list - -attr.string bzl:type 1 rules/lib/toplevel/attr#string - -attr.string_list bzl:type 1 rules/lib/toplevel/attr#string_list - list bzl:type 1 rules/lib/list - +module_ctx bzl:type 1 rules/lib/builtins/module_ctx - +module_ctx.download bzl:function 1 rules/lib/builtins/module_ctx#download - +module_ctx.download_and_extract bzl:function 1 rules/lib/builtins/module_ctx#download_and_extract - +module_ctx.execute bzl:function 1 rules/lib/builtins/module_ctx#execute - +module_ctx.extension_metadata bzl:function 1 rules/lib/builtins/module_ctx#extension_metadata - +module_ctx.extract bzl:function 1 rules/lib/builtins/module_ctx#extract - +module_ctx.file bzl:function 1 rules/lib/builtins/module_ctx#file - +module_ctx.getenv bzl:function 1 rules/lib/builtins/module_ctx#getenv - +module_ctx.is_dev_dependency bzl:obj 1 rules/lib/builtins/module_ctx#is_dev_dependency - +module_ctx.modules bzl:obj 1 rules/lib/builtins/module_ctx#modules - +module_ctx.os bzl:obj 1 rules/lib/builtins/module_ctx#os - +module_ctx.path bzl:function 1 rules/lib/builtins/module_ctx#path - +module_ctx.read bzl:function 1 rules/lib/builtins/module_ctx#read - +module_ctx.report_progress bzl:function 1 rules/lib/builtins/module_ctx#report_progress - +module_ctx.root_module_has_non_dev_dependency bzl:function 1 rules/lib/builtins/module_ctx#root_module_has_non_dev_dependency - +module_ctx.watch bzl:function 1 rules/lib/builtins/module_ctx#watch - +module_ctx.which bzl:function 1 rules/lib/builtins/module_ctx#which - native.existing_rule bzl:function 1 rules/lib/toplevel/native#existing_rule - native.existing_rules bzl:function 1 rules/lib/toplevel/native#existing_rules - native.exports_files bzl:function 1 rules/lib/toplevel/native#exports_files - @@ -66,6 +89,39 @@ native.package_name bzl:function 1 rules/lib/toplevel/native#package_name - native.package_relative_label bzl:function 1 rules/lib/toplevel/native#package_relative_label - native.repo_name bzl:function 1 rules/lib/toplevel/native#repo_name - native.repository_name bzl:function 1 rules/lib/toplevel/native#repository_name - +path bzl:type 1 rules/lib/builtins/path - +path.basename bzl:obj 1 rules/lib/builtins/path#basename +path.dirname bzl:obj 1 rules/lib/builtins/path#dirname +path.exists bzl:obj 1 rules/lib/builtins/path#exists +path.get_child bzl:function 1 rules/lib/builtins/path#get_child +path.is_dir bzl:obj 1 rules/lib/builtins/path#is_dir +path.readdir bzl:function 1 rules/lib/builtins/path#readdir +path.realpath bzl:obj 1 rules/lib/builtins/path#realpath +repository_ctx bzl:type 1 rules/lib/builtins/repository_ctx - +repository_ctx.attr bzl:obj 1 rules/lib/builtins/repository_ctx#attr +repository_ctx.delete bzl:function 1 rules/lib/builtins/repository_ctx#delete +repository_ctx.download bzl:function 1 rules/lib/builtins/repository_ctx#download +repository_ctx.download_and_extract bzl:function 1 rules/lib/builtins/repository_ctx#download_and_extract +repository_ctx.execute bzl:function 1 rules/lib/builtins/repository_ctx#execute +repository_ctx.extract bzl:function 1 rules/lib/builtins/repository_ctx#extract +repository_ctx.file bzl:function 1 rules/lib/builtins/repository_ctx#file +repository_ctx.getenv bzl:function 1 rules/lib/builtins/repository_ctx#getenv +repository_ctx.name bzl:obj 1 rules/lib/builtins/repository_ctx#name +repository_ctx.os bzl:obj 1 rules/lib/builtins/repository_ctx#os +repository_ctx.patch bzl:function 1 rules/lib/builtins/repository_ctx#patch +repository_ctx.path bzl:obj 1 rules/lib/builtins/repository_ctx#path +repository_ctx.read bzl:function 1 rules/lib/builtins/repository_ctx#read +repository_ctx.report_progress bzl:function 1 rules/lib/builtins/repository_ctx#report_progress +repository_ctx.symlink bzl:function 1 rules/lib/builtins/repository_ctx#symlink +repository_ctx.template bzl:function 1 rules/lib/builtins/repository_ctx#template +repository_ctx.watch bzl:function 1 rules/lib/builtins/repository_ctx#watch +repository_ctx.watch_tree bzl:function 1 rules/lib/builtins/repository_ctx#watch_tree +repository_ctx.which bzl:function 1 rules/lib/builtins/repository_ctx#which +repository_ctx.workspace_root bzl:obj 1 rules/lib/builtins/repository_ctx#workspace_root +repository_os bzl:type 1 rules/lib/builtins/repository_os - +repository_os.arch bzl:obj 1 rules/lib/builtins/repository_os#arch +repository_os.environ bzl:obj 1 rules/lib/builtins/repository_os#environ +repository_os.name bzl:obj 1 rules/lib/builtins/repository_os#name runfiles bzl:type 1 rules/lib/builtins/runfiles - runfiles.empty_filenames bzl:type 1 rules/lib/builtins/runfiles#empty_filenames - runfiles.files bzl:type 1 rules/lib/builtins/runfiles#files - @@ -75,27 +131,8 @@ runfiles.root_symlinks bzl:type 1 rules/lib/builtins/runfiles#root_symlinks - runfiles.symlinks bzl:type 1 rules/lib/builtins/runfiles#symlinks - str bzl:type 1 rules/lib/string - struct bzl:type 1 rules/lib/builtins/struct - +testing bzl:obj 1 rules/lib/toplevel/testing - +testing.ExecutionInfo bzl:function 1 rules/lib/toplevel/testing#ExecutionInfo - +testing.TestEnvironment bzl:function 1 rules/lib/toplevel/testing#TestEnvironment - +testing.analysis_test bzl:rule 1 rules/lib/toplevel/testing#analysis_test - toolchain_type bzl:type 1 ules/lib/builtins/toolchain_type.html - -Name bzl:type 1 concepts/labels#target-names - -CcInfo bzl:provider 1 rules/lib/providers/CcInfo - -CcInfo.linking_context bzl:provider-field 1 rules/lib/providers/CcInfo#linking_context - -ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - -repository_ctx bzl:type 1 rules/lib/builtins/repository_ctx - -module_ctx bzl:type 1 rules/lib/builtins/module_ctx - -module_ctx.download bzl:function 1 rules/lib/builtins/module_ctx#download - -module_ctx.download_and_extract bzl:function 1 rules/lib/builtins/module_ctx#download_and_extract - -module_ctx.execute bzl:function 1 rules/lib/builtins/module_ctx#execute - -module_ctx.extension_metadata bzl:function 1 rules/lib/builtins/module_ctx#extension_metadata - -module_ctx.extract bzl:function 1 rules/lib/builtins/module_ctx#extract - -module_ctx.file bzl:function 1 rules/lib/builtins/module_ctx#file - -module_ctx.getenv bzl:function 1 rules/lib/builtins/module_ctx#getenv - -module_ctx.is_dev_dependency bzl:obj 1 rules/lib/builtins/module_ctx#is_dev_dependency - -module_ctx.modules bzl:obj 1 rules/lib/builtins/module_ctx#modules - -module_ctx.os bzl:obj 1 rules/lib/builtins/module_ctx#os - -module_ctx.path bzl:function 1 rules/lib/builtins/module_ctx#path - -module_ctx.read bzl:function 1 rules/lib/builtins/module_ctx#read - -module_ctx.report_progress bzl:function 1 rules/lib/builtins/module_ctx#report_progress - -module_ctx.root_module_has_non_dev_dependency bzl:function 1 rules/lib/builtins/module_ctx#root_module_has_non_dev_dependency - -module_ctx.watch bzl:function 1 rules/lib/builtins/module_ctx#watch - -module_ctx.which bzl:function 1 rules/lib/builtins/module_ctx#which - -path bzl:type 1 rules/lib/builtins/path - From 80a81aa182c791fd854808b16ce704a72485cb23 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:35:15 +0900 Subject: [PATCH 67/75] more cleanup --- python/private/toolchains_repo.bzl | 3 --- python/versions.bzl | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index df16fb8cf7..c4343caad0 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -56,9 +56,6 @@ def python_toolchain_build_file_content( build_content: Text containing toolchain definitions """ - # We create a list of toolchain content from iterating over - # the enumeration of PLATFORMS. We enumerate PLATFORMS in - # order to get us an index to increment the increment. return "\n\n".join([ """\ py_toolchain_suite( diff --git a/python/versions.bzl b/python/versions.bzl index e1787b7b4a..1898359f0c 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -717,5 +717,6 @@ def gen_python_config_settings(name = ""): for platform in PLATFORMS.keys(): native.config_setting( name = "{name}{platform}".format(name = name, platform = platform), + flag_values = PLATFORMS[platform].flag_values, constraint_values = PLATFORMS[platform].compatible_with, ) From 8ad05af8dbaffead61a07db67667c89d4bf3f136 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:58:20 +0900 Subject: [PATCH 68/75] infer MINOR_MAPPING automatically when registering toolchains --- python/private/python.bzl | 28 +++++++++++++++++++++++++--- python/private/semver.bzl | 3 +-- tests/python/python_tests.bzl | 16 +++++++--------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index b8df4c97cc..a33e406317 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -15,7 +15,7 @@ "Python toolchain module extensions for use with bzlmod." load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") load(":python_repositories.bzl", "python_register_toolchains") @@ -69,7 +69,7 @@ def parse_mods(*, mctx, logger, debug = False, fail = fail): # overrides that can be changed by the root module overrides = struct( kwargs = {}, - minor_mapping = dict(MINOR_MAPPING), + minor_mapping = {}, default = { "base_url": DEFAULT_RELEASE_BASE_URL, "tool_versions": { @@ -441,6 +441,17 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): break + if not overrides.minor_mapping: + versions = {} + for version_string in available_versions: + v = semver(version_string) + versions.setdefault("{}.{}".format(v.major, v.minor), []).append((int(v.patch), version_string)) + + overrides.minor_mapping.update({ + major_minor: max(subset)[1] + for major_minor, subset in versions.items() + }) + if register_all: # FIXME @aignas 2024-08-30: this is technically not correct registrations.extend([ @@ -611,7 +622,18 @@ can result in spurious build failures. ), "minor_mapping": attr.string_dict( mandatory = False, - doc = "The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up toolchains.", + doc = """\ +The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up +toolchains. It defaults to the interpreter with the highest available patch +version for each minor version. For example if one registers `3.10.3`, `3.10.4` +and `3.11.4` then the default for the `minor_mapping` dict will be: +```starlark +{ + "3.10": "3.10.4", + "3.11": "3.11.4", +} +``` +""", default = {}, ), "register_all_versions": attr.bool(default = False, doc = "Add all versions"), diff --git a/python/private/semver.bzl b/python/private/semver.bzl index 65a608c949..3f495a75f0 100644 --- a/python/private/semver.bzl +++ b/python/private/semver.bzl @@ -25,8 +25,7 @@ def semver(version): """ major, _, version = version.partition(".") minor, _, version = version.partition(".") - patch, _, version = version.partition(".") - build, _, version = version.partition("+") + patch, _, build = version.partition("+") return struct( # use semver vocabulary here diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 9bfa02af21..b04bdce89e 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -15,6 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python:versions.bzl", "MINOR_MAPPING") load("//python/private:python.bzl", _parse_mods = "parse_mods") # buildifier: disable=bzl-visibility _tests = [] @@ -130,13 +131,10 @@ def _test_default(env): ), ) - env.expect.that_collection(py.overrides.minor_mapping.keys()).contains_exactly([ - "3.10", - "3.11", - "3.12", - "3.8", - "3.9", - ]) + # The value there should be consistent in bzlmod with the automatically + # calculated value Please update the MINOR_MAPPING in //python:versions.bzl + # when this part starts failing. + env.expect.that_dict(py.overrides.minor_mapping).contains_exactly(MINOR_MAPPING) env.expect.that_collection(py.overrides.kwargs).has_size(0) env.expect.that_collection(py.overrides.default.keys()).contains_exactly([ "base_url", @@ -482,7 +480,6 @@ def _test_register_all_versions(env): _override( base_url = "", available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], - minor_mapping = {"3.12": "3.12.4", "3.13": "3.13.0"}, register_all_versions = True, ), ], @@ -497,8 +494,9 @@ def _test_register_all_versions(env): "3.13.1", ]) env.expect.that_dict(py.overrides.minor_mapping).contains_exactly({ + # The mapping is calculated automatically "3.12": "3.12.4", - "3.13": "3.13.0", + "3.13": "3.13.1", }) env.expect.that_collection(py.toolchains).contains_exactly([ struct( From f8d143f2f2add457ca13976ee44c2e75cba204b5 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:04:40 +0900 Subject: [PATCH 69/75] chore: bazel mod deps --lockfile_mode=update --- examples/bzlmod/MODULE.bazel.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 527d84ffe1..d4dd7fc02f 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "51A1ELfMtsch4OKi/tarMeYVNxxXxLc6pYzSNLw+utY=", + "bzlTransitiveDigest": "oVdgglMbVVTKi8i59RPwHvlFpa5JUUQUQNY+Hd71x5o=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "AXapuzGz+VGyF07ikTgQE2XPud3/UBs3BwZtpr92kGA=", + "bzlTransitiveDigest": "qSZddiDGXTcAX4lPkTG0vUZaaHaQJBuP8r5WHu/bYgo=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", From f844962d291974467d3818b799ed38fc674d4215 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:50:47 +0900 Subject: [PATCH 70/75] fix a case when the first module in mctx.modules is not a root module --- python/private/python.bzl | 267 ++++++++++++++++++---------------- tests/python/python_tests.bzl | 52 +++++-- 2 files changed, 176 insertions(+), 143 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index a33e406317..6ad91adc61 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -67,33 +67,10 @@ def parse_mods(*, mctx, logger, debug = False, fail = fail): seen_versions = {} # overrides that can be changed by the root module - overrides = struct( - kwargs = {}, - minor_mapping = {}, - default = { - "base_url": DEFAULT_RELEASE_BASE_URL, - "tool_versions": { - version: { - # Use a dicts straight away so that we could do URL overrides for a - # single version. - "sha256": dict(item["sha256"]), - "strip_prefix": { - platform: item["strip_prefix"] - for platform in item["sha256"] - }, - "url": { - platform: [item["url"]] - for platform in item["sha256"] - }, - } - for version, item in TOOL_VERSIONS.items() - }, - }, - ) + overrides = _process_overrides(modules = mctx.modules) for mod in mctx.modules: module_toolchain_versions = [] - requested_toolchains = _process_tag_classes( mod, seen_versions = seen_versions, @@ -330,116 +307,134 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): - registrations = [] - - for tag in mod.tags.toolchain: - registrations.append(_create_toolchain_attrs_struct( - tag = tag, - toolchain_tag_count = len(mod.tags.toolchain), - )) - seen_versions[tag.python_version] = True - - if not mod.is_root: - return registrations +def _process_overrides(*, modules, fail = fail): + overrides = struct( + kwargs = {}, + minor_mapping = {}, + default = { + "base_url": DEFAULT_RELEASE_BASE_URL, + "tool_versions": { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() + }, + }, + # A falsey value here for now + register_all = [], + ) available_versions = overrides.default["tool_versions"] - for tag in mod.tags.single_version_override: - if tag.sha256 or tag.urls: - if not (tag.sha256 and tag.urls): - fail("Both `sha256` and `urls` overrides need to be provided together") + for mod in modules: + if not mod.is_root: + break + + for tag in mod.tags.single_version_override: + if tag.sha256 or tag.urls: + if not (tag.sha256 and tag.urls): + fail("Both `sha256` and `urls` overrides need to be provided together") + + for platform in tag.sha256 or []: + if platform not in PLATFORMS: + fail("The platform must be one of {allowed} but got '{got}'".format( + allowed = sorted(PLATFORMS), + got = platform, + )) + + sha256 = dict(tag.sha256) or available_versions[tag.python_version]["sha256"] + override = { + "sha256": sha256, + "strip_prefix": { + platform: tag.strip_prefix + for platform in sha256 + }, + "url": { + platform: list(tag.urls) + for platform in tag.sha256 + } or available_versions[tag.python_version]["url"], + } + + if tag.patches: + override["patch_strip"] = { + platform: tag.patch_strip + for platform in sha256 + } + override["patches"] = { + platform: list(tag.patches) + for platform in sha256 + } - for platform in tag.sha256 or []: - if platform not in PLATFORMS: - fail("The platform must be one of {allowed} but got '{got}'".format( - allowed = sorted(PLATFORMS), - got = platform, + available_versions[tag.python_version] = {k: v for k, v in override.items() if v} + + if tag.distutils_content: + overrides.kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content + if tag.distutils: + overrides.kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + + for tag in mod.tags.single_version_platform_override: + if tag.python_version not in available_versions: + if not tag.urls or not tag.sha256 or not tag.strip_prefix: + fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) + available_versions[tag.python_version] = {} + + if tag.coverage_tool: + available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + if tag.patch_strip: + available_versions[tag.python_version].setdefault("patch_strip", {})[tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.python_version].setdefault("patches", {})[tag.platform] = list(tag.patches) + if tag.sha256: + available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 + if tag.strip_prefix: + available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix + if tag.urls: + available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + + for tag in mod.tags.override: + overrides.kwargs["base_url"] = tag.base_url + if tag.available_python_versions: + all_versions = dict(available_versions) + available_versions.clear() + available_versions.update({ + v: all_versions[v] if v in all_versions else fail("unknown version '{}', known versions are: {}".format( + v, + sorted(all_versions), )) + for v in tag.available_python_versions + }) - sha256 = dict(tag.sha256) or available_versions[tag.python_version]["sha256"] - override = { - "sha256": sha256, - "strip_prefix": { - platform: tag.strip_prefix - for platform in sha256 - }, - "url": { - platform: list(tag.urls) - for platform in tag.sha256 - } or available_versions[tag.python_version]["url"], - } + if tag.register_all_versions: + overrides.register_all.append(True) - if tag.patches: - override["patch_strip"] = { - platform: tag.patch_strip - for platform in sha256 - } - override["patches"] = { - platform: list(tag.patches) - for platform in sha256 - } + if tag.minor_mapping: + for minor_version, full_version in tag.minor_mapping.items(): + parsed = semver(minor_version) + if parsed.patch or parsed.build: + fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) + parsed = semver(full_version) + if not parsed.patch: + fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) + + overrides.minor_mapping.clear() + overrides.minor_mapping.update(tag.minor_mapping) - available_versions[tag.python_version] = {k: v for k, v in override.items() if v} - - if tag.distutils_content: - overrides.kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content - if tag.distutils: - overrides.kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils - - for tag in mod.tags.single_version_platform_override: - if tag.python_version not in available_versions: - if not tag.urls or not tag.sha256 or not tag.strip_prefix: - fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) - available_versions[tag.python_version] = {} - - if tag.coverage_tool: - available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool - if tag.patch_strip: - available_versions[tag.python_version].setdefault("patch_strip", {})[tag.platform] = tag.patch_strip - if tag.patches: - available_versions[tag.python_version].setdefault("patches", {})[tag.platform] = list(tag.patches) - if tag.sha256: - available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 - if tag.strip_prefix: - available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix - if tag.urls: - available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls - - register_all = False - for tag in mod.tags.override: - overrides.kwargs["base_url"] = tag.base_url - if tag.available_python_versions: - all_versions = dict(available_versions) - available_versions.clear() - available_versions.update({ - v: all_versions[v] if v in all_versions else fail("unknown version '{}', known versions are: {}".format( - v, - sorted(all_versions), - )) - for v in tag.available_python_versions - }) - - if tag.register_all_versions: - register_all = True - - if tag.minor_mapping: - for minor_version, full_version in tag.minor_mapping.items(): - parsed = semver(minor_version) - if parsed.patch or parsed.build: - fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) - parsed = semver(full_version) - if not parsed.patch: - fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) - - overrides.minor_mapping.clear() - overrides.minor_mapping.update(tag.minor_mapping) - - for key in sorted(AUTH_ATTRS) + ["ignore_root_user_error"]: - if getattr(tag, key, None): - overrides.default[key] = getattr(tag, key) - - break + for key in sorted(AUTH_ATTRS) + ["ignore_root_user_error"]: + if getattr(tag, key, None): + overrides.default[key] = getattr(tag, key) + + # Process only the first one + break if not overrides.minor_mapping: versions = {} @@ -452,11 +447,27 @@ def _process_tag_classes(mod, *, seen_versions, overrides, fail = fail): for major_minor, subset in versions.items() }) - if register_all: + return overrides + +def _process_tag_classes(mod, *, overrides, seen_versions): + registrations = [] + + for tag in mod.tags.toolchain: + registrations.append(_create_toolchain_attrs_struct( + tag = tag, + toolchain_tag_count = len(mod.tags.toolchain), + )) + + seen_versions[tag.python_version] = True + + if not mod.is_root: + return registrations + + if overrides.register_all: # FIXME @aignas 2024-08-30: this is technically not correct registrations.extend([ _create_toolchain_attrs_struct(python_version = v) - for v in available_versions.keys() + overrides.minor_mapping.keys() + for v in overrides.default["tool_versions"].keys() + overrides.minor_mapping.keys() if v not in seen_versions ]) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index b04bdce89e..15ce755969 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -30,7 +30,7 @@ def _mock_mctx(*modules, environ = {}): struct( name = modules[0].name, tags = modules[0].tags, - is_root = True, + is_root = modules[0].is_root, ), ] + [ struct( @@ -42,7 +42,7 @@ def _mock_mctx(*modules, environ = {}): ], ) -def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = []): +def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = [], is_root = True): return struct( name = name, tags = struct( @@ -51,6 +51,7 @@ def _mod(*, name, toolchain = [], override = [], single_version_override = [], s single_version_override = single_version_override, single_version_platform_override = single_version_platform_override, ), + is_root = is_root, ) def _toolchain(python_version, *, is_default = False, **kwargs): @@ -153,6 +154,34 @@ def _test_default(env): _tests.append(_test_default) +def _test_default_some_module(env): + py = parse_mods( + mctx = _mock_mctx( + _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), + ), + ) + + # The value there should be consistent in bzlmod with the automatically + # calculated value Please update the MINOR_MAPPING in //python:versions.bzl + # when this part starts failing. + env.expect.that_dict(py.overrides.minor_mapping).contains_exactly(MINOR_MAPPING) + env.expect.that_collection(py.overrides.kwargs).has_size(0) + env.expect.that_collection(py.overrides.default.keys()).contains_exactly([ + "base_url", + "ignore_root_user_error", + "tool_versions", + ]) + env.expect.that_str(py.default_python_version).equals("3.11") + + want_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + env.expect.that_collection(py.toolchains).contains_exactly([want_toolchain]) + +_tests.append(_test_default_some_module) + def _test_default_with_patch(env): py = parse_mods( mctx = _mock_mctx( @@ -174,27 +203,20 @@ _tests.append(_test_default_with_patch) def _test_default_non_rules_python(env): py = parse_mods( mctx = _mock_mctx( - _mod(name = "my_module", toolchain = [_toolchain("3.12")]), - _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), + # NOTE @aignas 2024-09-06: the first item in the module_ctx.modules + # could be a non-root module, which is the case if the root module + # does not make any calls to the extension. + _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), ), ) - env.expect.that_str(py.default_python_version).equals("3.12") - - my_module_toolchain = struct( - name = "python_3_12", - python_version = "3.12", - register_coverage_tool = False, - ) + env.expect.that_str(py.default_python_version).equals("3.11") rules_python_toolchain = struct( name = "python_3_11", python_version = "3.11", register_coverage_tool = False, ) - env.expect.that_collection(py.toolchains).contains_exactly([ - rules_python_toolchain, - my_module_toolchain, # default toolchain is last - ]).in_order() + env.expect.that_collection(py.toolchains).contains_exactly([rules_python_toolchain]) _tests.append(_test_default_non_rules_python) From f70676c3ec70f52b744553f9e0d688098f66c714 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:53:05 +0900 Subject: [PATCH 71/75] further cleanup --- python/private/python.bzl | 76 +++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 6ad91adc61..79c673f4b9 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -308,33 +308,30 @@ def _fail_multiple_default_toolchains(first, second): )) def _process_overrides(*, modules, fail = fail): - overrides = struct( - kwargs = {}, - minor_mapping = {}, - default = { - "base_url": DEFAULT_RELEASE_BASE_URL, - "tool_versions": { - version: { - # Use a dicts straight away so that we could do URL overrides for a - # single version. - "sha256": dict(item["sha256"]), - "strip_prefix": { - platform: item["strip_prefix"] - for platform in item["sha256"] - }, - "url": { - platform: [item["url"]] - for platform in item["sha256"] - }, - } - for version, item in TOOL_VERSIONS.items() - }, + kwargs = {} + minor_mapping = {} + default = { + "base_url": DEFAULT_RELEASE_BASE_URL, + "tool_versions": { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() }, - # A falsey value here for now - register_all = [], - ) + } + register_all = False - available_versions = overrides.default["tool_versions"] + available_versions = default["tool_versions"] for mod in modules: if not mod.is_root: @@ -378,9 +375,9 @@ def _process_overrides(*, modules, fail = fail): available_versions[tag.python_version] = {k: v for k, v in override.items() if v} if tag.distutils_content: - overrides.kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content + kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content if tag.distutils: - overrides.kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils for tag in mod.tags.single_version_platform_override: if tag.python_version not in available_versions: @@ -402,20 +399,19 @@ def _process_overrides(*, modules, fail = fail): available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls for tag in mod.tags.override: - overrides.kwargs["base_url"] = tag.base_url + kwargs["base_url"] = tag.base_url if tag.available_python_versions: all_versions = dict(available_versions) - available_versions.clear() - available_versions.update({ + available_versions = { v: all_versions[v] if v in all_versions else fail("unknown version '{}', known versions are: {}".format( v, sorted(all_versions), )) for v in tag.available_python_versions - }) + } if tag.register_all_versions: - overrides.register_all.append(True) + register_all.append(True) if tag.minor_mapping: for minor_version, full_version in tag.minor_mapping.items(): @@ -426,28 +422,32 @@ def _process_overrides(*, modules, fail = fail): if not parsed.patch: fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) - overrides.minor_mapping.clear() - overrides.minor_mapping.update(tag.minor_mapping) + minor_mapping = tag.minor_mapping for key in sorted(AUTH_ATTRS) + ["ignore_root_user_error"]: if getattr(tag, key, None): - overrides.default[key] = getattr(tag, key) + default[key] = getattr(tag, key) # Process only the first one break - if not overrides.minor_mapping: + if not minor_mapping: versions = {} for version_string in available_versions: v = semver(version_string) versions.setdefault("{}.{}".format(v.major, v.minor), []).append((int(v.patch), version_string)) - overrides.minor_mapping.update({ + minor_mapping.update({ major_minor: max(subset)[1] for major_minor, subset in versions.items() }) - return overrides + return struct( + kwargs = kwargs, + minor_mapping = minor_mapping, + default = default, + register_all = register_all, + ) def _process_tag_classes(mod, *, overrides, seen_versions): registrations = [] From d844c01fac4fbae77b1a86f06f98650736e584a1 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:54:42 +0900 Subject: [PATCH 72/75] cleanup --- python/private/python_repositories.bzl | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index a86b1d9fc4..4bad908076 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -599,8 +599,6 @@ def python_register_toolchains( **kwargs: passed to each {obj}`python_repository` call. """ - tool_versions = tool_versions or TOOL_VERSIONS - if BZLMOD_ENABLED: # you cannot used native.register_toolchains when using bzlmod. register_toolchains = False From f0209a21c2cd8326e94e6921f3078712d12e43ae Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:54:57 +0900 Subject: [PATCH 73/75] cleanup --- python/private/python_repositories.bzl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index 4bad908076..a6c53b64f8 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -665,10 +665,6 @@ def python_register_toolchains( urls = urls, strip_prefix = strip_prefix, coverage_tool = coverage_tool, - # Will be one of - # * auth_patterns - # * ignore_root_user_error - # * netrc **kwargs ) if register_toolchains: From 293685ee7f588cdfdfb30897031781f58d6143cb Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 14:33:08 +0900 Subject: [PATCH 74/75] fixup --- python/private/python.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 79c673f4b9..34be0bfcb5 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -411,7 +411,7 @@ def _process_overrides(*, modules, fail = fail): } if tag.register_all_versions: - register_all.append(True) + register_all = True if tag.minor_mapping: for minor_version, full_version in tag.minor_mapping.items(): From dc07b3e9b00675cb74e79a0c06f646f5e9b5229f Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 6 Sep 2024 14:47:05 +0900 Subject: [PATCH 75/75] simplify --- python/private/python.bzl | 78 +++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/python/private/python.bzl b/python/private/python.bzl index 34be0bfcb5..f47308773d 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -310,34 +310,40 @@ def _fail_multiple_default_toolchains(first, second): def _process_overrides(*, modules, fail = fail): kwargs = {} minor_mapping = {} - default = { - "base_url": DEFAULT_RELEASE_BASE_URL, - "tool_versions": { - version: { - # Use a dicts straight away so that we could do URL overrides for a - # single version. - "sha256": dict(item["sha256"]), - "strip_prefix": { - platform: item["strip_prefix"] - for platform in item["sha256"] - }, - "url": { - platform: [item["url"]] - for platform in item["sha256"] - }, - } - for version, item in TOOL_VERSIONS.items() - }, + default = {} + base_url = DEFAULT_RELEASE_BASE_URL + available_versions = { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() } register_all = False - available_versions = default["tool_versions"] - for mod in modules: if not mod.is_root: break + overriden_keys = [] for tag in mod.tags.single_version_override: + key = (tag.python_version,) + if key not in overriden_keys: + overriden_keys.append(key) + else: + fail("Only a single python.single_override can be present for each 'python_version', observed a duplicate override for '{}'".format( + *key + )) + if tag.sha256 or tag.urls: if not (tag.sha256 and tag.urls): fail("Both `sha256` and `urls` overrides need to be provided together") @@ -380,6 +386,14 @@ def _process_overrides(*, modules, fail = fail): kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils for tag in mod.tags.single_version_platform_override: + key = (tag.python_version, tag.platform) + if key not in overriden_keys: + overriden_keys.append(key) + else: + fail("Only a single python.single_version_platform_override can be present for each ('python_version', 'platform') tuple, duplicate override for ('{}', '{}')".format( + *key + )) + if tag.python_version not in available_versions: if not tag.urls or not tag.sha256 or not tag.strip_prefix: fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) @@ -399,16 +413,23 @@ def _process_overrides(*, modules, fail = fail): available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls for tag in mod.tags.override: - kwargs["base_url"] = tag.base_url + key = ("",) + if key not in overriden_keys: + overriden_keys.append(overriden_keys) + else: + fail("Only a single 'python.override' can be present") + + base_url = tag.base_url if tag.available_python_versions: all_versions = dict(available_versions) - available_versions = { + available_versions.clear() + available_versions.update({ v: all_versions[v] if v in all_versions else fail("unknown version '{}', known versions are: {}".format( v, sorted(all_versions), )) for v in tag.available_python_versions - } + }) if tag.register_all_versions: register_all = True @@ -428,20 +449,21 @@ def _process_overrides(*, modules, fail = fail): if getattr(tag, key, None): default[key] = getattr(tag, key) - # Process only the first one - break - if not minor_mapping: versions = {} for version_string in available_versions: v = semver(version_string) versions.setdefault("{}.{}".format(v.major, v.minor), []).append((int(v.patch), version_string)) - minor_mapping.update({ + minor_mapping = { major_minor: max(subset)[1] for major_minor, subset in versions.items() - }) + } + default.update({ + "base_url": base_url, + "tool_versions": available_versions, + }) return struct( kwargs = kwargs, minor_mapping = minor_mapping,