Revert "chore: cleanup bzlmod codebase after pipstar enabling"

This reverts commit 10692ce.

Author: aignas

python/private/pypi/extension.bzl

@@ -21,6 +21,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:repo_utils.bzl", "repo_utils")
+load(":evaluate_markers.bzl", EVALUATE_MARKERS_SRCS = "SRCS")
 load(":hub_builder.bzl", "hub_builder")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -671,6 +672,13 @@ EXPERIMENTAL: this may be removed without notice.
             doc = """\
 A dict of labels to wheel names that is typically generated by the whl_modifications.
 The labels are JSON config files describing the modifications.
+""",
+        ),
+        "_evaluate_markers_srcs": attr.label_list(
+            default = EVALUATE_MARKERS_SRCS,
+            doc = """\
+The list of labels to use as SRCS for the marker evaluation code. This ensures that the
+code will be re-evaluated when any of files in the default changes.
 """,
         ),
     }, **ATTRS)

python/private/pypi/hub_builder.bzl

@@ -5,7 +5,7 @@ load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:version.bzl", "version")
 load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
-load(":evaluate_markers.bzl", "evaluate_markers")
+load(":evaluate_markers.bzl", "evaluate_markers_py", evaluate_markers_star = "evaluate_markers")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":pep508_env.bzl", "env")
 load(":pep508_evaluate.bzl", "evaluate")
@@ -175,6 +175,8 @@ def _add_whl_library(self, *, python_version, whl, repo):
         # disallow building from sdist.
         return
 
+    platforms = self._platforms[python_version]
+
     # TODO @aignas 2025-06-29: we should not need the version in the repo_name if
     # we are using pipstar and we are downloading the wheel using the downloader
     repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name)
@@ -186,6 +188,17 @@ def _add_whl_library(self, *, python_version, whl, repo):
         ))
     self._whl_libraries[repo_name] = repo.args
 
+    if not self._config.enable_pipstar and "experimental_target_platforms" in repo.args:
+        self._whl_libraries[repo_name] |= {
+            "experimental_target_platforms": sorted({
+                # TODO @aignas 2025-07-07: this should be solved in a better way
+                platforms[candidate].triple.partition("_")[-1]: None
+                for p in repo.args["experimental_target_platforms"]
+                for candidate in platforms
+                if candidate.endswith(p)
+            }),
+        }
+
     mapping = self._whl_map.setdefault(whl.name, {})
     if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name:
         fail(
@@ -316,9 +329,41 @@ def _evaluate_markers(self, pip_attr):
     if self._evaluate_markers_fn:
         return self._evaluate_markers_fn
 
-    return lambda _, requirements: evaluate_markers(
-        requirements = requirements,
-        platforms = self._platforms[pip_attr.python_version],
+    if self._config.enable_pipstar:
+        return lambda _, requirements: evaluate_markers_star(
+            requirements = requirements,
+            platforms = self._platforms[pip_attr.python_version],
+        )
+
+    interpreter = _detect_interpreter(self, pip_attr)
+
+    # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
+    # in the PATH or if specified as a label. We will configure the env
+    # markers when evaluating the requirement lines based on the output
+    # from the `requirements_files_by_platform` which should have something
+    # similar to:
+    # {
+    #    "//:requirements.txt": ["cp311_linux_x86_64", ...]
+    # }
+    #
+    # We know the target python versions that we need to evaluate the
+    # markers for and thus we don't need to use multiple python interpreter
+    # instances to perform this manipulation. This function should be executed
+    # only once by the underlying code to minimize the overhead needed to
+    # spin up a Python interpreter.
+    return lambda module_ctx, requirements: evaluate_markers_py(
+        module_ctx,
+        requirements = {
+            k: {
+                p: self._platforms[pip_attr.python_version][p].triple
+                for p in plats
+            }
+            for k, plats in requirements.items()
+        },
+        python_interpreter = interpreter.path,
+        python_interpreter_target = interpreter.target,
+        srcs = pip_attr._evaluate_markers_srcs,
+        logger = self._logger,
     )
 
 def _create_whl_repos(
@@ -390,6 +435,7 @@ def _create_whl_repos(
                 auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
                 python_version = _major_minor_version(pip_attr.python_version),
                 is_multiple_versions = whl.is_multiple_versions,
+                enable_pipstar = self._config.enable_pipstar,
             )
             _add_whl_library(
                 self,
@@ -417,6 +463,8 @@ def _common_args(self, module_ctx, *, pip_attr):
         python_interpreter = interpreter.path,
         python_interpreter_target = interpreter.target,
     )
+    if not self._config.enable_pipstar:
+        maybe_args["experimental_target_platforms"] = pip_attr.experimental_target_platforms
 
     whl_library_args.update({k: v for k, v in maybe_args.items() if v})
     maybe_args_with_default = dict(
@@ -464,7 +512,8 @@ def _whl_repo(
         netrc,
         auth_patterns,
         python_version,
-        use_downloader):
+        use_downloader,
+        enable_pipstar = False):
     args = dict(whl_library_args)
     args["requirement"] = src.requirement_line
     is_whl = src.filename.endswith(".whl")
@@ -507,6 +556,14 @@ def _whl_repo(
     args["urls"] = [src.url]
     args["sha256"] = src.sha256
     args["filename"] = src.filename
+    if not enable_pipstar:
+        args["experimental_target_platforms"] = [
+            # Get rid of the version for the target platforms because we are
+            # passing the interpreter any way. Ideally we should search of ways
+            # how to pass the target platforms through the hub repo.
+            p.partition("_")[2]
+            for p in src.target_platforms
+        ]
 
     return struct(
         repo_name = whl_repo_name(src.filename, src.sha256),

tests/pypi/hub_builder/hub_builder_tests.bzl

@@ -40,16 +40,19 @@ simple==0.0.1 \
 
 def hub_builder(
         env,
+        enable_pipstar = False,
         debug = False,
         config = None,
         minor_mapping = {},
+        evaluate_markers_fn = None,
         simpleapi_download_fn = None,
         available_interpreters = {}):
     builder = _hub_builder(
         name = "pypi",
         module_name = "unit_test",
         config = config or struct(
             # no need to evaluate the markers with the interpreter
+            enable_pipstar = enable_pipstar,
             platforms = {
                 "{}_{}{}".format(os, cpu, freethreaded): _plat(
                     name = "{}_{}{}".format(os, cpu, freethreaded),
@@ -73,13 +76,13 @@ def hub_builder(
             netrc = None,
             auth_patterns = None,
         ),
-        evaluate_markers_fn = None,
         whl_overrides = {},
         minor_mapping = minor_mapping or {"3.15": "3.15.19"},
         available_interpreters = available_interpreters or {
             "python_3_15_host": "unit_test_interpreter_target",
         },
         simpleapi_download_fn = simpleapi_download_fn or (lambda *a, **k: {}),
+        evaluate_markers_fn = evaluate_markers_fn,
         logger = repo_utils.logger(
             struct(
                 os = struct(
@@ -300,6 +303,14 @@ _tests.append(_test_simple_multiple_python_versions)
 def _test_simple_with_markers(env):
     builder = hub_builder(
         env,
+        evaluate_markers_fn = lambda _, requirements, **__: {
+            key: [
+                platform
+                for platform in platforms
+                if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
+            ]
+            for key, platforms in requirements.items()
+        },
     )
     builder.pip_parse(
         _mock_mctx(
@@ -394,6 +405,7 @@ def _test_torch_experimental_index_url(env):
         env,
         config = struct(
             netrc = None,
+            enable_pipstar = False,
             auth_patterns = {},
             platforms = {
                 "{}_{}".format(os, cpu): _plat(
@@ -419,6 +431,15 @@ def _test_torch_experimental_index_url(env):
             "python_3_12_host": "unit_test_interpreter_target",
         },
         minor_mapping = {"3.12": "3.12.19"},
+        evaluate_markers_fn = lambda _, requirements, **__: {
+            # todo once 2692 is merged, this is going to be easier to test.
+            key: [
+                platform
+                for platform in platforms
+                if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
+            ]
+            for key, platforms in requirements.items()
+        },
         simpleapi_download_fn = mocksimpleapi_download,
     )
     builder.pip_parse(
@@ -495,6 +516,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
     pypi.whl_libraries().contains_exactly({
         "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["linux_x86_64"],
             "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "torch==2.4.1+cpu",
@@ -503,6 +525,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
         },
         "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["linux_aarch64"],
             "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "torch==2.4.1",
@@ -511,6 +534,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
         },
         "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["windows_x86_64"],
             "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "torch==2.4.1+cpu",
@@ -519,6 +543,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
         },
         "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["osx_aarch64"],
             "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "torch==2.4.1",
@@ -596,6 +621,8 @@ simple==0.0.3 \
         "pypi_315_extra": {
             "dep_template": "@pypi//{name}:{target}",
             "download_only": True,
+            # TODO @aignas 2025-04-20: ensure that this is in the hub repo
+            # "experimental_target_platforms": ["cp315_linux_x86_64"],
             "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "extra==0.0.1 --hash=sha256:deadb00f",
@@ -794,6 +821,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
     pypi.whl_libraries().contains_exactly({
         "pypi_315_any_name": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": [
+                "linux_aarch64",
+                "linux_x86_64",
+                "osx_aarch64",
+                "windows_aarch64",
+            ],
             "extra_pip_args": ["--extra-args-for-sdist-building"],
             "filename": "any-name.tar.gz",
             "python_interpreter_target": "unit_test_interpreter_target",
@@ -803,6 +836,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
         },
         "pypi_315_direct_without_sha_0_0_1_py3_none_any": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": [
+                "linux_aarch64",
+                "linux_x86_64",
+                "osx_aarch64",
+                "windows_aarch64",
+            ],
             "filename": "direct_without_sha-0.0.1-py3-none-any.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "direct_without_sha==0.0.1",
@@ -823,6 +862,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
         },
         "pypi_315_simple_py3_none_any_deadb00f": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": [
+                "linux_aarch64",
+                "linux_x86_64",
+                "osx_aarch64",
+                "windows_aarch64",
+            ],
             "filename": "simple-0.0.1-py3-none-any.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "simple==0.0.1",
@@ -831,6 +876,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
         },
         "pypi_315_some_pkg_py3_none_any_deadbaaf": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": [
+                "linux_aarch64",
+                "linux_x86_64",
+                "osx_aarch64",
+                "windows_aarch64",
+            ],
             "filename": "some_pkg-0.0.1-py3-none-any.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "some_pkg==0.0.1",
@@ -839,6 +890,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
         },
         "pypi_315_some_py3_none_any_deadb33f": {
             "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": [
+                "linux_aarch64",
+                "linux_x86_64",
+                "osx_aarch64",
+                "windows_aarch64",
+            ],
             "filename": "some-other-pkg-0.0.1-py3-none-any.whl",
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "some_other_pkg==0.0.1",
@@ -868,6 +925,14 @@ _tests.append(_test_simple_get_index)
 def _test_optimum_sys_platform_extra(env):
     builder = hub_builder(
         env,
+        evaluate_markers_fn = lambda _, requirements, **__: {
+            key: [
+                platform
+                for platform in platforms
+                if ("darwin" in key and "osx" in platform) or ("linux" in key and "linux" in platform)
+            ]
+            for key, platforms in requirements.items()
+        },
     )
     builder.pip_parse(
         _mock_mctx(
@@ -881,14 +946,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux'
         _parse(
             hub_name = "pypi",
             python_version = "3.15",
-            requirements_by_platform = {
-                "universal.txt": "linux_*,osx_*",
-            },
+            requirements_lock = "universal.txt",
         ),
     )
     pypi = builder.build()
 
-    pypi.exposed_packages().contains_exactly(["optimum"])
+    # FIXME @aignas 2025-09-07: we should expose the `optimum` package
+    pypi.exposed_packages().contains_exactly([])
     pypi.group_map().contains_exactly({})
     pypi.whl_map().contains_exactly({
         "optimum": {
@@ -931,7 +995,9 @@ _tests.append(_test_optimum_sys_platform_extra)
 def _test_pipstar_platforms(env):
     builder = hub_builder(
         env,
+        enable_pipstar = True,
         config = struct(
+            enable_pipstar = True,
             netrc = None,
             auth_patterns = {},
             platforms = {