tests(pypi): add tests for namespace shims generation (#3066)

This adds functional tests for the generated pkgutil namespace files.

The test works by creating two wheels with the necessary structure:
* An `__init__.py` file isn't in the wheel for the namespace package
* They are both part of the namespace package.
* The test verifies both are importable.

These are the tests for
#3059

Author: rickeylev

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
 
 test --test_output=errors
 

MODULE.bazel

@@ -158,6 +158,8 @@ internal_dev_deps = use_extension(
 use_repo(
     internal_dev_deps,
     "buildkite_config",
+    "implicit_namespace_ns_sub1",
+    "implicit_namespace_ns_sub2",
     "rules_python_runtime_env_tc_info",
     "somepkg_with_build_files",
     "whl_with_build_files",

python/private/internal_dev_deps.bzl

@@ -30,6 +30,7 @@ def _internal_dev_deps_impl(mctx):
     )
     runtime_env_repo(name = "rules_python_runtime_env_tc_info")
 
+    # Setup for //tests/whl_with_build_files
     whl_from_dir_repo(
         name = "whl_with_build_files",
         root = "//tests/whl_with_build_files:testdata/BUILD.bazel",
@@ -41,6 +42,31 @@ def _internal_dev_deps_impl(mctx):
         requirement = "somepkg",
     )
 
+    # Setup for //tests/implicit_namespace_packages
+    whl_from_dir_repo(
+        name = "implicit_namespace_ns_sub1_whl",
+        root = "//tests/implicit_namespace_packages:testdata/ns-sub1/BUILD.bazel",
+        output = "ns_sub1-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "implicit_namespace_ns_sub1",
+        whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl",
+        requirement = "ns-sub1",
+        enable_implicit_namespace_pkgs = False,
+    )
+
+    whl_from_dir_repo(
+        name = "implicit_namespace_ns_sub2_whl",
+        root = "//tests/implicit_namespace_packages:testdata/ns-sub2/BUILD.bazel",
+        output = "ns_sub2-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "implicit_namespace_ns_sub2",
+        whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl",
+        requirement = "ns-sub2",
+        enable_implicit_namespace_pkgs = False,
+    )
+
 internal_dev_deps = module_extension(
     implementation = _internal_dev_deps_impl,
     doc = "This extension creates internal rules_python dev dependencies.",

python/private/pypi/whl_library_targets.bzl

@@ -30,7 +30,7 @@ load(
     "WHEEL_FILE_IMPL_LABEL",
     "WHEEL_FILE_PUBLIC_LABEL",
 )
-load(":namespace_pkgs.bzl", "create_inits")
+load(":namespace_pkgs.bzl", _create_inits = "create_inits")
 load(":pep508_deps.bzl", "deps")
 
 def whl_library_targets_from_requires(
@@ -120,6 +120,7 @@ def whl_library_targets(
             py_binary = py_binary,
             py_library = py_library,
             env_marker_setting = env_marker_setting,
+            create_inits = _create_inits,
         )):
     """Create all of the whl_library targets.
 
@@ -334,7 +335,7 @@ def whl_library_targets(
         if not enable_implicit_namespace_pkgs:
             srcs = srcs + getattr(native, "select", select)({
                 Label("//python/config_settings:is_venvs_site_packages"): [],
-                "//conditions:default": create_inits(
+                "//conditions:default": rules.create_inits(
                     srcs = srcs + data + pyi_srcs,
                     ignored_dirnames = [],  # If you need to ignore certain folders, you can patch rules_python here to do so.
                     root = "site-packages",

tests/implicit_namespace_packages/BUILD.bazel

@@ -0,0 +1,12 @@
+load("//python:py_test.bzl", "py_test")
+load("//tests/support:support.bzl", "SUPPORTS_BZLMOD_UNIXY")
+
+py_test(
+    name = "namespace_packages_test",
+    srcs = ["namespace_packages_test.py"],
+    target_compatible_with = SUPPORTS_BZLMOD_UNIXY,
+    deps = [
+        "@implicit_namespace_ns_sub1//:pkg",
+        "@implicit_namespace_ns_sub2//:pkg",
+    ],
+)

tests/implicit_namespace_packages/namespace_packages_test.py

@@ -0,0 +1,24 @@
+import unittest
+
+
+class NamespacePackagesTest(unittest.TestCase):
+
+    def test_both_importable(self):
+        import nspkg
+        import nspkg.subpkg1
+        import nspkg.subpkg1.subpkgmod
+        import nspkg.subpkg2.subpkgmod
+
+        self.assertEqual("nspkg.subpkg1", nspkg.subpkg1.expected_name)
+        self.assertEqual(
+            "nspkg.subpkg1.subpkgmod", nspkg.subpkg1.subpkgmod.expected_name
+        )
+
+        self.assertEqual("nspkg.subpkg2", nspkg.subpkg2.expected_name)
+        self.assertEqual(
+            "nspkg.subpkg2.subpkgmod", nspkg.subpkg2.subpkgmod.expected_name
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA

@@ -0,0 +1 @@
+Wheel-Version: 1.0

tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/RECORD

@@ -0,0 +1 @@
+expected_name = "nspkg.subpkg1"