feat(py_wheel): Normalize name and version (#1331)

Added the `incompatible_normalize_name` feature flag to normalize the
package distribution name according to latest Python packaging
standards. Defaults to `False` for the time being.

Added the `incompatible_normalize_version` feature flag to normalize the
package version according to PEP440 standard. This also adds support for
local version specifiers (versions with a `+` in them), in accordance
with PEP440. Defaults to `False` for the time being.

Instead of following the obsolete PEP 427 escaping procedure for
distribution names and versions, we should use the rules specified by
https://packaging.python.org/en/latest/specifications (sections "Package
name normalization" and "Binary distribution format"). For the versions,
this means normalizing them according to PEP 440.

Added as feature flags to avoid forcing the user to deal with breaking
changes when upgrading `rules_python`:

- Distribution names have stronger requirements now: "A valid name
consists only of ASCII letters and numbers, period, underscore and
hyphen. It must start and end with a letter or number."
https://packaging.python.org/en/latest/specifications/name-normalization/

- Versions must be valid PEP 440 version identifiers. Previously
versions such as "0.1-2-3" would have been accepted; that is no longer
the case.

- The file name of generated wheels may have changed, if the
distribution name or the version identifier wasn't in normalized form.

- The wheelmaker now depends on `packaging.version`, which means the
`py_wheel` user now needs to load pip dependencies in their
`WORKSPACE.bazel` file:
  ```
load("@rules_python//python/pip_install:repositories.bzl",
"pip_install_dependencies")

  pip_install_dependencies()
  ```

Fixes #883. Fixes #1132.

---------

Co-authored-by: Ignas Anikevicius <[email protected]>
Co-authored-by: Ignas Anikevicius <[email protected]>

Author: vonschultz

CHANGELOG.md

@@ -55,6 +55,14 @@ A brief description of the categories of changes:
   authentication against private HTTP hosts serving Python toolchain binaries.
 * `//python:packaging_bzl` added, a `bzl_library` for the Starlark
   files `//python:packaging.bzl` requires.
+* (py_wheel) Added the `incompatible_normalize_name` feature flag to
+  normalize the package distribution name according to latest Python
+  packaging standards. Defaults to `False` for the time being.
+* (py_wheel) Added the `incompatible_normalize_version` feature flag
+  to normalize the package version according to PEP440 standard. This
+  also adds support for local version specifiers (versions with a `+`
+  in them), in accordance with PEP440. Defaults to `False` for the
+  time being.
 
 ### Removed
 

docs/packaging.md

@@ -54,6 +54,8 @@ py_wheel(
     testonly = True,  # Set this to verify the generated .dist target doesn't break things
     # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
     distribution = "example_minimal_library",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     version = "0.0.1",
     deps = [
@@ -76,6 +78,8 @@ py_wheel(
     testonly = True,
     abi = "$(ABI)",
     distribution = "example_minimal_library",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "$(PYTHON_TAG)",
     toolchains = ["//examples/wheel:make_variable_tags"],
     version = "$(VERSION)",
@@ -95,6 +99,8 @@ py_wheel(
     name = "minimal_with_py_library_with_stamp",
     # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
     distribution = "example_minimal_library{BUILD_USER}",
+    incompatible_normalize_name = False,
+    incompatible_normalize_version = False,
     python_tag = "py3",
     stamp = 1,
     version = "0.1.{BUILD_TIMESTAMP}",
@@ -123,6 +129,8 @@ py_wheel(
     name = "minimal_with_py_package",
     # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl"
     distribution = "example_minimal_package",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     version = "0.0.1",
     deps = [":example_pkg"],
@@ -156,6 +164,8 @@ py_wheel(
         "//examples/wheel:README.md": "README",
     },
     homepage = "www.example.com",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     license = "Apache 2.0",
     project_urls = {
         "Bug Tracker": "www.example.com/issues",
@@ -177,6 +187,8 @@ py_wheel(
     entry_points = {
         "console_scripts": ["main = foo.bar:baz"],
     },
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     strip_path_prefixes = [
         "examples",
@@ -191,6 +203,8 @@ py_wheel(
     name = "custom_package_root_multi_prefix",
     # Package data. We're building "custom_custom_package_root_multi_prefix-0.0.1-py3-none-any.whl"
     distribution = "example_custom_package_root_multi_prefix",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     strip_path_prefixes = [
         "examples/wheel/lib",
@@ -206,6 +220,8 @@ py_wheel(
     name = "custom_package_root_multi_prefix_reverse_order",
     # Package data. We're building "custom_custom_package_root_multi_prefix_reverse_order-0.0.1-py3-none-any.whl"
     distribution = "example_custom_package_root_multi_prefix_reverse_order",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     strip_path_prefixes = [
         "examples/wheel",
@@ -220,6 +236,8 @@ py_wheel(
 py_wheel(
     name = "python_requires_in_a_package",
     distribution = "example_python_requires_in_a_package",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_requires = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
     python_tag = "py3",
     version = "0.0.1",
@@ -231,6 +249,8 @@ py_wheel(
 py_wheel(
     name = "use_rule_with_dir_in_outs",
     distribution = "use_rule_with_dir_in_outs",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     python_tag = "py3",
     version = "0.0.1",
     deps = [
@@ -244,6 +264,8 @@ py_wheel(
     name = "python_abi3_binary_wheel",
     abi = "abi3",
     distribution = "example_python_abi3_binary_wheel",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
     # these platform strings must line up with test_python_abi3_binary_wheel() in wheel_test.py
     platform = select({
         ":aarch64-apple-darwin": "macosx_11_0_arm64",
@@ -258,16 +280,32 @@ py_wheel(
 )
 
 py_wheel(
-    name = "filename_escaping",
+    name = "legacy_filename_escaping",
     # Per https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode
     # runs of non-alphanumeric, non-digit symbols should be replaced with a single underscore.
     # Unicode non-ascii letters should *not* be replaced with underscore.
     distribution = "file~~name-escaping",
+    incompatible_normalize_name = False,
+    incompatible_normalize_version = False,
     python_tag = "py3",
     version = "0.0.1-r7",
     deps = [":example_pkg"],
 )
 
+py_wheel(
+    name = "filename_escaping",
+    # Per https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode
+    # runs of "-", "_" and "." should be replaced with a single underscore.
+    # Unicode non-ascii letters aren't allowed according to
+    # https://packaging.python.org/en/latest/specifications/name-normalization/.
+    distribution = "File--Name-Escaping",
+    incompatible_normalize_name = True,
+    incompatible_normalize_version = True,
+    python_tag = "py3",
+    version = "v0.0.1.RC1+ubuntu-r7",
+    deps = [":example_pkg"],
+)
+
 py_test(
     name = "wheel_test",
     srcs = ["wheel_test.py"],
@@ -277,6 +315,7 @@ py_test(
         ":custom_package_root_multi_prefix_reverse_order",
         ":customized",
         ":filename_escaping",
+        ":legacy_filename_escaping",
         ":minimal_with_py_library",
         ":minimal_with_py_library_with_stamp",
         ":minimal_with_py_package",

examples/wheel/BUILD.bazel

@@ -153,13 +153,51 @@ def test_customized_wheel(self):
 second = second.main:s""",
             )
 
+    def test_legacy_filename_escaping(self):
+        filename = os.path.join(
+            os.environ['TEST_SRCDIR'],
+            'rules_python',
+            'examples',
+            'wheel',
+            'file_name_escaping-0.0.1_r7-py3-none-any.whl',
+        )
+        with zipfile.ZipFile(filename) as zf:
+            self.assertEquals(
+                zf.namelist(),
+                [
+                    'examples/wheel/lib/data.txt',
+                    'examples/wheel/lib/module_with_data.py',
+                    'examples/wheel/lib/simple_module.py',
+                    'examples/wheel/main.py',
+                    # PEP calls for replacing only in the archive filename.
+                    # Alas setuptools also escapes in the dist-info directory
+                    # name, so let's be compatible.
+                    'file_name_escaping-0.0.1_r7.dist-info/WHEEL',
+                    'file_name_escaping-0.0.1_r7.dist-info/METADATA',
+                    'file_name_escaping-0.0.1_r7.dist-info/RECORD',
+                ],
+            )
+            metadata_contents = zf.read(
+                'file_name_escaping-0.0.1_r7.dist-info/METADATA'
+            )
+            self.assertEquals(
+                metadata_contents,
+                b"""\
+Metadata-Version: 2.1
+Name: file~~name-escaping
+Version: 0.0.1-r7
+
+UNKNOWN
+""",
+            )
+
     def test_filename_escaping(self):
         filename = os.path.join(
             os.environ["TEST_SRCDIR"],
             "rules_python",
             "examples",
             "wheel",
-            "file_name_escaping-0.0.1_r7-py3-none-any.whl",
+            "file_name_escaping-0.0.1rc1+ubuntu.r7-py3-none-any.whl",
         )
         with zipfile.ZipFile(filename) as zf:
             self.assertEqual(
@@ -172,20 +210,20 @@ def test_filename_escaping(self):
                     # PEP calls for replacing only in the archive filename.
                     # Alas setuptools also escapes in the dist-info directory
                     # name, so let's be compatible.
-                    "file_name_escaping-0.0.1_r7.dist-info/WHEEL",
-                    "file_name_escaping-0.0.1_r7.dist-info/METADATA",
-                    "file_name_escaping-0.0.1_r7.dist-info/RECORD",
+                    "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/WHEEL",
+                    "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/METADATA",
+                    "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/RECORD",
                 ],
             )
             metadata_contents = zf.read(
-                "file_name_escaping-0.0.1_r7.dist-info/METADATA"
+                "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/METADATA"
             )
             self.assertEqual(
                 metadata_contents,
                 b"""\
 Metadata-Version: 2.1
-Name: file~~name-escaping
-Version: 0.0.1-r7
+Name: File--Name-Escaping
+Version: 0.0.1rc1+ubuntu.r7
 
 UNKNOWN
 """,

examples/wheel/wheel_test.py

@@ -77,6 +77,7 @@ bzl_library(
         ":py_binary_bzl",
         "//python/private:py_package.bzl",
         "//python/private:py_wheel_bzl",
+        "//python/private:py_wheel_normalize_pep440.bzl",
         "//python/private:stamp_bzl",
         "//python/private:util_bzl",
     ],

python/BUILD.bazel

@@ -236,6 +236,7 @@ exports_files(
         "coverage.patch",
         "py_package.bzl",
         "py_wheel.bzl",
+        "py_wheel_normalize_pep440.bzl",
         "reexports.bzl",
         "stamp.bzl",
         "util.bzl",