metadata: initial PEP 753 bits (#17470)

* metadata: initial PEP 753 bits

Signed-off-by: William Woodruff <[email protected]>

* fix lint

Signed-off-by: William Woodruff <[email protected]>

* update comment

Signed-off-by: William Woodruff <[email protected]>

* use a shorter URL

Signed-off-by: William Woodruff <[email protected]>

---------

Signed-off-by: William Woodruff <[email protected]>
Co-authored-by: Dustin Ingram <[email protected]>

Author: woodruffw

tests/unit/forklift/test_metadata.py

@@ -14,7 +14,6 @@
 import pytest
 
 from packaging.version import Version
-from sqlalchemy.dialects.postgresql import ENUM
 from webob.multidict import MultiDict
 
 from warehouse.forklift import metadata
@@ -91,7 +90,7 @@ def test_length_is_limited(self, field_name, length):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "a" * (length - 1)}
+            **{field_name: "a" * (length - 1)},
         )
         meta = metadata.parse(None, form_data=data)
         assert getattr(meta, field_name) == "a" * (length - 1)
@@ -101,7 +100,7 @@ def test_length_is_limited(self, field_name, length):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "a" * (length + 1)}
+            **{field_name: "a" * (length + 1)},
         )
         with pytest.raises(ExceptionGroup) as excinfo:
             metadata.parse(None, form_data=data)
@@ -113,7 +112,7 @@ def test_valid_emails(self, field_name):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "[email protected]"}
+            **{field_name: "[email protected]"},
         )
         meta = metadata.parse(None, form_data=data)
         assert getattr(meta, field_name) == "[email protected]"
@@ -124,7 +123,7 @@ def test_invalid_emails(self, field_name):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "Foo <test>"}
+            **{field_name: "Foo <test>"},
         )
         with pytest.raises(ExceptionGroup) as excinfo:
             metadata.parse(None, form_data=data)
@@ -237,7 +236,7 @@ def test_valid_dists(self, field_name):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "foo>=1.0"}
+            **{field_name: "foo>=1.0"},
         )
         meta = metadata.parse(None, form_data=data)
         assert [str(r) for r in getattr(meta, field_name)] == ["foo>=1.0"]
@@ -252,7 +251,7 @@ def test_invalid_dists(self, field_name):
                 metadata_version="2.1",
                 name="spam",
                 version="2.0",
-                **{field_name: "foo >= dog"}
+                **{field_name: "foo >= dog"},
             )
             with pytest.raises(
                 (
@@ -269,7 +268,7 @@ def test_invalid_dists(self, field_name):
             metadata_version="2.1",
             name="spam",
             version="2.0",
-            **{field_name: "foo @ https://example.com/foo-1.0.tar.gz"}
+            **{field_name: "foo @ https://example.com/foo-1.0.tar.gz"},
         )
         with pytest.raises(ExceptionGroup) as excinfo:
             metadata.parse(None, form_data=data)
@@ -295,7 +294,7 @@ def test_valid_dynamic_but_missing_from_our_enum(self, monkeypatch):
         considers to be valid, but don't exist in our enum and would otherwise fail
         when inserting them into the database
         """
-        monkeypatch.setattr(metadata, "DynamicFieldsEnum", ENUM())
+        monkeypatch.setattr(metadata, "DYNAMIC_FIELDS", [])
         data = MultiDict(metadata_version="2.2", name="spam", version="2.0")
         data.add("dynamic", "author")
         with pytest.raises(ExceptionGroup) as excinfo:
@@ -353,3 +352,19 @@ def test_empty_strings_are_ignored(self):
 
         meta = metadata.parse_form_metadata(data)
         assert meta.description_content_type is None
+
+
+@pytest.mark.parametrize(
+    ("label", "expected"),
+    [
+        ("Home-page", "homepage"),
+        ("homepage", "homepage"),
+        ("Home Page", "homepage"),
+        ("HomePage", "homepage"),
+        ("HOMEPAGE", "homepage"),
+        ("What's New", "whatsnew"),
+        ("Change_Log", "changelog"),
+    ],
+)
+def test_normalize_project_url_label(label, expected):
+    assert metadata.normalize_project_url_label(label) == expected

warehouse/forklift/metadata.py

@@ -12,6 +12,7 @@
 
 import email.message
 import email.utils
+import string
 import typing
 
 import email_validator
@@ -30,11 +31,42 @@
 from trove_classifiers import all_classifiers, deprecated_classifiers
 from webob.multidict import MultiDict
 
-from warehouse.packaging.models import DynamicFieldsEnum
 from warehouse.utils import http
 
 SUPPORTED_METADATA_VERSIONS = {"1.0", "1.1", "1.2", "2.1", "2.2", "2.3", "2.4"}
 
+DYNAMIC_FIELDS = [
+    "Platform",
+    "Supported-Platform",
+    "Summary",
+    "Description",
+    "Description-Content-Type",
+    "Keywords",
+    "Home-Page",  # Deprecated, but technically permitted by PEP 643
+    "Download-Url",  # Deprecated, but technically permitted by PEP 643
+    "Author",
+    "Author-Email",
+    "Maintainer",
+    "Maintainer-Email",
+    "License",
+    "License-Expression",
+    "License-File",
+    "Classifier",
+    "Requires-Dist",
+    "Requires-Python",
+    "Requires-External",
+    "Project-Url",
+    "Provides-Extra",
+    "Provides-Dist",
+    "Obsoletes-Dist",
+    # Although the following are deprecated fields, they are technically
+    # permitted as dynamic by PEP 643
+    # https://github.com/pypa/setuptools/issues/4797#issuecomment-2589514950
+    "Requires",
+    "Provides",
+    "Obsoletes",
+]
+
 # Mapping of fields on a Metadata instance to any limits on the length of that
 # field. Fields without a limit will naturally be unlimited in length.
 _LENGTH_LIMITS = {
@@ -241,7 +273,7 @@ def _validate_metadata(metadata: Metadata, *, backfill: bool = False):
     for field in {"dynamic"}:
         if (value := getattr(metadata, field)) is not None:
             for key in value:
-                if key not in map(str.lower, DynamicFieldsEnum.enums):
+                if key not in map(str.lower, DYNAMIC_FIELDS):
                     errors.append(
                         InvalidMetadata(
                             _RAW_TO_EMAIL_MAPPING.get(field, field),
@@ -362,3 +394,12 @@ def parse_form_metadata(data: MultiDict) -> Metadata:
     # way this function is implemented, our `TypedDict` can only have valid key
     # names.
     return Metadata.from_raw(typing.cast(RawMetadata, raw))
+
+
+def normalize_project_url_label(label: str) -> str:
+    # Normalize a Project-URL label according to the label normalization
+    # rules in the "Well-Known Project URLs in Metadata" specification:
+    # <https://packaging.python.org/en/latest/specifications/well-known-project-urls/#label-normalization>
+    chars_to_remove = string.punctuation + string.whitespace
+    removal_map = str.maketrans("", "", chars_to_remove)
+    return label.translate(removal_map).lower()

warehouse/packaging/models.py

@@ -66,6 +66,7 @@
 from warehouse.authnz import Permissions
 from warehouse.classifiers.models import Classifier
 from warehouse.events.models import HasEvents
+from warehouse.forklift import metadata
 from warehouse.integrations.vulnerabilities.models import VulnerabilityRecord
 from warehouse.observations.models import HasObservations
 from warehouse.organizations.models import (
@@ -559,35 +560,7 @@ class ReleaseURL(db.Model):
 
 
 DynamicFieldsEnum = ENUM(
-    "Platform",
-    "Supported-Platform",
-    "Summary",
-    "Description",
-    "Description-Content-Type",
-    "Keywords",
-    "Home-Page",  # Deprecated, but technically permitted by PEP 643
-    "Download-Url",  # Deprecated, but technically permitted by PEP 643
-    "Author",
-    "Author-Email",
-    "Maintainer",
-    "Maintainer-Email",
-    "License",
-    "License-Expression",
-    "License-File",
-    "Classifier",
-    "Requires-Dist",
-    "Requires-Python",
-    "Requires-External",
-    "Project-Url",
-    "Provides-Extra",
-    "Provides-Dist",
-    "Obsoletes-Dist",
-    # Although the following are deprecated fields, they are technically
-    # permitted as dynamic by PEP 643
-    # https://github.com/pypa/setuptools/issues/4797#issuecomment-2589514950
-    "Requires",
-    "Provides",
-    "Obsoletes",
+    *metadata.DYNAMIC_FIELDS,
     name="release_dynamic_fields",
 )
 
@@ -763,10 +736,12 @@ def urls(self):
             _urls["Download"] = self.download_url
 
         for name, url in self.project_urls.items():
-            # avoid duplicating homepage/download links in case the same
-            # url is specified in the pkginfo twice (in the Home-page
-            # or Download-URL field and again in the Project-URL fields)
-            comp_name = name.casefold().replace("-", "").replace("_", "")
+            # NOTE: This avoids duplicating the homepage and download URL links
+            # if they're present both as project URLs and as standalone fields.
+            # The deduplication is done with the project label normalization rules
+            # adopted with PEP 753.
+            # See https://peps.python.org/pep-0753/
+            comp_name = metadata.normalize_project_url_label(name)
             if comp_name == "homepage" and url == _urls.get("Homepage"):
                 continue
             if comp_name == "downloadurl" and url == _urls.get("Download"):