Skip to content

Commit d29cbe9

Browse files
didi
authored
Warn on upload about non-normalized wheel distribution name (#17378)
* Update tests to use correct wheel filenames * Add a failing test * Warn on upload about non-normalized wheel distribution name * Add note from #17546 * Add missing import * Linting * Add a failing test * Check for all possible permutations * Make the notification email more clear
1 parent d028de2 commit d29cbe9

File tree

11 files changed

+319
-18
lines changed

11 files changed

+319
-18
lines changed

‎tests/unit/email/test_init.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6316,6 +6316,94 @@ def test_environment_ignored_in_trusted_publisher_emails(
63166316
),
63176317
]
63186318

6319+
def test_pep427_emails(
6320+
self,
6321+
pyramid_request,
6322+
pyramid_config,
6323+
monkeypatch,
6324+
):
6325+
stub_user = pretend.stub(
6326+
id="id",
6327+
username="username",
6328+
name="",
6329+
email="[email protected]",
6330+
primary_email=pretend.stub(email="[email protected]", verified=True),
6331+
)
6332+
subject_renderer = pyramid_config.testing_add_renderer(
6333+
"email/pep427-name-email/subject.txt"
6334+
)
6335+
subject_renderer.string_response = "Email Subject"
6336+
body_renderer = pyramid_config.testing_add_renderer(
6337+
"email/pep427-name-email/body.txt"
6338+
)
6339+
body_renderer.string_response = "Email Body"
6340+
html_renderer = pyramid_config.testing_add_renderer(
6341+
"email/pep427-name-email/body.html"
6342+
)
6343+
html_renderer.string_response = "Email HTML Body"
6344+
6345+
send_email = pretend.stub(
6346+
delay=pretend.call_recorder(lambda *args, **kwargs: None)
6347+
)
6348+
pyramid_request.task = pretend.call_recorder(lambda *args, **kwargs: send_email)
6349+
monkeypatch.setattr(email, "send_email", send_email)
6350+
6351+
pyramid_request.db = pretend.stub(
6352+
query=lambda a: pretend.stub(
6353+
filter=lambda *a: pretend.stub(
6354+
one=lambda: pretend.stub(user_id=stub_user.id)
6355+
)
6356+
),
6357+
)
6358+
pyramid_request.user = stub_user
6359+
pyramid_request.registry.settings = {"mail.sender": "[email protected]"}
6360+
6361+
project_name = "Test_Project"
6362+
filename = "Test_Project-1.0-py3-none-any.whl"
6363+
6364+
result = email.send_pep427_name_email(
6365+
pyramid_request,
6366+
stub_user,
6367+
project_name=project_name,
6368+
filename=filename,
6369+
normalized_name="test_project",
6370+
)
6371+
6372+
assert result == {
6373+
"project_name": project_name,
6374+
"normalized_name": "test_project",
6375+
"filename": filename,
6376+
}
6377+
subject_renderer.assert_(project_name=project_name)
6378+
body_renderer.assert_(project_name=project_name)
6379+
html_renderer.assert_(project_name=project_name)
6380+
6381+
assert pyramid_request.task.calls == [pretend.call(send_email)]
6382+
assert send_email.delay.calls == [
6383+
pretend.call(
6384+
f"{stub_user.username} <{stub_user.email}>",
6385+
{
6386+
"sender": None,
6387+
"subject": "Email Subject",
6388+
"body_text": "Email Body",
6389+
"body_html": (
6390+
"<html>\n<head></head>\n"
6391+
"<body><p>Email HTML Body</p></body>\n</html>\n"
6392+
),
6393+
},
6394+
{
6395+
"tag": "account:email:sent",
6396+
"user_id": stub_user.id,
6397+
"additional": {
6398+
"from_": "[email protected]",
6399+
"to": stub_user.email,
6400+
"subject": "Email Subject",
6401+
"redact_ip": False,
6402+
},
6403+
},
6404+
)
6405+
]
6406+
63196407

63206408
class TestUserTermsOfServiceUpdateEmail:
63216409
def test_user_terms_of_service_updated(

‎tests/unit/forklift/test_legacy.py

Lines changed: 108 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from warehouse.classifiers.models import Classifier
4242
from warehouse.forklift import legacy, metadata
4343
from warehouse.macaroons import IMacaroonService, caveats, security_policy
44+
from warehouse.metrics import IMetricsService
4445
from warehouse.oidc.interfaces import SignedClaims
4546
from warehouse.oidc.utils import PublisherTokenContext
4647
from warehouse.packaging.interfaces import IFileStorage, IProjectService
@@ -2762,8 +2763,12 @@ def test_upload_succeeds_with_wheel(
27622763
release = ReleaseFactory.create(project=project, version="1.0")
27632764
RoleFactory.create(user=user, project=project)
27642765

2765-
filename = f"{project.name}-{release.version}-cp34-none-{plat}.whl"
2766-
filebody = _get_whl_testdata(name=project.name, version=release.version)
2766+
filename = "{}-{}-cp34-none-{}.whl".format(
2767+
project.normalized_name.replace("-", "_"), release.version, plat
2768+
)
2769+
filebody = _get_whl_testdata(
2770+
name=project.normalized_name.replace("-", "_"), version=release.version
2771+
)
27672772
filestoragehash = _storage_hash(filebody)
27682773

27692774
pyramid_config.testing_securitypolicy(identity=user)
@@ -3112,15 +3117,20 @@ def test_upload_succeeds_with_wheel_after_sdist(
31123117
EmailFactory.create(user=user)
31133118
project = ProjectFactory.create()
31143119
release = ReleaseFactory.create(project=project, version="1.0")
3115-
FileFactory.create(
3116-
release=release,
3117-
packagetype="sdist",
3118-
filename=f"{project.name}-{release.version}.tar.gz",
3120+
filename = "{}-{}.tar.gz".format(
3121+
project.normalized_name.replace("-", "_"),
3122+
release.version,
31193123
)
3124+
FileFactory.create(release=release, packagetype="sdist", filename=filename)
31203125
RoleFactory.create(user=user, project=project)
31213126

3122-
filename = f"{project.name}-{release.version}-cp34-none-any.whl"
3123-
filebody = _get_whl_testdata(name=project.name, version=release.version)
3127+
filename = "{}-{}-cp34-none-any.whl".format(
3128+
project.normalized_name.replace("-", "_"),
3129+
release.version,
3130+
)
3131+
filebody = _get_whl_testdata(
3132+
name=project.normalized_name.replace("-", "_"), version=release.version
3133+
)
31243134
filestoragehash = _storage_hash(filebody)
31253135

31263136
pyramid_config.testing_securitypolicy(identity=user)
@@ -3363,7 +3373,10 @@ def test_upload_fails_with_missing_metadata_wheel(
33633373
with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
33643374
zfp.writestr("some_file", "some_data")
33653375

3366-
filename = f"{project.name}-{release.version}-cp34-none-any.whl"
3376+
filename = "{}-{}-cp34-none-any.whl".format(
3377+
project.normalized_name.replace("-", "_"),
3378+
release.version,
3379+
)
33673380
filebody = temp_f.getvalue()
33683381

33693382
db_request.POST = MultiDict(
@@ -4888,6 +4901,76 @@ def test_upload_with_token_api_warns_if_trusted_publisher_configured(
48884901
if not warning_already_sent:
48894902
assert not warning_exists
48904903

4904+
@pytest.mark.parametrize("project_name", ["Some_Thing", "some.thing"])
4905+
def test_upload_warns_pep427(
4906+
self,
4907+
monkeypatch,
4908+
pyramid_config,
4909+
db_request,
4910+
metrics,
4911+
project_service,
4912+
macaroon_service,
4913+
project_name,
4914+
):
4915+
project = ProjectFactory.create(name=project_name)
4916+
owner = UserFactory.create()
4917+
maintainer = UserFactory.create()
4918+
RoleFactory.create(user=owner, project=project, role_name="Owner")
4919+
RoleFactory.create(user=maintainer, project=project, role_name="Maintainer")
4920+
4921+
pyramid_config.testing_securitypolicy(identity=owner)
4922+
4923+
filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
4924+
data = _get_whl_testdata(name=project.name, version="1.0")
4925+
digest = hashlib.md5(data).hexdigest()
4926+
4927+
db_request.POST = MultiDict(
4928+
{
4929+
"metadata_version": "1.2",
4930+
"name": project.name,
4931+
"version": "1.0",
4932+
"filetype": "bdist_wheel",
4933+
"pyversion": "py3",
4934+
"md5_digest": digest,
4935+
"content": pretend.stub(
4936+
filename=filename,
4937+
file=io.BytesIO(data),
4938+
type="application/zip",
4939+
),
4940+
}
4941+
)
4942+
4943+
storage_service = pretend.stub(store=lambda path, filepath, meta: None)
4944+
4945+
db_request.find_service = lambda svc, name=None, context=None: {
4946+
IFileStorage: storage_service,
4947+
IMacaroonService: macaroon_service,
4948+
IMetricsService: metrics,
4949+
IProjectService: project_service,
4950+
}.get(svc)
4951+
db_request.user_agent = "warehouse-tests/6.6.6"
4952+
db_request.help_url = pretend.call_recorder(lambda **kw: "/the/help/url/")
4953+
4954+
send_email = pretend.call_recorder(lambda *a, **kw: None)
4955+
monkeypatch.setattr(legacy, "send_pep427_name_email", send_email)
4956+
monkeypatch.setattr(
4957+
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
4958+
)
4959+
4960+
resp = legacy.file_upload(db_request)
4961+
4962+
assert resp.status_code == 200
4963+
4964+
assert send_email.calls == [
4965+
pretend.call(
4966+
db_request,
4967+
{owner, maintainer},
4968+
project_name=project.name,
4969+
filename=filename,
4970+
normalized_name="some_thing",
4971+
),
4972+
]
4973+
48914974
@pytest.mark.parametrize(
48924975
("filename", "function_name", "extra_kwargs"),
48934976
[
@@ -5069,8 +5152,12 @@ def test_upload_succeeds_creates_release_metadata_2_4(
50695152
digest = _ZIP_PKG_MD5
50705153
data = _ZIP_PKG_TESTDATA
50715154
elif filetype == "bdist_wheel":
5072-
filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
5073-
data = _get_whl_testdata(name=project.name, version="1.0")
5155+
filename = "{}-{}-py3-none-any.whl".format(
5156+
project.normalized_name.replace("-", "_"), "1.0"
5157+
)
5158+
data = _get_whl_testdata(
5159+
name=project.normalized_name.replace("-", "_"), version="1.0"
5160+
)
50745161
digest = hashlib.md5(data).hexdigest()
50755162
monkeypatch.setattr(
50765163
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
@@ -5184,13 +5271,20 @@ def test_upload_fails_missing_license_file_metadata_2_4(
51845271
data = _ZIP_PKG_TESTDATA
51855272
license_filename = "fake_package-1.0/LICENSE"
51865273
elif filetype == "bdist_wheel":
5187-
filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
5188-
data = _get_whl_testdata(name=project.name, version="1.0")
5274+
filename = "{}-{}-py3-none-any.whl".format(
5275+
project.normalized_name.replace("-", "_"),
5276+
"1.0",
5277+
)
5278+
data = _get_whl_testdata(
5279+
name=project.normalized_name.replace("-", "_"), version="1.0"
5280+
)
51895281
digest = hashlib.md5(data).hexdigest()
51905282
monkeypatch.setattr(
51915283
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
51925284
)
5193-
license_filename = f"{project.name}-1.0.dist-info/licenses/LICENSE"
5285+
license_filename = "{}-1.0.dist-info/licenses/LICENSE".format(
5286+
project.normalized_name.replace("-", "_")
5287+
)
51945288

51955289
pyramid_config.testing_securitypolicy(identity=user)
51965290
db_request.user = user

‎warehouse/email/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1111,6 +1111,15 @@ def send_user_terms_of_service_updated(request, user):
11111111
}
11121112

11131113

1114+
@_email("pep427-name-email")
1115+
def send_pep427_name_email(request, users, project_name, filename, normalized_name):
1116+
return {
1117+
"project_name": project_name,
1118+
"filename": filename,
1119+
"normalized_name": normalized_name,
1120+
}
1121+
1122+
11141123
def includeme(config):
11151124
email_sending_class = config.maybe_dotted(config.registry.settings["mail.backend"])
11161125
config.register_service_factory(email_sending_class.create_service, IEmailSender)

‎warehouse/forklift/legacy.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB
5353
from warehouse.email import (
5454
send_api_token_used_in_trusted_publisher_project_email,
55+
send_pep427_name_email,
5556
send_pep625_extension_email,
5657
send_pep625_name_email,
5758
send_pep625_version_email,
@@ -1372,6 +1373,25 @@ def file_upload(request):
13721373
f"{canonical_name.replace('-', '_')!r}.",
13731374
)
13741375

1376+
# The parse_wheel_filename function does not enforce lowercasing,
1377+
# and also returns a normalized name, so we must get the original
1378+
# distribution name from the filename manually
1379+
name_from_filename, _ = filename.split("-", 1)
1380+
1381+
# PEP 427 / PEP 503: Enforcement of project name normalization.
1382+
# Filenames that do not start with the fully normalized project name
1383+
# will not be permitted.
1384+
# https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode
1385+
normalized_name = project.normalized_name.replace("-", "_")
1386+
if name_from_filename != normalized_name:
1387+
send_pep427_name_email(
1388+
request,
1389+
set(project.users),
1390+
project_name=project.name,
1391+
filename=filename,
1392+
normalized_name=normalized_name,
1393+
)
1394+
13751395
if meta.version != version:
13761396
request.metrics.increment(
13771397
"warehouse.upload.failed",

‎warehouse/templates/email/pep427-name-email/body.html

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{#
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
-#}
14+
{% extends "email/_base/body.html" %}
15+
16+
{% set domain = request.registry.settings.get('warehouse.domain') %}
17+
18+
{% block content %}
19+
<p>
20+
This email is notifying you of an upcoming deprecation that we have
21+
determined may affect you as a result of your recent upload to
22+
'{{ project_name }}'.
23+
</p>
24+
<p>
25+
In the future, PyPI will require all newly uploaded binary distribution
26+
filenames to comply with the <a href="https://packaging.python.org/en/latest/specifications/binary-distribution-format/">binary distribution format</a>.
27+
Any binary distributions already uploaded will remain in place
28+
as-is and do not need to be updated.
29+
</p>
30+
<p>
31+
Specifically, your recent upload of '{{ filename }}' is incompatible with
32+
the distribution format specification because the filename does not contain
33+
the normalized project name '{{ normalized_name }}'.
34+
</p>
35+
<p>
36+
In most cases, this can be resolved by upgrading the version of your build
37+
tooling to a later version that fully supports the specification and
38+
produces compliant filenames. You do not need to remove the file.
39+
</p>
40+
<p>
41+
If you have questions, you can email
42+
<a href="mailto:[email protected]">[email protected]</a> to communicate with the PyPI
43+
[email protected] to communicate with the PyPI administrators.
44+
</p>
45+
{% endblock %}

‎warehouse/templates/email/pep427-name-email/body.txt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{#
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
-#}
14+
{% extends "email/_base/body.txt" %}
15+
16+
{% block content %}
17+
18+
This email is notifying you of an upcoming deprecation that we have determined may affect you as a result of your recent upload to '{{ project_name }}'.
19+
20+
In the future, PyPI will require all newly uploaded binary distribution filenames to comply with the <a href="https://packaging.python.org/en/latest/specifications/binary-distribution-format/">binary distribution format</a>. Any binary distributions already uploaded will remain in place as-is and do not need to be updated.
21+
22+
Specifically, your recent upload of '{{ filename }}' is incompatible with the distribution format specification because the filename does not contain the normalized project name '{{ normalized_name }}'.
23+
24+
In most cases, this can be resolved by upgrading the version of your build tooling to a later version that fully supports the specification and produces compliant filenames. You do not need to remove the file.
25+
26+
If you have questions, you can email [email protected] to communicate with the PyPI administrators.
27+
28+
{% endblock %}

0 commit comments

Comments
 (0)