Skip RECORD checks for RECORD.jws and RECORD.p7s (#18503)

* Skip RECORD checks for RECORD.jws and RECORD.p7s

* Linting

---------

Co-authored-by: Dustin Ingram <[email protected]>

Author: sethmlarson

tests/unit/forklift/test_legacy.py

@@ -3641,6 +3641,157 @@ def test_upload_record_does_not_warn_windows_path_separators(
         assert send_email.calls == []
         assert resp.status_code == 200
 
+    @pytest.mark.parametrize("exempt_filename", ["RECORD.jws", "RECORD.p7s"])
+    def test_upload_record_check_does_not_include_jws_p7s(
+        self, monkeypatch, pyramid_config, db_request, exempt_filename
+    ):
+        """
+        Certain filenames are required not to be included in RECORD
+        (RECORD.jws, RECORD.p7s) so we can't warn if they aren't included in
+        the RECORD.
+        """
+
+        user = UserFactory.create()
+        pyramid_config.testing_securitypolicy(identity=user)
+        db_request.user = user
+        db_request.user_agent = "warehouse-tests/6.6.6"
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create()
+        release = ReleaseFactory.create(project=project, version="1.0")
+        RoleFactory.create(user=user, project=project)
+
+        temp_f = io.BytesIO()
+        project_name = project.normalized_name.replace("-", "_")
+        with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
+            zfp.mkdir("some-dir/")  # Directories!
+            zfp.mkdir(f"{project_name}-{release.version}.dist-info/")
+
+            zfp.writestr(f"{project_name}-{release.version}.dist-info/METADATA", "")
+            zfp.writestr(
+                f"{project_name}-{release.version}.dist-info/{exempt_filename}", ""
+            )
+            zfp.writestr(
+                f"{project_name}-{release.version}.dist-info/RECORD",
+                dedent(
+                    f"""\
+                    {project_name}-{release.version}.dist-info/METADATA,
+                    {project_name}-{release.version}.dist-info/RECORD,
+                    """,
+                ),
+            )
+
+        filename = "{}-{}-cp34-none-any.whl".format(
+            project.normalized_name.replace("-", "_"),
+            release.version,
+        )
+        filebody = temp_f.getvalue()
+
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "1.2",
+                "name": project.name,
+                "version": release.version,
+                "filetype": "bdist_wheel",
+                "pyversion": "cp34",
+                "md5_digest": hashlib.md5(filebody).hexdigest(),
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(filebody),
+                    type="application/zip",
+                ),
+            }
+        )
+
+        monkeypatch.setattr(
+            legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
+        )
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+        }.get(svc)
+
+        send_email = pretend.call_recorder(lambda *a, **kw: None)
+        monkeypatch.setattr(legacy, "send_wheel_record_mismatch_email", send_email)
+
+        resp = legacy.file_upload(db_request)
+
+        assert send_email.calls == []
+        assert resp.status_code == 200
+
+    @pytest.mark.parametrize("exempt_filename", ["RECORD.jws", "RECORD.p7s"])
+    def test_upload_record_check_exclusions_only_in_dist_info(
+        self,
+        monkeypatch,
+        pyramid_config,
+        db_request,
+        exempt_filename,
+    ):
+        user = UserFactory.create()
+        pyramid_config.testing_securitypolicy(identity=user)
+        db_request.user = user
+        db_request.user_agent = "warehouse-tests/6.6.6"
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create()
+        release = ReleaseFactory.create(project=project, version="1.0")
+        RoleFactory.create(user=user, project=project)
+
+        temp_f = io.BytesIO()
+        project_name = project.normalized_name.replace("-", "_")
+        with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
+            zfp.writestr("some_file", "some_data")
+            zfp.writestr(f"{project_name}-{release.version}.dist-info/METADATA", "")
+            zfp.writestr(f"not-dist-info/{exempt_filename}", "")
+            zfp.writestr(
+                f"{project_name}-{release.version}.dist-info/RECORD",
+                f"{project_name}-{release.version}.dist-info/RECORD,",
+            )
+
+        filename = "{}-{}-cp34-none-any.whl".format(
+            project.normalized_name.replace("-", "_"),
+            release.version,
+        )
+        filebody = temp_f.getvalue()
+
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "1.2",
+                "name": project.name,
+                "version": release.version,
+                "filetype": "bdist_wheel",
+                "pyversion": "cp34",
+                "md5_digest": hashlib.md5(filebody).hexdigest(),
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(filebody),
+                    type="application/zip",
+                ),
+            }
+        )
+
+        monkeypatch.setattr(
+            legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
+        )
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+        }.get(svc)
+
+        send_email = pretend.call_recorder(lambda *a, **kw: None)
+        monkeypatch.setattr(legacy, "send_wheel_record_mismatch_email", send_email)
+
+        resp = legacy.file_upload(db_request)
+        assert send_email.calls == [
+            pretend.call(
+                db_request,
+                {user},
+                project_name=project.name,
+                filename=filename,
+            ),
+        ]
+        assert resp.status_code == 200
+
     def test_upload_fails_with_missing_metadata_wheel(
         self, monkeypatch, pyramid_config, db_request
     ):

warehouse/forklift/legacy.py

@@ -1482,6 +1482,12 @@ def file_upload(request):
             against the files listed in the RECORD. Mismatches are reported via email.
             """
             record_filename = f"{name}-{version}.dist-info/RECORD"
+            # Files that must be missing from 'RECORD',
+            # so we ignore them when cross-checking.
+            record_exemptions = {
+                f"{name}-{version}.dist-info/RECORD.jws",
+                f"{name}-{version}.dist-info/RECORD.p7s",
+            }
             try:
                 with zipfile.ZipFile(temporary_filename) as zfp:
                     wheel_record_contents = zfp.read(record_filename).decode()
@@ -1490,7 +1496,9 @@ def file_upload(request):
                     for fn, *_ in csv.reader(wheel_record_contents.splitlines())
                 }
                 zip_entries = {
-                    fn for fn in zfp.namelist() if not _zip_filename_is_dir(fn)
+                    fn
+                    for fn in zfp.namelist()
+                    if not _zip_filename_is_dir(fn) and fn not in record_exemptions
                 }
             except (UnicodeError, KeyError, csv.Error) as e:
                 request.metrics.increment(