Initial version

This the result of honesty and hdeps testing this code fairly
extensively, but now as a separate project with actual tests.

Author: thatch

README.md

@@ -1,5 +1,11 @@
-# metadata_please
+# metadata\_please
 
+There are a couple of pretty decent ways to read metadata (`importlib-metadata`,
+and `pkginfo`) but they tend to be pretty heavyweight.  This lib aims to do two
+things, with as minimal dependencies as possible:
+
+1. Support just enough metadata to be able to look up deps.
+2. Do "the thing that pip does" when deciding what dist-info dir to look at.
 
 # Version Compat
 
@@ -8,7 +14,7 @@ compatibility) only on 3.10-3.12.  Linting requires 3.12 for full fidelity.
 
 # License
 
-metadata_please is copyright [Tim Hatch](https://timhatch.com/), and licensed under
+metadata\_please is copyright [Tim Hatch](https://timhatch.com/), and licensed under
 the MIT license.  I am providing code in this repository to you under an open
 source license.  This is my personal repository; the license you receive to
 my code is from me and not from my employer. See the `LICENSE` file for details.

metadata_please/__init__.py

@@ -0,0 +1,16 @@
+from .sdist import (
+    basic_metadata_from_tar_sdist,
+    basic_metadata_from_zip_sdist,
+    from_tar_sdist,
+    from_zip_sdist,
+)
+from .wheel import basic_metadata_from_wheel, from_wheel
+
+__all__ = [
+    "basic_metadata_from_tar_sdist",
+    "basic_metadata_from_zip_sdist",
+    "basic_metadata_from_wheel",
+    "from_zip_sdist",
+    "from_tar_sdist",
+    "from_wheel",
+]

metadata_please/sdist.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from tarfile import TarFile
+from zipfile import ZipFile
+
+from .types import BasicMetadata, convert_sdist_requires
+
+
+def from_zip_sdist(zf: ZipFile) -> bytes:
+    """
+    Returns an emulated dist-info metadata contents from the given ZipFile.
+    """
+    requires = [f for f in zf.namelist() if f.endswith("/requires.txt")]
+    requires.sort(key=len)
+    data = zf.read(requires[0])
+    assert data is not None
+    requires, extras = convert_sdist_requires(data.decode("utf-8"))
+
+    buf: list[str] = []
+    for req in requires:
+        buf.append(f"Requires-Dist: {req}\n")
+    for extra in sorted(extras):
+        buf.append(f"Provides-Extra: {extra}\n")
+    return ("".join(buf)).encode("utf-8")
+
+
+def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata:
+    requires = [f for f in zf.namelist() if f.endswith("/requires.txt")]
+    requires.sort(key=len)
+    data = zf.read(requires[0])
+    assert data is not None
+    return BasicMetadata.from_sdist_pkg_info_and_requires(b"", data)
+
+
+def from_tar_sdist(tf: TarFile) -> bytes:
+    """
+    Returns an emulated dist-info metadata contents from the given TarFile.
+    """
+    # XXX Why do ZipFile and TarFile not have a common interface ?!
+    requires = [f for f in tf.getnames() if f.endswith("/requires.txt")]
+    requires.sort(key=len)
+
+    fo = tf.extractfile(requires[0])
+    assert fo is not None
+
+    requires, extras = convert_sdist_requires(fo.read().decode("utf-8"))
+
+    buf: list[str] = []
+    for req in requires:
+        buf.append(f"Requires-Dist: {req}\n")
+    for extra in sorted(extras):
+        buf.append(f"Provides-Extra: {extra}\n")
+    return ("".join(buf)).encode("utf-8")
+
+
+def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata:
+    # XXX Why do ZipFile and TarFile not have a common interface ?!
+    requires = [f for f in tf.getnames() if f.endswith("/requires.txt")]
+    requires.sort(key=len)
+
+    fo = tf.extractfile(requires[0])
+    assert fo is not None
+
+    return BasicMetadata.from_sdist_pkg_info_and_requires(b"", fo.read())

metadata_please/tests/__init__.py

@@ -1,5 +1,8 @@
-# from .foo import FooTest
+from .sdist import TarSdistTest, ZipSdistTest
+from .wheel import WheelTest
 
 __all__ = [
-    # "FooTest",
+    "WheelTest",
+    "ZipSdistTest",
+    "TarSdistTest",
 ]

metadata_please/tests/_tar.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Sequence
+
+
+class MemoryTarFile:
+    def __init__(self, names: Sequence[str], read_value: bytes = b"foo") -> None:
+        self.names = names
+        self.read_value = read_value
+        self.files_read: list[str] = []
+
+    def getnames(self) -> Sequence[str]:
+        return self.names[:]
+
+    def extractfile(self, filename: str) -> BytesIO:
+        return BytesIO(self.read_value)

metadata_please/tests/_zip.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+
+class MemoryZipFile:
+    def __init__(self, names: Sequence[str], read_value: bytes = b"foo") -> None:
+        self.names = names
+        self.read_value = read_value
+        self.files_read: list[str] = []
+
+    def namelist(self) -> Sequence[str]:
+        return self.names[:]
+
+    def read(self, filename: str) -> bytes:
+        self.files_read.append(filename)
+        return self.read_value

metadata_please/tests/sdist.py

@@ -0,0 +1,108 @@
+import unittest
+
+from ..sdist import (
+    basic_metadata_from_tar_sdist,
+    basic_metadata_from_zip_sdist,
+    from_tar_sdist,
+    from_zip_sdist,
+)
+from ._tar import MemoryTarFile
+from ._zip import MemoryZipFile
+
+
+class ZipSdistTest(unittest.TestCase):
+    def test_requires_as_expected(self) -> None:
+        z = MemoryZipFile(
+            ["foo.egg-info/requires.txt", "foo/__init__.py"],
+            read_value=b"""\
+a
+[e]
+b
+""",
+        )
+        metadata = from_zip_sdist(z)  # type: ignore
+        self.assertEqual(
+            b"""\
+Requires-Dist: a
+Requires-Dist: b; extra == 'e'
+Provides-Extra: e
+""",
+            metadata,
+        )
+
+    def test_basic_metadata(self) -> None:
+        z = MemoryZipFile(
+            ["foo.egg-info/requires.txt", "foo/__init__.py"],
+            read_value=b"""\
+a
+[e]
+b
+""",
+        )
+        bm = basic_metadata_from_zip_sdist(z)  # type: ignore
+        self.assertEqual(
+            ["a", "b; extra == 'e'"],
+            bm.reqs,
+        )
+        self.assertEqual({"e"}, bm.provides_extra)
+
+    def test_basic_metadata_absl_py_09(self) -> None:
+        z = MemoryZipFile(
+            ["foo.egg-info/requires.txt", "foo/__init__.py"],
+            read_value=b"""\
+six
+
+[:python_version < "3.4"]
+enum34
+[test:python_version < "3.4"]
+pytest
+""",
+        )
+        bm = basic_metadata_from_zip_sdist(z)  # type: ignore
+        self.assertEqual(
+            [
+                "six",
+                'enum34; python_version < "3.4"',
+                # Quoting on the following line is an implementation detail
+                "pytest; (python_version < \"3.4\") and extra == 'test'",
+            ],
+            bm.reqs,
+        )
+        self.assertEqual({"test"}, bm.provides_extra)
+
+
+class TarSdistTest(unittest.TestCase):
+    def test_requires_as_expected(self) -> None:
+        t = MemoryTarFile(
+            ["foo.egg-info/requires.txt", "foo/__init__.py"],
+            read_value=b"""\
+a
+[e]
+b
+""",
+        )
+        metadata = from_tar_sdist(t)  # type: ignore
+        self.assertEqual(
+            b"""\
+Requires-Dist: a
+Requires-Dist: b; extra == 'e'
+Provides-Extra: e
+""",
+            metadata,
+        )
+
+    def test_basic_metadata(self) -> None:
+        t = MemoryTarFile(
+            ["foo.egg-info/requires.txt", "foo/__init__.py"],
+            read_value=b"""\
+a
+[e]
+b
+""",
+        )
+        bm = basic_metadata_from_tar_sdist(t)  # type: ignore
+        self.assertEqual(
+            ["a", "b; extra == 'e'"],
+            bm.reqs,
+        )
+        self.assertEqual({"e"}, bm.provides_extra)

metadata_please/tests/wheel.py

@@ -0,0 +1,42 @@
+import unittest
+
+from ..wheel import basic_metadata_from_wheel, from_wheel, InvalidWheel
+from ._zip import MemoryZipFile
+
+
+class WheelTest(unittest.TestCase):
+    def test_well_behaved(self) -> None:
+        z = MemoryZipFile(["foo.dist-info/METADATA", "foo/__init__.py"])
+        self.assertEqual(b"foo", from_wheel(z, "foo"))  # type: ignore
+        self.assertEqual(["foo.dist-info/METADATA"], z.files_read)
+
+    def test_actually_empty(self) -> None:
+        z = MemoryZipFile([])
+        with self.assertRaisesRegex(InvalidWheel, "Zero .dist-info dirs in wheel"):
+            from_wheel(z, "foo")  # type: ignore
+
+    def test_no_dist_info(self) -> None:
+        z = MemoryZipFile(["foo/__init__.py"])
+        with self.assertRaisesRegex(InvalidWheel, "Zero .dist-info dirs in wheel"):
+            from_wheel(z, "foo")  # type: ignore
+
+    def test_too_many_dist_info(self) -> None:
+        z = MemoryZipFile(["foo.dist-info/METADATA", "bar.dist-info/METADATA"])
+        with self.assertRaisesRegex(
+            InvalidWheel,
+            r"2 .dist-info dirs where there should be just one: \['bar.dist-info', 'foo.dist-info'\]",
+        ):
+            from_wheel(z, "foo")  # type: ignore
+
+    def test_bad_project_name(self) -> None:
+        z = MemoryZipFile(["foo.dist-info/METADATA", "foo/__init__.py"])
+        with self.assertRaisesRegex(InvalidWheel, "Mismatched foo.dist-info for bar"):
+            from_wheel(z, "bar")  # type: ignore
+
+    def test_basic_metadata(self) -> None:
+        z = MemoryZipFile(
+            ["foo.dist-info/METADATA", "foo/__init__.py"],
+            read_value=b"Requires-Dist: foo\n",
+        )
+        bm = basic_metadata_from_wheel(z, "foo")  # type: ignore
+        self.assertEqual(["foo"], bm.reqs)

metadata_please/types.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from email import message_from_string
+from typing import Sequence
+
+
+@dataclass(frozen=True)
+class BasicMetadata:
+    # Popualted from Requires-Dist or requires.txt
+    reqs: Sequence[str]
+    # Populated from Provides-Extra
+    provides_extra: set[str]
+
+    @classmethod
+    def from_metadata(cls, metadata: bytes) -> BasicMetadata:
+        msg = message_from_string(metadata.decode("utf-8"))
+        return BasicMetadata(
+            msg.get_all("Requires-Dist") or (),
+            set(msg.get_all("Provides-Extra") or ()),
+        )
+
+    @classmethod
+    def from_sdist_pkg_info_and_requires(
+        cls, pkg_info: bytes, requires: bytes
+    ) -> BasicMetadata:
+        # We can either get Provides-Extra from this, or from the section
+        # headers in requires.txt...
+        # msg = message_from_string(pkg_info.decode("utf-8"))
+        return cls(
+            *convert_sdist_requires(requires.decode("utf-8")),
+        )
+
+
+def convert_sdist_requires(data: str) -> tuple[list[str], set[str]]:
+    # This is reverse engineered from looking at a couple examples, but there
+    # does not appear to be a formal spec.  Mentioned at
+    # https://setuptools.readthedocs.io/en/latest/formats.html#requires-txt
+    # This snippet has existed in `honesty` for a couple of years now.
+    current_markers = None
+    extras: set[str] = set()
+    lst: list[str] = []
+    for line in data.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        elif line[:1] == "[" and line[-1:] == "]":
+            current_markers = line[1:-1]
+            if ":" in current_markers:
+                # absl-py==0.9.0 and requests==2.22.0 are good examples of this
+                extra, markers = current_markers.split(":", 1)
+                if extra:
+                    extras.add(extra)
+                    current_markers = f"({markers}) and extra == {extra!r}"
+                else:
+                    current_markers = markers
+            else:
+                # this is an extras_require
+                extras.add(current_markers)
+                current_markers = f"extra == {current_markers!r}"
+        else:
+            if current_markers:
+                lst.append(f"{line}; {current_markers}")
+            else:
+                lst.append(line)
+    return lst, extras