Dedupe packages in requirements-build.txt

Author: syedriko

requirements-build.txt

@@ -42,21 +42,18 @@ hatch-vcs==0.5.0
     #   platformdirs
     #   termcolor
     #   urllib3
-hatchling==1.26.3
+hatchling==1.29.0
     # via
     #   hatch-fancy-pypi-readme
     #   llama-stack-client
     #   openai
-hatchling==1.29.0
-    # via
     #   attrs
     #   banks
     #   chardet
     #   einops
     #   expandvars
     #   filelock
     #   fsspec
-    #   hatch-fancy-pypi-readme
     #   hatch-vcs
     #   jsonschema
     #   latex2mathml
@@ -130,9 +127,6 @@ setuptools-scm==10.0.5
     #   tabulate
     #   tenacity
     #   tqdm
-setuptools-scm==9.2.2
-    # via
-    #   hatch-vcs
     #   urllib3
 tomlkit==0.14.0
     # via uv-dynamic-versioning
@@ -157,9 +151,8 @@ wheel==0.46.3
     #   wrapt
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==82.0.0
-    # via charset-normalizer
 setuptools==82.0.1
+    # via charset-normalizer
     # via
     #   calver
     #   certifi

scripts/dedupe_requirements_build.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Collapse duplicate top-level pins in pip-compile / pybuild-deps output.
+
+``pybuild-deps compile`` can emit multiple ``name==version`` blocks for the same
+distribution when different build paths resolve different versions. pip then
+sees conflicting requirements; we keep the highest version and merge comments.
+"""
+
+from __future__ import annotations
+
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+from packaging.utils import canonicalize_name
+from packaging.version import Version
+
+
+def parse_pin_line(line: str) -> tuple[str, str] | None:
+    """Parse a single ``name==version`` line."""
+    s = line.strip()
+    if not s or s.startswith("#"):
+        return None
+    if "==" not in s:
+        return None
+    name, _, ver = s.partition("==")
+    name, ver = name.strip(), ver.strip()
+    if not name or not ver:
+        return None
+    return name, ver
+
+
+def is_pin_line(line: str) -> bool:
+    """Return True if ``line`` is a top-level pin (not a comment continuation)."""
+    if not line.strip():
+        return False
+    if line[0].isspace():
+        return False
+    return parse_pin_line(line) is not None
+
+
+def split_segments(lines: list[str]) -> list[tuple[str, list[str]]]:
+    """Split into ``('raw', ...)`` or ``('pin', block)`` segments."""
+    out: list[tuple[str, list[str]]] = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if is_pin_line(line):
+            block = [line]
+            i += 1
+            while i < len(lines) and lines[i].startswith((" ", "\t")):
+                block.append(lines[i])
+                i += 1
+            out.append(("pin", block))
+        else:
+            out.append(("raw", [line]))
+            i += 1
+    return out
+
+
+def dedupe_pin_blocks(blocks: list[list[str]]) -> list[str]:
+    """Pick the highest version and merge ``# via`` comment lines from all blocks."""
+
+    def version_key(b: list[str]) -> Version:
+        parsed = parse_pin_line(b[0])
+        assert parsed is not None
+        return Version(parsed[1])
+
+    winner = max(blocks, key=version_key)
+    merged_tail: list[str] = []
+    seen: set[str] = set()
+    for b in blocks:
+        for line in b[1:]:
+            key = line.strip()
+            if key and key not in seen:
+                seen.add(key)
+                merged_tail.append(line)
+    return [winner[0], *merged_tail]
+
+
+def dedupe_file(path: Path) -> None:
+    """Rewrite *path* in place with duplicate pins collapsed."""
+    lines = path.read_text().splitlines(keepends=True)
+    segments = split_segments(lines)
+    by_name: dict[str, list[list[str]]] = defaultdict(list)
+    for kind, payload in segments:
+        if kind != "pin":
+            continue
+        parsed = parse_pin_line(payload[0])
+        assert parsed is not None
+        name, _ = parsed
+        by_name[canonicalize_name(name)].append(payload)
+
+    deduped: dict[str, list[str]] = {
+        k: dedupe_pin_blocks(v) for k, v in by_name.items()
+    }
+
+    out: list[str] = []
+    seen_pin: set[str] = set()
+    for kind, payload in segments:
+        if kind == "raw":
+            out.extend(payload)
+            continue
+        parsed = parse_pin_line(payload[0])
+        assert parsed is not None
+        name, _ = parsed
+        canon = canonicalize_name(name)
+        if canon in seen_pin:
+            continue
+        seen_pin.add(canon)
+        out.extend(deduped[canon])
+
+    path.write_text("".join(out))
+
+
+def main() -> None:
+    """CLI: single argument, path to ``requirements-build.txt``."""
+    if len(sys.argv) != 2:
+        print(
+            "usage: dedupe_requirements_build.py <requirements-build.txt>",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+    dedupe_file(Path(sys.argv[1]))
+
+
+if __name__ == "__main__":
+    main()

scripts/konflux_requirements.sh

@@ -91,6 +91,8 @@ uv run pybuild-deps compile --output-file="$BUILD_FILE" "$SOURCE_FILE"
 
 # pin maturin to the version available in the Red Hat registry
 sed -i 's/maturin==[0-9.]*/maturin==1.10.2/' "$BUILD_FILE"
+# pybuild-deps can emit duplicate pins for the same package at different versions
+uv run python scripts/dedupe_requirements_build.py "$BUILD_FILE"
 
 # remove intermediate files
 rm "$RAW_REQ_FILE" "$WHEEL_FILE" "$WHEEL_FILE_PYPI" "$SOURCE_FILE"