[deps] raydepsets: lockfile parser

ci/raydepsets/BUILD.bazel

@@ -9,6 +9,14 @@ py_library(
     ],
 )
 
+py_library(
+    name = "parser",
+    srcs = ["parser.py"],
+    deps = [
+        ci_require("pip_requirements_parser"),
+    ],
+)
+
 py_library(
     name = "raydepsets_lib",
     srcs = [
@@ -66,6 +74,19 @@ py_library(
     ],
 )
 
+py_test(
+    name = "test_parser",
+    srcs = ["tests/test_parser.py"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ci_require("pytest"),
+        ":parser",
+    ],
+)
+
 py_test(
     name = "test_workspace",
     srcs = ["tests/test_workspace.py"],

ci/raydepsets/parser.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from pip_requirements_parser import Requirement, RequirementsFile
+
+
+def parse_lock_file(lock_file_path: str):
+    rf = RequirementsFile.from_file(lock_file_path)
+    return rf.requirements
+
+
+def write_lock_file(requirements: List[Requirement], lock_file_path: str):
+    with open(lock_file_path, "w") as f:
+        for req in requirements:
+            f.write(req.requirement_line.line + "\n")

ci/raydepsets/tests/test_parser.py

@@ -0,0 +1,102 @@
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from ci.raydepsets.parser import parse_lock_file, write_lock_file
+
+
+def test_parse_lock_file():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text(
+            "emoji==2.9.0 \\\n"
+            "    --hash=sha256:abc123\n"
+            "pyperclip==1.6.0 \\\n"
+            "    --hash=sha256:def456\n"
+        )
+        reqs = parse_lock_file(str(lock_file))
+        names = [req.name for req in reqs]
+        assert "emoji" in names
+        assert "pyperclip" in names
+        assert len(reqs) == 2
+
+
+def test_parse_lock_file_with_index_url():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text(
+            "--index-url https://pypi.org/simple\n"
+            "\n"
+            "emoji==2.9.0 \\\n"
+            "    --hash=sha256:abc123\n"
+        )
+        reqs = parse_lock_file(str(lock_file))
+        assert len(reqs) == 1
+        assert reqs[0].name == "emoji"
+
+
+def test_parse_lock_file_empty():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text("")
+        reqs = parse_lock_file(str(lock_file))
+        assert len(reqs) == 0
+
+
+def test_parse_lock_file_comments_only():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text("# This is a comment\n" "# Another comment\n")
+        reqs = parse_lock_file(str(lock_file))
+        assert len(reqs) == 0
+
+
+def test_write_requirements_file():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text(
+            "emoji==2.9.0 \\\n"
+            "    --hash=sha256:abc123\n"
+            "pyperclip==1.6.0 \\\n"
+            "    --hash=sha256:def456\n"
+        )
+        reqs = parse_lock_file(str(lock_file))
+
+        output_file = Path(tmpdir) / "output.txt"
+        write_lock_file(reqs, str(output_file))
+
+        output_text = output_file.read_text()
+        assert "emoji==2.9.0" in output_text
+        assert "pyperclip==1.6.0" in output_text
+
+
+def test_write_requirements_file_empty():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_file = Path(tmpdir) / "output.txt"
+        write_lock_file([], str(output_file))
+        assert output_file.read_text() == ""
+
+
+def test_roundtrip_preserves_packages():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        lock_file = Path(tmpdir) / "requirements.txt"
+        lock_file.write_text(
+            "emoji==2.9.0 \\\n"
+            "    --hash=sha256:abc123\n"
+            "pyperclip==1.6.0 \\\n"
+            "    --hash=sha256:def456\n"
+        )
+        reqs = parse_lock_file(str(lock_file))
+
+        output_file = Path(tmpdir) / "output.txt"
+        write_lock_file(reqs, str(output_file))
+
+        reqs2 = parse_lock_file(str(output_file))
+        assert len(reqs) == len(reqs2)
+        assert [r.name for r in reqs] == [r.name for r in reqs2]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/requirements_py310.in

@@ -25,6 +25,7 @@ aws_requests_auth
 tzdata
 tqdm
 networkx
+pip_requirements_parser
 -r requirements-doc.txt
 
 # Upgrades

release/requirements_py310.txt

@@ -462,9 +462,9 @@ docutils==0.20.1 \
     #   sphinx
     #   sphinx-click
     #   sphinx-jsonschema
-exceptiongroup==1.2.1 \
-    --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \
-    --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16
+exceptiongroup==1.3.1 \
+    --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \
+    --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598
     # via
     #   anyio
     #   ipython
@@ -1190,6 +1190,7 @@ packaging==25.0 \
     #   -r release/requirements_py310.in
     #   anyscale
     #   ipykernel
+    #   pip-requirements-parser
     #   pydata-sphinx-theme
     #   pytest
     #   sphinx
@@ -1206,6 +1207,10 @@ pexpect==4.9.0 \
     --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \
     --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f
     # via ipython
+pip-requirements-parser==32.0.1 \
+    --hash=sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526 \
+    --hash=sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3
+    # via -r release/requirements_py310.in
 platformdirs==4.2.2 \
     --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \
     --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3
@@ -1448,7 +1453,9 @@ pynacl==1.5.0 \
 pyparsing==3.1.2 \
     --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \
     --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742
-    # via httplib2
+    # via
+    #   httplib2
+    #   pip-requirements-parser
 pytest==8.2.0 \
     --hash=sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233 \
     --hash=sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f
@@ -1628,9 +1635,9 @@ referencing==0.35.1 \
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.32.3 \
-    --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
-    --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+requests==2.32.5 \
+    --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \
+    --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf
     # via
     #   -r release/requirements-doc.txt
     #   -r release/requirements_py310.in
@@ -1965,9 +1972,54 @@ toml==0.10.2 \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
     --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
     # via jupytext
-tomli==2.0.1 \
-    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
-    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+tomli==2.4.0 \
+    --hash=sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729 \
+    --hash=sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b \
+    --hash=sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d \
+    --hash=sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df \
+    --hash=sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576 \
+    --hash=sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d \
+    --hash=sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1 \
+    --hash=sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a \
+    --hash=sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e \
+    --hash=sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc \
+    --hash=sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702 \
+    --hash=sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6 \
+    --hash=sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd \
+    --hash=sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4 \
+    --hash=sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776 \
+    --hash=sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a \
+    --hash=sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66 \
+    --hash=sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87 \
+    --hash=sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2 \
+    --hash=sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f \
+    --hash=sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475 \
+    --hash=sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f \
+    --hash=sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95 \
+    --hash=sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9 \
+    --hash=sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3 \
+    --hash=sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9 \
+    --hash=sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76 \
+    --hash=sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da \
+    --hash=sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8 \
+    --hash=sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51 \
+    --hash=sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86 \
+    --hash=sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8 \
+    --hash=sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0 \
+    --hash=sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b \
+    --hash=sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1 \
+    --hash=sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e \
+    --hash=sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d \
+    --hash=sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c \
+    --hash=sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867 \
+    --hash=sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a \
+    --hash=sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c \
+    --hash=sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0 \
+    --hash=sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4 \
+    --hash=sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614 \
+    --hash=sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132 \
+    --hash=sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa \
+    --hash=sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087
     # via
     #   pytest
     #   sphinx
@@ -2018,6 +2070,7 @@ typing-extensions==4.11.0 \
     #   azure-core
     #   azure-identity
     #   azure-storage-blob
+    #   exceptiongroup
     #   myst-nb
     #   pydantic
     #   pydantic-core