diff --git a/ci/raydepsets/BUILD.bazel b/ci/raydepsets/BUILD.bazel index fa37cf6184ab..2e0a3eed4160 100644 --- a/ci/raydepsets/BUILD.bazel +++ b/ci/raydepsets/BUILD.bazel @@ -19,8 +19,9 @@ py_library( ":workspace", ci_require("bazel-runfiles"), ci_require("click"), - ci_require("pyyaml"), ci_require("networkx"), + ci_require("pip_requirements_parser"), + ci_require("pyyaml"), ], ) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index c3f6ea03b34e..43938365a7b2 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -12,6 +12,7 @@ import click import runfiles from networkx import DiGraph, ancestors as networkx_ancestors, topological_sort +from pip_requirements_parser import RequirementsFile from ci.raydepsets.workspace import Depset, Workspace @@ -494,6 +495,22 @@ def _override_uv_flags(flags: List[str], args: List[str]) -> List[str]: return new_args + _flatten_flags(flags) +def parse_lock_file(lock_file_path: str) -> RequirementsFile: + """ + Parses a lock file and returns a RequirementsFile object, which contains + all information from the file, including requirements, options, and comments. + """ + return RequirementsFile.from_file(lock_file_path) + + +def write_lock_file(requirements_file: RequirementsFile, lock_file_path: str): + """ + Writes a RequirementsFile object to a lock file, preserving all its content. + """ + with open(lock_file_path, "w") as f: + f.write(requirements_file.dumps()) + + def _uv_binary(): """Get the path to the uv binary for the current platform.""" r = runfiles.Create() diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index ed3de694bd83..58f95a1db53c 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -20,6 +20,8 @@ _override_uv_flags, _uv_binary, build, + parse_lock_file, + write_lock_file, ) from ci.raydepsets.tests.utils import ( append_to_file, @@ -885,6 +887,93 @@ def test_build_all_configs(self): assert len(manager.build_graph.nodes) == 12 assert len(manager.build_graph.edges) == 8 + def test_parse_lock_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text( + "emoji==2.9.0 \\\n" + " --hash=sha256:abc123\n" + "pyperclip==1.6.0 \\\n" + " --hash=sha256:def456\n" + ) + rf = parse_lock_file(str(lock_file)) + names = [req.name for req in rf.requirements] + assert "emoji" in names + assert "pyperclip" in names + assert len(rf.requirements) == 2 + + def test_parse_lock_file_with_index_url(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text( + "--index-url https://pypi.org/simple\n" + "\n" + "emoji==2.9.0 \\\n" + " --hash=sha256:abc123\n" + ) + rf = parse_lock_file(str(lock_file)) + assert len(rf.requirements) == 1 + assert rf.requirements[0].name == "emoji" + + def test_parse_lock_file_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text("") + rf = parse_lock_file(str(lock_file)) + assert len(rf.requirements) == 0 + + def test_parse_lock_file_comments_only(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text("# This is a comment\n# Another comment\n") + rf = parse_lock_file(str(lock_file)) + assert len(rf.requirements) == 0 + + def test_write_lock_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text( + "emoji==2.9.0 \\\n" + " --hash=sha256:abc123\n" + "pyperclip==1.6.0 \\\n" + " --hash=sha256:def456\n" + ) + rf = parse_lock_file(str(lock_file)) + + output_file = Path(tmpdir) / "output.txt" + write_lock_file(rf, str(output_file)) + + output_text = output_file.read_text() + assert "emoji==2.9.0" in output_text + assert "pyperclip==1.6.0" in output_text + + def test_write_lock_file_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "empty.txt" + lock_file.write_text("") + rf = parse_lock_file(str(lock_file)) + + output_file = Path(tmpdir) / "output.txt" + write_lock_file(rf, str(output_file)) + assert output_file.read_text().strip() == "" + + def test_roundtrip_preserves_packages(self): + with tempfile.TemporaryDirectory() as tmpdir: + lock_file = Path(tmpdir) / "requirements.txt" + lock_file.write_text( + "emoji==2.9.0 \\\n" + " --hash=sha256:abc123\n" + "pyperclip==1.6.0 \\\n" + " --hash=sha256:def456\n" + ) + rf = parse_lock_file(str(lock_file)) + + output_file = Path(tmpdir) / "output.txt" + write_lock_file(rf, str(output_file)) + + rf2 = parse_lock_file(str(output_file)) + assert rf.dumps() == rf2.dumps() + if __name__ == "__main__": sys.exit(pytest.main(["-vvv", __file__])) diff --git a/release/requirements_py310.in b/release/requirements_py310.in index b6247f64a9c6..ba5bc311820c 100644 --- a/release/requirements_py310.in +++ b/release/requirements_py310.in @@ -25,6 +25,7 @@ aws_requests_auth tzdata tqdm networkx +pip_requirements_parser -r requirements-doc.txt # Upgrades diff --git a/release/requirements_py310.txt b/release/requirements_py310.txt index 627e1d83b859..0a8fd02cef26 100644 --- a/release/requirements_py310.txt +++ b/release/requirements_py310.txt @@ -462,9 +462,9 @@ docutils==0.20.1 \ # sphinx # sphinx-click # sphinx-jsonschema -exceptiongroup==1.2.1 \ - --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ - --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 +exceptiongroup==1.3.1 \ + --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \ + --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598 # via # anyio # ipython @@ -1190,6 +1190,7 @@ packaging==25.0 \ # -r release/requirements_py310.in # anyscale # ipykernel + # pip-requirements-parser # pydata-sphinx-theme # pytest # sphinx @@ -1206,6 +1207,10 @@ pexpect==4.9.0 \ --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \ --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f # via ipython +pip-requirements-parser==32.0.1 \ + --hash=sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526 \ + --hash=sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3 + # via -r release/requirements_py310.in platformdirs==4.2.2 \ --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 @@ -1448,7 +1453,9 @@ pynacl==1.5.0 \ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 - # via httplib2 + # via + # httplib2 + # pip-requirements-parser pytest==8.2.0 \ --hash=sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233 \ --hash=sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f @@ -1628,9 +1635,9 @@ referencing==0.35.1 \ # via # jsonschema # jsonschema-specifications -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf # via # -r release/requirements-doc.txt # -r release/requirements_py310.in @@ -1965,9 +1972,54 @@ toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f # via jupytext -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f +tomli==2.4.0 \ + --hash=sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729 \ + --hash=sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b \ + --hash=sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d \ + --hash=sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df \ + --hash=sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576 \ + --hash=sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d \ + --hash=sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1 \ + --hash=sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a \ + --hash=sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e \ + --hash=sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc \ + --hash=sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702 \ + --hash=sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6 \ + --hash=sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd \ + --hash=sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4 \ + --hash=sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776 \ + --hash=sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a \ + --hash=sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66 \ + --hash=sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87 \ + --hash=sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2 \ + --hash=sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f \ + --hash=sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475 \ + --hash=sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f \ + --hash=sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95 \ + --hash=sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9 \ + --hash=sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3 \ + --hash=sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9 \ + --hash=sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76 \ + --hash=sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da \ + --hash=sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8 \ + --hash=sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51 \ + --hash=sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86 \ + --hash=sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8 \ + --hash=sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0 \ + --hash=sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b \ + --hash=sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1 \ + --hash=sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e \ + --hash=sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d \ + --hash=sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c \ + --hash=sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867 \ + --hash=sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a \ + --hash=sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c \ + --hash=sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0 \ + --hash=sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4 \ + --hash=sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614 \ + --hash=sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132 \ + --hash=sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa \ + --hash=sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087 # via # pytest # sphinx @@ -2018,6 +2070,7 @@ typing-extensions==4.11.0 \ # azure-core # azure-identity # azure-storage-blob + # exceptiongroup # myst-nb # pydantic # pydantic-core