feat: support pyproject.toml in compile_pip_requirements (#1519)

aignas · rickeylev · web-flow · commit cb56a0fcbeda · 2023-11-03T00:25:41.000Z
With this PR we can also use `pyproject.toml` in addition
to `requirements.in` which helps in making the requirements
in a more structured form. For example, we could parse the
toml itself and create aliases in the hub repos only for the
packages outlined in the `pyproject.toml` file. The same for
`gazelle`, we could restrict `gazelle_python.yaml` contents
to only the dependencies listed in `pyproject.toml`.

Examples can be migrated once we agree on the interface.

Summary:
- feat: support pyproject.toml in compile_pip_requirements
- chore: use pyproject.toml for sphinx doc requirements

---------

Co-authored-by: Richard Levasseur &lt;rlevasseur@google.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -48,6 +48,10 @@ A brief description of the categories of changes:
   default, which will cause `gazelle` to change third-party dependency labels
   from `@pip_foo//:pkg` to `@pip//foo` by default.
 
+* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `src`
+  or `requirements_in` attributes are unspecified, matching the upstream
+  `pip-compile` behaviour more closely.
+
 Breaking changes:
 
 * (pip) `pip_install` repository rule in this release has been disabled and
@@ -76,6 +80,9 @@ Breaking changes:
 * (bzlmod) Added `.whl` patching support via `patches` and `patch_strip`
   arguments to the new `pip.override` tag class.
 
+* (pip) Support for using [PEP621](https://peps.python.org/pep-0621/) compliant
+  `pyproject.toml` for creating a resolved `requirements.txt` file.
+
 ## [0.26.0] - 2023-10-06
 
 ### Changed
diff --git a/docs/sphinx/BUILD.bazel b/docs/sphinx/BUILD.bazel
@@ -102,8 +102,8 @@ sphinx_build_binary(
 # Run bazel run //docs/sphinx:requirements.update
 compile_pip_requirements(
     name = "requirements",
+    src = "pyproject.toml",
     requirements_darwin = "requirements_darwin.txt",
-    requirements_in = "requirements.in",
     requirements_txt = "requirements_linux.txt",
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
 )
diff --git a/docs/sphinx/pyproject.toml b/docs/sphinx/pyproject.toml
@@ -0,0 +1,12 @@
+[project]
+name = "rules_python_docs"
+version = "0.0.0"
+
+dependencies = [
+    # NOTE: This is only used as input to create the resolved requirements.txt
+    # file, which is what builds, both Bazel and Readthedocs, both use.
+    "sphinx",
+    "myst-parser",
+    "sphinx_rtd_theme",
+    "readthedocs-sphinx-ext",
+]
diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in
diff --git a/docs/sphinx/requirements_darwin.txt b/docs/sphinx/requirements_darwin.txt
@@ -184,7 +184,7 @@ mdurl==0.1.2 \
 myst-parser==1.0.0 \
     --hash=sha256:502845659313099542bd38a2ae62f01360e7dd4b1310f025dd014dfc0439cdae \
     --hash=sha256:69fb40a586c6fa68995e6521ac0a525793935db7e724ca9bac1d33be51be9a4c
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 packaging==23.0 \
     --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
     --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
@@ -240,7 +240,7 @@ pyyaml==6.0 \
 readthedocs-sphinx-ext==2.2.3 \
     --hash=sha256:6583c26791a5853ee9e57ce9db864e2fb06808ba470f805d74d53fc50811e012 \
     --hash=sha256:e9d911792789b88ae12e2be94d88c619f89a4fa1fe9e42c1505c9930a07163d8
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 requests==2.31.0 \
     --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
     --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
@@ -255,14 +255,14 @@ sphinx==6.1.3 \
     --hash=sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2 \
     --hash=sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc
     # via
-    #   -r docs/sphinx/requirements.in
     #   myst-parser
+    #   rules-python-docs (docs/sphinx/pyproject.toml)
     #   sphinx-rtd-theme
     #   sphinxcontrib-jquery
 sphinx-rtd-theme==1.2.0 \
     --hash=sha256:a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8 \
     --hash=sha256:f823f7e71890abe0ac6aaa6013361ea2696fc8d3e1fa798f463e82bdb77eeff2
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 sphinxcontrib-applehelp==1.0.4 \
     --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
     --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
diff --git a/docs/sphinx/requirements_linux.txt b/docs/sphinx/requirements_linux.txt
@@ -184,7 +184,7 @@ mdurl==0.1.2 \
 myst-parser==1.0.0 \
     --hash=sha256:502845659313099542bd38a2ae62f01360e7dd4b1310f025dd014dfc0439cdae \
     --hash=sha256:69fb40a586c6fa68995e6521ac0a525793935db7e724ca9bac1d33be51be9a4c
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 packaging==23.0 \
     --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
     --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
@@ -240,7 +240,7 @@ pyyaml==6.0 \
 readthedocs-sphinx-ext==2.2.3 \
     --hash=sha256:6583c26791a5853ee9e57ce9db864e2fb06808ba470f805d74d53fc50811e012 \
     --hash=sha256:e9d911792789b88ae12e2be94d88c619f89a4fa1fe9e42c1505c9930a07163d8
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 requests==2.31.0 \
     --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
     --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
@@ -255,14 +255,14 @@ sphinx==6.1.3 \
     --hash=sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2 \
     --hash=sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc
     # via
-    #   -r docs/sphinx/requirements.in
     #   myst-parser
+    #   rules-python-docs (docs/sphinx/pyproject.toml)
     #   sphinx-rtd-theme
     #   sphinxcontrib-jquery
 sphinx-rtd-theme==1.2.0 \
     --hash=sha256:a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8 \
     --hash=sha256:f823f7e71890abe0ac6aaa6013361ea2696fc8d3e1fa798f463e82bdb77eeff2
-    # via -r docs/sphinx/requirements.in
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 sphinxcontrib-applehelp==1.0.4 \
     --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
     --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel
@@ -11,7 +11,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 
 compile_pip_requirements(
     name = "requirements",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock.txt",
     requirements_windows = "requirements_windows.txt",
 )
diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel
@@ -16,7 +16,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 # with pip-compile.
 compile_pip_requirements_3_9(
     name = "requirements_3_9",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_9.txt",
     requirements_windows = "requirements_windows_3_9.txt",
 )
@@ -25,7 +25,7 @@ compile_pip_requirements_3_9(
 # with pip-compile.
 compile_pip_requirements_3_10(
     name = "requirements_3_10",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_10.txt",
     requirements_windows = "requirements_windows_3_10.txt",
 )
diff --git a/examples/bzlmod/other_module/BUILD.bazel b/examples/bzlmod/other_module/BUILD.bazel
@@ -4,6 +4,6 @@ load("@python_versions//3.11:defs.bzl", compile_pip_requirements_311 = "compile_
 # override in the MODULE.bazel.
 compile_pip_requirements_311(
     name = "requirements",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_11.txt",
 )
diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel
@@ -16,7 +16,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 # with pip-compile.
 compile_pip_requirements(
     name = "requirements",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock.txt",
     requirements_windows = "requirements_windows.txt",
 )
diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel
@@ -5,24 +5,24 @@ load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requir
 
 compile_pip_requirements_3_8(
     name = "requirements_3_8",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_8.txt",
 )
 
 compile_pip_requirements_3_9(
     name = "requirements_3_9",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_9.txt",
 )
 
 compile_pip_requirements_3_10(
     name = "requirements_3_10",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_10.txt",
 )
 
 compile_pip_requirements_3_11(
     name = "requirements_3_11",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock_3_11.txt",
 )
diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel
@@ -53,7 +53,7 @@ py_console_script_binary(
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
     requirements_txt = "requirements_lock.txt",
 )
 
diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel
@@ -4,7 +4,10 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
 # This rule adds a convenient way to update the requirements.txt
 # lockfile based on the requirements.in.
-compile_pip_requirements(name = "requirements")
+compile_pip_requirements(
+    name = "requirements",
+    src = "requirements.in",
+)
 
 # The requirements.bzl file is generated with a reference to the interpreter for the host platform.
 # In order to check in a platform-agnostic file, we have to replace that reference with the symbol
diff --git a/examples/pip_repository_annotations/BUILD.bazel b/examples/pip_repository_annotations/BUILD.bazel
@@ -9,6 +9,7 @@ exports_files(
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",
+    src = "requirements.in",
 )
 
 py_test(
diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl
@@ -19,6 +19,7 @@ load("//python/pip_install:repositories.bzl", "requirement")
 
 def compile_pip_requirements(
         name,
+        src = None,
         extra_args = [],
         extra_deps = [],
         generate_hashes = True,
@@ -48,12 +49,17 @@ def compile_pip_requirements(
 
     Args:
         name: base name for generated targets, typically "requirements".
+        src: file containing inputs to dependency resolution. If not specified,
+            defaults to `pyproject.toml`. Supported formats are:
+            * a requirements text file, usually named `requirements.in`
+            * A `.toml` file, where the `project.dependencies` list is used as per
+              [PEP621](https://peps.python.org/pep-0621/).
         extra_args: passed to pip-compile.
         extra_deps: extra dependencies passed to pip-compile.
         generate_hashes: whether to put hashes in the requirements_txt file.
         py_binary: the py_binary rule to be used.
         py_test: the py_test rule to be used.
-        requirements_in: file expressing desired dependencies.
+        requirements_in: file expressing desired dependencies. Deprecated, use src instead.
         requirements_txt: result of "compiling" the requirements.in file.
         requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes.
         requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes.
@@ -62,7 +68,11 @@ def compile_pip_requirements(
         visibility: passed to both the _test and .update rules.
         **kwargs: other bazel attributes passed to the "_test" rule.
     """
-    requirements_in = name + ".in" if requirements_in == None else requirements_in
+    if requirements_in and src:
+        fail("Only one of 'src' and 'requirements_in' attributes can be used")
+    else:
+        src = requirements_in or src or "pyproject.toml"
+
     requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt
 
     # "Default" target produced by this macro
@@ -74,7 +84,7 @@ def compile_pip_requirements(
         visibility = visibility,
     )
 
-    data = [name, requirements_in, requirements_txt] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
+    data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
 
     # Use the Label constructor so this is expanded in the context of the file
     # where it appears, which is to say, in @rules_python
@@ -83,7 +93,7 @@ def compile_pip_requirements(
     loc = "$(rlocationpath {})"
 
     args = [
-        loc.format(requirements_in),
+        loc.format(src),
         loc.format(requirements_txt),
         "//%s:%s.update" % (native.package_name(), name),
         "--resolver=backtracking",
@@ -105,6 +115,7 @@ def compile_pip_requirements(
         requirement("colorama"),
         requirement("importlib_metadata"),
         requirement("more_itertools"),
+        requirement("packaging"),
         requirement("pep517"),
         requirement("pip"),
         requirement("pip_tools"),
diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel
@@ -21,22 +21,22 @@ EOF
 
 compile_pip_requirements(
     name = "requirements",
+    src = "requirements.txt",
     data = [
         "requirements.in",
         "requirements_extra.in",
     ],
-    requirements_in = "requirements.txt",
     requirements_txt = "requirements_lock.txt",
 )
 
 compile_pip_requirements(
     name = "requirements_nohashes",
+    src = "requirements.txt",
     data = [
         "requirements.in",
         "requirements_extra.in",
     ],
     generate_hashes = False,
-    requirements_in = "requirements.txt",
     requirements_txt = "requirements_nohashes_lock.txt",
 )
 
@@ -55,12 +55,12 @@ EOF
 
 compile_pip_requirements(
     name = "os_specific_requirements",
+    src = "requirements_os_specific.in",
     data = [
         "requirements_extra.in",
         "requirements_os_specific.in",
     ],
     requirements_darwin = "requirements_lock_darwin.txt",
-    requirements_in = "requirements_os_specific.in",
     requirements_linux = "requirements_lock_linux.txt",
     requirements_txt = "requirements_lock.txt",
     requirements_windows = "requirements_lock_windows.txt",
diff --git a/tests/pip_repository_entry_points/BUILD.bazel b/tests/pip_repository_entry_points/BUILD.bazel
@@ -5,6 +5,7 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 # This rule adds a convenient way to update the requirements file.
 compile_pip_requirements(
     name = "requirements",
+    src = "requirements.in",
     requirements_windows = ":requirements_windows.txt",
 )
 
diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel
@@ -2,6 +2,7 @@ load("//python:pip.bzl", "compile_pip_requirements")
 
 compile_pip_requirements(
     name = "requirements",
+    src = "requirements.in",
     requirements_darwin = "requirements_darwin.txt",
     requirements_windows = "requirements_windows.txt",
 )

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")`
`11`	`11`
`12`	`12`	`compile_pip_requirements(`
`13`	`13`	`name = "requirements",`
`14`		`- requirements_in = "requirements.in",`
	`14`	`+ src = "requirements.in",`
`15`	`15`	`requirements_txt = "requirements_lock.txt",`
`16`	`16`	`requirements_windows = "requirements_windows.txt",`
`17`	`17`	`)`
Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,6 @@ load("@python_versions//3.11:defs.bzl", compile_pip_requirements_311 = "compile_`
`4`	`4`	`# override in the MODULE.bazel.`
`5`	`5`	`compile_pip_requirements_311(`
`6`	`6`	`name = "requirements",`
`7`		`- requirements_in = "requirements.in",`
	`7`	`+ src = "requirements.in",`
`8`	`8`	`requirements_txt = "requirements_lock_3_11.txt",`
`9`	`9`	`)`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")`
`16`	`16`	`# with pip-compile.`
`17`	`17`	`compile_pip_requirements(`
`18`	`18`	`name = "requirements",`
`19`		`- requirements_in = "requirements.in",`
	`19`	`+ src = "requirements.in",`
`20`	`20`	`requirements_txt = "requirements_lock.txt",`
`21`	`21`	`requirements_windows = "requirements_windows.txt",`
`22`	`22`	`)`