Skip to content

Commit b5f2d06

Browse files
f0rmigaf0rmiga
authored
fix: pip_compile to handle multiple generated requirements.in (#909)
* fix: pip_compile to handle multiple generated requirements.in Signed-off-by: Thulio Ferraz Assis <[email protected]> * fix: only use requirements.in absolute when generated Signed-off-by: Thulio Ferraz Assis <[email protected]> * fix: use hermetic toolchain to lock the requirements Signed-off-by: Thulio Ferraz Assis <[email protected]> * fix: tests on CI Signed-off-by: Thulio Ferraz Assis <[email protected]> * fix: windows path separator madness Signed-off-by: Thulio Ferraz Assis <[email protected]> Signed-off-by: Thulio Ferraz Assis <[email protected]>
1 parent 518c873 commit b5f2d06

File tree

11 files changed

+107
-29
lines changed

11 files changed

+107
-29
lines changed

‎.bazelci/presubmit.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,22 @@ tasks:
167167
working_directory: examples/pip_repository_annotations
168168
platform: windows
169169

170+
integration_test_compile_pip_requirements_linux:
171+
<<: *reusable_build_test_all
172+
name: compile_pip_requirements integration tests on Linux
173+
working_directory: tests/compile_pip_requirements
174+
platform: ubuntu2004
175+
integration_test_compile_pip_requirements_macos:
176+
<<: *reusable_build_test_all
177+
name: compile_pip_requirements integration tests on macOS
178+
working_directory: tests/compile_pip_requirements
179+
platform: macos
180+
integration_test_compile_pip_requirements_windows:
181+
<<: *reusable_build_test_all
182+
name: compile_pip_requirements integration tests on Windows
183+
working_directory: tests/compile_pip_requirements
184+
platform: windows
185+
170186
integration_test_pip_repository_entry_points_linux:
171187
<<: *reusable_build_test_all
172188
name: pip_repository_entry_points integration tests on Linux

‎.bazelrc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
# This lets us glob() up all the files inside the examples to make them inputs to tests
44
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
55
# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
6-
build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
7-
query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
6+
build --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps
7+
query --deleted_packages=examples/build_file_generation,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps
88

99
test --test_output=errors
1010

‎examples/pip_parse/BUILD‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,16 +55,6 @@ alias(
5555
actual = entry_point("yamllint"),
5656
)
5757

58-
# The requirements.in file can be checked in to the source tree or it can be
59-
# generated. Pretend that we do some generating of the file. For this example,
60-
# the "template" is already the file we want.
61-
genrule(
62-
name = "generate_requirements_in",
63-
srcs = ["requirements.in.template"],
64-
outs = ["requirements.in"],
65-
cmd = "cp $(SRCS) $(OUTS)",
66-
)
67-
6858
# This rule adds a convenient way to update the requirements file.
6959
compile_pip_requirements(
7060
name = "requirements",

‎examples/pip_parse/requirements.in.template‎ renamed to ‎examples/pip_parse/requirements.in‎

File renamed without changes.

‎python/pip_install/pip_compile.py‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"Set defaults for the pip-compile command to run it under Bazel"
22

33
import os
4+
import re
45
import sys
56
from pathlib import Path
67
from shutil import copyfile
@@ -26,19 +27,17 @@ def _select_golden_requirements_file(
2627
return requirements_txt
2728

2829

29-
def _fix_up_requirements_in_path(
30-
resolved_requirements_in, requirements_in, output_file
31-
):
30+
def _fix_up_requirements_in_path(absolute_prefix, output_file):
3231
"""Fix up references to the input file inside of the generated requirements file.
3332
3433
We don't want fully resolved, absolute paths in the generated requirements file.
3534
The paths could differ for every invocation. Replace them with a predictable path.
3635
"""
3736
output_file = Path(output_file)
38-
fixed_requirements_text = output_file.read_text().replace(
39-
resolved_requirements_in, requirements_in
40-
)
41-
output_file.write_text(fixed_requirements_text)
37+
contents = output_file.read_text()
38+
contents = contents.replace(absolute_prefix, "")
39+
contents = re.sub(r"\\(?!(\n|\r\n))", "/", contents)
40+
output_file.write_text(contents)
4241

4342

4443
if __name__ == "__main__":
@@ -58,9 +57,11 @@ def _fix_up_requirements_in_path(
5857
requirements_windows = parse_str_none(sys.argv.pop(1))
5958
update_target_label = sys.argv.pop(1)
6059

61-
# The requirements_in file could be generated. We need to get the path to it before we change
62-
# directory into the workspace directory.
63-
resolved_requirements_in = str(Path(requirements_in).resolve())
60+
# The requirements_in file could be generated, so we will need to remove the
61+
# absolute prefixes in the locked requirements output file.
62+
requirements_in_path = Path(requirements_in)
63+
resolved_requirements_in = str(requirements_in_path.resolve())
64+
absolute_prefix = resolved_requirements_in[: -len(str(requirements_in_path))]
6465

6566
# Before loading click, set the locale for its parser.
6667
# If it leaks through to the system setting, it may fail:
@@ -116,17 +117,17 @@ def _fix_up_requirements_in_path(
116117
sys.argv.append("--generate-hashes")
117118
sys.argv.append("--output-file")
118119
sys.argv.append(requirements_txt if UPDATE else requirements_out)
119-
sys.argv.append(resolved_requirements_in)
120+
sys.argv.append(
121+
requirements_in if requirements_in_path.exists() else resolved_requirements_in
122+
)
120123

121124
if UPDATE:
122125
print("Updating " + requirements_txt)
123126
try:
124127
cli()
125128
except SystemExit as e:
126129
if e.code == 0:
127-
_fix_up_requirements_in_path(
128-
resolved_requirements_in, requirements_in, requirements_txt
129-
)
130+
_fix_up_requirements_in_path(absolute_prefix, requirements_txt)
130131
raise
131132
else:
132133
# cli will exit(0) on success
@@ -145,9 +146,7 @@ def _fix_up_requirements_in_path(
145146
)
146147
sys.exit(1)
147148
elif e.code == 0:
148-
_fix_up_requirements_in_path(
149-
resolved_requirements_in, requirements_in, requirements_out
150-
)
149+
_fix_up_requirements_in_path(absolute_prefix, requirements_out)
151150
golden_filename = _select_golden_requirements_file(
152151
requirements_txt,
153152
requirements_linux,

‎tests/compile_pip_requirements/.bazelrc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
test --test_output=errors
2+
3+
# Windows requires these for multi-python support:
4+
build --enable_runfiles
5+
startup --windows_enable_symlinks

‎tests/compile_pip_requirements/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bazel-*

‎tests/compile_pip_requirements/BUILD.bazel‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
2+
3+
genrule(
4+
name = "generate_requirements_extra_in",
5+
srcs = [],
6+
outs = ["requirements_extra.in"],
7+
cmd = "echo 'setuptools~=65.6.3' > $@",
8+
)
9+
10+
genrule(
11+
name = "generate_requirements_in",
12+
srcs = [],
13+
outs = ["requirements.in"],
14+
cmd = """
15+
cat > $@ <<EOF
16+
-r requirements_extra.in
17+
pip~=22.3.1
18+
EOF
19+
""",
20+
)
21+
22+
compile_pip_requirements(
23+
name = "pip_dependencies",
24+
data = [
25+
"requirements_extra.in",
26+
],
27+
extra_args = [
28+
"--allow-unsafe",
29+
"--resolver=backtracking",
30+
],
31+
requirements_in = "requirements.in",
32+
requirements_txt = "requirements_lock.txt",
33+
)

‎tests/compile_pip_requirements/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# compile_pip_requirements
2+
3+
This test ensures that generated requirements.in can be used by compile_pip_requirements.

‎tests/compile_pip_requirements/WORKSPACE‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
local_repository(
2+
name = "rules_python",
3+
path = "../..",
4+
)
5+
6+
load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
7+
8+
pip_install_dependencies()
9+
10+
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
11+
12+
python_register_toolchains(
13+
name = "python39",
14+
python_version = "3.9",
15+
)

0 commit comments

Comments
 (0)