Skip to content

Commit 52db4c9

Browse files
committed
Merge branch 'main' into feat/improve-uv-mgmt
2 parents d00ee31 + e7d2f09 commit 52db4c9

File tree

11 files changed

+103
-36
lines changed

11 files changed

+103
-36
lines changed

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ Unreleased changes template.
6767
([#1169](https://github.com/bazelbuild/rules_python/issues/1169)).
6868
* (gazelle) Don't collapse depsets to a list or into args when generating the modules mapping file.
6969
Support spilling modules mapping args into a params file.
70+
* (pypi) From now on `python` invocations in repository and module extension
71+
evaluation contexts will invoke Python interpreter with `-B` to avoid
72+
creating `.pyc` files.
73+
* (deps) doublestar 4.7.1 (required for recent Gazelle versions)
7074

7175
{#v0-0-0-added}
7276
### Added
@@ -78,6 +82,8 @@ Unreleased changes template.
7882
docs but usage of the extension may result in your setup breaking without any
7983
notice. What is more, the URLs and SHA256 values will be retrieved from the
8084
GitHub releases page metadata published by the `uv` project.
85+
* (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when
86+
building wheels from `sdist`.
8187

8288
{#v0-0-0-removed}
8389
### Removed

gazelle/deps.bzl

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@
1414

1515
"This file managed by `bazel run //:gazelle_update_repos`"
1616

17-
load(
18-
"@bazel_gazelle//:deps.bzl",
19-
_go_repository = "go_repository",
20-
)
17+
load("@bazel_gazelle//:deps.bzl", _go_repository = "go_repository")
2118
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
2219

2320
def go_repository(name, **kwargs):
@@ -70,8 +67,8 @@ def go_deps():
7067
go_repository(
7168
name = "com_github_bmatcuk_doublestar_v4",
7269
importpath = "github.com/bmatcuk/doublestar/v4",
73-
sum = "h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=",
74-
version = "v4.6.1",
70+
sum = "h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q=",
71+
version = "v4.7.1",
7572
)
7673

7774
go_repository(

gazelle/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/bazelbuild/bazel-gazelle v0.31.1
77
github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82
88
github.com/bazelbuild/rules_go v0.41.0
9-
github.com/bmatcuk/doublestar/v4 v4.6.1
9+
github.com/bmatcuk/doublestar/v4 v4.7.1
1010
github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b
1111
github.com/emirpasic/gods v1.18.1
1212
github.com/ghodss/yaml v1.0.0

gazelle/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0v
88
github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
99
github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
1010
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
11+
github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q=
12+
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
1113
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
1214
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
1315
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

python/private/pypi/attrs.bzl

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@
1515
"common attributes for whl_library and pip_repository"
1616

1717
ATTRS = {
18+
"add_libdir_to_library_search_path": attr.bool(
19+
default = False,
20+
doc = """
21+
If true, add the lib dir of the bundled interpreter to the library search path via `LDFLAGS`.
22+
23+
:::{versionadded} VERSION_NEXT_FEATURE
24+
:::
25+
""",
26+
),
1827
"download_only": attr.bool(
1928
doc = """
2029
Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of

python/private/pypi/evaluate_markers.bzl

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpre
5555
pypi_repo_utils.execute_checked(
5656
mrctx,
5757
op = "ResolveRequirementEnvMarkers({})".format(in_file),
58+
python = pypi_repo_utils.resolve_python_interpreter(
59+
mrctx,
60+
python_interpreter = python_interpreter,
61+
python_interpreter_target = python_interpreter_target,
62+
),
5863
arguments = [
59-
pypi_repo_utils.resolve_python_interpreter(
60-
mrctx,
61-
python_interpreter = python_interpreter,
62-
python_interpreter_target = python_interpreter_target,
63-
),
6464
"-m",
6565
"python.private.pypi.requirements_parser.resolve_target_platforms",
6666
in_file,

python/private/pypi/extension.bzl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ def _create_whl_repos(
203203
)
204204
maybe_args = dict(
205205
# The following values are safe to omit if they have false like values
206+
add_libdir_to_library_search_path = pip_attr.add_libdir_to_library_search_path,
206207
annotation = whl_modifications.get(whl_name),
207208
download_only = pip_attr.download_only,
208209
enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,

python/private/pypi/patch_whl.bzl

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ other patches ensures that the users have overview on exactly what has changed
2727
within the wheel.
2828
"""
2929

30-
load("//python/private:repo_utils.bzl", "repo_utils")
3130
load(":parse_whl_name.bzl", "parse_whl_name")
31+
load(":pypi_repo_utils.bzl", "pypi_repo_utils")
3232

3333
_rules_python_root = Label("//:BUILD.bazel")
3434

@@ -102,10 +102,14 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
102102
record_patch = rctx.path("RECORD.patch")
103103
whl_patched = patched_whl_name(whl_input.basename)
104104

105-
repo_utils.execute_checked(
105+
pypi_repo_utils.execute_checked(
106106
rctx,
107+
python = python_interpreter,
108+
srcs = [
109+
Label("//python/private/pypi:repack_whl.py"),
110+
Label("//tools:wheelmaker.py"),
111+
],
107112
arguments = [
108-
python_interpreter,
109113
"-m",
110114
"python.private.pypi.repack_whl",
111115
"--record-patch",

python/private/pypi/pypi_repo_utils.bzl

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,30 @@ def _construct_pypath(mrctx, *, entries):
104104
])
105105
return pypath
106106

107-
def _execute_checked(mrctx, *, srcs, **kwargs):
107+
def _execute_prep(mrctx, *, python, srcs, **kwargs):
108+
for src in srcs:
109+
# This will ensure that we will re-evaluate the bzlmod extension or
110+
# refetch the repository_rule when the srcs change. This should work on
111+
# Bazel versions without `mrctx.watch` as well.
112+
repo_utils.watch(mrctx, mrctx.path(src))
113+
114+
environment = kwargs.pop("environment", {})
115+
pythonpath = environment.get("PYTHONPATH", "")
116+
if pythonpath and not types.is_string(pythonpath):
117+
environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
118+
kwargs["environment"] = environment
119+
120+
# -B is added to prevent the repo-phase invocation from creating timestamp
121+
# based pyc files, which contributes to race conditions and non-determinism
122+
kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", [])
123+
return kwargs
124+
125+
def _execute_checked(mrctx, *, python, srcs, **kwargs):
108126
"""Helper function to run a python script and modify the PYTHONPATH to include external deps.
109127
110128
Args:
111129
mrctx: Handle to the module_ctx or repository_ctx.
130+
python: The python interpreter to use.
112131
srcs: The src files that the script depends on. This is important to
113132
ensure that the Bazel repository cache or the bzlmod lock file gets
114133
invalidated when any one file changes. It is advisable to use
@@ -118,26 +137,34 @@ def _execute_checked(mrctx, *, srcs, **kwargs):
118137
the `environment` has a value `PYTHONPATH` and it is a list, then
119138
it will be passed to `construct_pythonpath` function.
120139
"""
140+
return repo_utils.execute_checked(
141+
mrctx,
142+
**_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
143+
)
121144

122-
for src in srcs:
123-
# This will ensure that we will re-evaluate the bzlmod extension or
124-
# refetch the repository_rule when the srcs change. This should work on
125-
# Bazel versions without `mrctx.watch` as well.
126-
repo_utils.watch(mrctx, mrctx.path(src))
127-
128-
env = kwargs.pop("environment", {})
129-
pythonpath = env.get("PYTHONPATH", "")
130-
if pythonpath and not types.is_string(pythonpath):
131-
env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
145+
def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs):
146+
"""Helper function to run a python script and modify the PYTHONPATH to include external deps.
132147
133-
return repo_utils.execute_checked(
148+
Args:
149+
mrctx: Handle to the module_ctx or repository_ctx.
150+
python: The python interpreter to use.
151+
srcs: The src files that the script depends on. This is important to
152+
ensure that the Bazel repository cache or the bzlmod lock file gets
153+
invalidated when any one file changes. It is advisable to use
154+
`RECORD` files for external deps and the list of srcs from the
155+
rules_python repo for any scripts.
156+
**kwargs: Arguments forwarded to `repo_utils.execute_checked`. If
157+
the `environment` has a value `PYTHONPATH` and it is a list, then
158+
it will be passed to `construct_pythonpath` function.
159+
"""
160+
return repo_utils.execute_checked_stdout(
134161
mrctx,
135-
environment = env,
136-
**kwargs
162+
**_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
137163
)
138164

139165
pypi_repo_utils = struct(
140166
construct_pythonpath = _construct_pypath,
141167
execute_checked = _execute_checked,
168+
execute_checked_stdout = _execute_checked_stdout,
142169
resolve_python_interpreter = _resolve_python_interpreter,
143170
)

python/private/pypi/whl_library.bzl

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,15 @@ def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None):
7575
if not is_standalone_interpreter(rctx, python_interpreter, logger = logger):
7676
return []
7777

78-
stdout = repo_utils.execute_checked_stdout(
78+
stdout = pypi_repo_utils.execute_checked_stdout(
7979
rctx,
8080
op = "GetPythonVersionForUnixCflags",
81+
python = python_interpreter,
8182
arguments = [
82-
python_interpreter,
8383
"-c",
8484
"import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
8585
],
86+
srcs = [],
8687
)
8788
_python_version = stdout
8889
include_path = "{}/include/python{}".format(
@@ -139,11 +140,28 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
139140
if rctx.attr.enable_implicit_namespace_pkgs:
140141
args.append("--enable_implicit_namespace_pkgs")
141142

143+
env = {}
142144
if rctx.attr.environment != None:
143-
args += [
144-
"--environment",
145-
json.encode(struct(arg = rctx.attr.environment)),
146-
]
145+
for key, value in rctx.attr.environment.items():
146+
env[key] = value
147+
148+
# This is super hacky, but working out something nice is tricky.
149+
# This is in particular needed for psycopg2 which attempts to link libpython.a,
150+
# in order to point the linker at the correct python intepreter.
151+
if rctx.attr.add_libdir_to_library_search_path:
152+
if "LDFLAGS" in env:
153+
fail("Can't set both environment LDFLAGS and add_libdir_to_library_search_path")
154+
command = [pypi_repo_utils.resolve_python_interpreter(rctx), "-c", "import sys ; sys.stdout.write('{}/lib'.format(sys.exec_prefix))"]
155+
result = rctx.execute(command)
156+
if result.return_code != 0:
157+
fail("Failed to get LDFLAGS path: command: {}, exit code: {}, stdout: {}, stderr: {}".format(command, result.return_code, result.stdout, result.stderr))
158+
libdir = result.stdout
159+
env["LDFLAGS"] = "-L{}".format(libdir)
160+
161+
args += [
162+
"--environment",
163+
json.encode(struct(arg = env)),
164+
]
147165

148166
return args
149167

@@ -181,7 +199,6 @@ def _whl_library_impl(rctx):
181199
python_interpreter_target = rctx.attr.python_interpreter_target,
182200
)
183201
args = [
184-
python_interpreter,
185202
"-m",
186203
"python.private.pypi.whl_installer.wheel_installer",
187204
"--requirement",
@@ -247,6 +264,7 @@ def _whl_library_impl(rctx):
247264
# truncate the requirement value when logging it / reporting
248265
# progress since it may contain several ' --hash=sha256:...
249266
# --hash=sha256:...' substrings that fill up the console
267+
python = python_interpreter,
250268
op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]),
251269
arguments = args,
252270
environment = environment,
@@ -295,6 +313,7 @@ def _whl_library_impl(rctx):
295313
pypi_repo_utils.execute_checked(
296314
rctx,
297315
op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
316+
python = python_interpreter,
298317
arguments = args + [
299318
"--whl-file",
300319
whl_path,

0 commit comments

Comments
 (0)