Skip to content

Commit 756264a

Browse files
rickeylevrickeylev
authored
feat: bzl file per rule/provider (#1122)
This is basically a performance optimization. Bazel can only see that a bzl file changed, not the particular contents. This means that any downstream bzl file loading it is invalidated, even if it doesn't load any of the affected code. As an example, if a package only loads `py_library.bzl`, then changing `py_test.bzl` doesn't need to invalidate all libraries. * Also removes some more extraneous license comments Work towards #1069
1 parent 31d0efd commit 756264a

14 files changed

+353
-96
lines changed

‎docs/BUILD.bazel‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,11 @@ bzl_library(
5656
"//python:defs.bzl",
5757
"//python/private:reexports.bzl",
5858
],
59-
deps = [":bazel_python_tools"],
59+
deps = [
60+
":bazel_python_tools",
61+
"//python:defs_bzl",
62+
"//python/private:reexports_bzl",
63+
],
6064
)
6165

6266
bzl_library(

‎python/BUILD.bazel‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@ that @rules_python//python is only concerned with the core rules.
2424
"""
2525

2626
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
27-
load(":defs.bzl", "current_py_toolchain")
27+
load(":current_py_toolchain.bzl", "current_py_toolchain")
2828

2929
package(default_visibility = ["//visibility:public"])
3030

31-
licenses(["notice"]) # Apache 2.0
31+
licenses(["notice"])
3232

3333
filegroup(
3434
name = "distribution",
@@ -41,13 +41,22 @@ filegroup(
4141
visibility = ["//:__pkg__"],
4242
)
4343

44+
# ========= bzl_library targets end =========
45+
46+
bzl_library(
47+
name = "current_py_toolchain_bzl",
48+
srcs = ["current_py_toolchain.bzl"],
49+
)
50+
4451
bzl_library(
4552
name = "defs_bzl",
4653
srcs = [
4754
"defs.bzl",
4855
],
4956
visibility = ["//visibility:public"],
5057
deps = [
58+
":current_py_toolchain_bzl",
59+
":py_import_bzl",
5160
"//python/private:bazel_tools_bzl",
5261
"//python/private:reexports_bzl",
5362
],
@@ -64,6 +73,57 @@ bzl_library(
6473
],
6574
)
6675

76+
bzl_library(
77+
name = "py_binary_bzl",
78+
srcs = ["py_binary.bzl"],
79+
deps = ["//python/private:reexports_bzl"],
80+
)
81+
82+
bzl_library(
83+
name = "py_import_bzl",
84+
srcs = ["py_import.bzl"],
85+
deps = [":py_info_bzl"],
86+
)
87+
88+
bzl_library(
89+
name = "py_info_bzl",
90+
srcs = ["py_info.bzl"],
91+
deps = ["//python/private:reexports_bzl"],
92+
)
93+
94+
bzl_library(
95+
name = "py_library_bzl",
96+
srcs = ["py_library.bzl"],
97+
deps = ["//python/private:reexports_bzl"],
98+
)
99+
100+
bzl_library(
101+
name = "py_runtime_bzl",
102+
srcs = ["py_runtime.bzl"],
103+
deps = ["//python/private:reexports_bzl"],
104+
)
105+
106+
bzl_library(
107+
name = "py_runtime_pair_bzl",
108+
srcs = ["py_runtime_pair.bzl"],
109+
deps = ["//python/private:reexports_bzl"],
110+
)
111+
112+
bzl_library(
113+
name = "py_runtime_info_bzl",
114+
srcs = ["py_runtime_info.bzl"],
115+
deps = ["//python/private:reexports_bzl"],
116+
)
117+
118+
bzl_library(
119+
name = "py_test_bzl",
120+
srcs = ["py_test.bzl"],
121+
deps = ["//python/private:reexports_bzl"],
122+
)
123+
124+
# NOTE: Remember to add bzl_library targets to //tests:bzl_libraries
125+
# ========= bzl_library targets end =========
126+
67127
# Filegroup of bzl files that can be used by downstream rules for documentation generation
68128
filegroup(
69129
name = "bzl",

‎python/current_py_toolchain.bzl‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Public entry point for current_py_toolchain rule."""
16+
17+
def _current_py_toolchain_impl(ctx):
18+
toolchain = ctx.toolchains[ctx.attr._toolchain]
19+
20+
direct = []
21+
transitive = []
22+
vars = {}
23+
24+
if toolchain.py3_runtime and toolchain.py3_runtime.interpreter:
25+
direct.append(toolchain.py3_runtime.interpreter)
26+
transitive.append(toolchain.py3_runtime.files)
27+
vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path
28+
29+
if toolchain.py2_runtime and toolchain.py2_runtime.interpreter:
30+
direct.append(toolchain.py2_runtime.interpreter)
31+
transitive.append(toolchain.py2_runtime.files)
32+
vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path
33+
34+
files = depset(direct, transitive = transitive)
35+
return [
36+
toolchain,
37+
platform_common.TemplateVariableInfo(vars),
38+
DefaultInfo(
39+
runfiles = ctx.runfiles(transitive_files = files),
40+
files = files,
41+
),
42+
]
43+
44+
current_py_toolchain = rule(
45+
doc = """
46+
This rule exists so that the current python toolchain can be used in the `toolchains` attribute of
47+
other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has
48+
happened, to a rule which expects a concrete implementation of a toolchain, rather than a
49+
toolchain_type which could be resolved to that toolchain.
50+
""",
51+
implementation = _current_py_toolchain_impl,
52+
attrs = {
53+
"_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))),
54+
},
55+
toolchains = [
56+
str(Label("@bazel_tools//tools/python:toolchain_type")),
57+
],
58+
)

‎python/defs.bzl‎

Lines changed: 4 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -27,105 +27,18 @@ load(
2727
_py_runtime_pair = "py_runtime_pair",
2828
_py_test = "py_test",
2929
)
30+
load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain")
31+
load(":py_import.bzl", _py_import = "py_import")
3032

3133
# Exports of native-defined providers.
3234

3335
PyInfo = internal_PyInfo
3436

3537
PyRuntimeInfo = internal_PyRuntimeInfo
3638

37-
def _current_py_toolchain_impl(ctx):
38-
toolchain = ctx.toolchains[ctx.attr._toolchain]
39+
current_py_toolchain = _current_py_toolchain
3940

40-
direct = []
41-
transitive = []
42-
vars = {}
43-
44-
if toolchain.py3_runtime and toolchain.py3_runtime.interpreter:
45-
direct.append(toolchain.py3_runtime.interpreter)
46-
transitive.append(toolchain.py3_runtime.files)
47-
vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path
48-
49-
if toolchain.py2_runtime and toolchain.py2_runtime.interpreter:
50-
direct.append(toolchain.py2_runtime.interpreter)
51-
transitive.append(toolchain.py2_runtime.files)
52-
vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path
53-
54-
files = depset(direct, transitive = transitive)
55-
return [
56-
toolchain,
57-
platform_common.TemplateVariableInfo(vars),
58-
DefaultInfo(
59-
runfiles = ctx.runfiles(transitive_files = files),
60-
files = files,
61-
),
62-
]
63-
64-
current_py_toolchain = rule(
65-
doc = """
66-
This rule exists so that the current python toolchain can be used in the `toolchains` attribute of
67-
other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has
68-
happened, to a rule which expects a concrete implementation of a toolchain, rather than a
69-
toolchain_type which could be resolved to that toolchain.
70-
""",
71-
implementation = _current_py_toolchain_impl,
72-
attrs = {
73-
"_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))),
74-
},
75-
toolchains = [
76-
str(Label("@bazel_tools//tools/python:toolchain_type")),
77-
],
78-
)
79-
80-
def _py_import_impl(ctx):
81-
# See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 .
82-
import_paths = [
83-
"/".join([ctx.workspace_name, x.short_path])
84-
for x in ctx.files.srcs
85-
]
86-
87-
return [
88-
DefaultInfo(
89-
default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True),
90-
),
91-
PyInfo(
92-
transitive_sources = depset(transitive = [
93-
dep[PyInfo].transitive_sources
94-
for dep in ctx.attr.deps
95-
]),
96-
imports = depset(direct = import_paths, transitive = [
97-
dep[PyInfo].imports
98-
for dep in ctx.attr.deps
99-
]),
100-
),
101-
]
102-
103-
py_import = rule(
104-
doc = """This rule allows the use of Python packages as dependencies.
105-
106-
It imports the given `.egg` file(s), which might be checked in source files,
107-
fetched externally as with `http_file`, or produced as outputs of other rules.
108-
109-
It may be used like a `py_library`, in the `deps` of other Python rules.
110-
111-
This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import).
112-
""",
113-
implementation = _py_import_impl,
114-
attrs = {
115-
"deps": attr.label_list(
116-
doc = "The list of other libraries to be linked in to the " +
117-
"binary target.",
118-
providers = [PyInfo],
119-
),
120-
"srcs": attr.label_list(
121-
doc = "The list of Python package files provided to Python targets " +
122-
"that depend on this target. Note that currently only the .egg " +
123-
"format is accepted. For .whl files, try the whl_library rule. " +
124-
"We accept contributions to extend py_import to handle .whl.",
125-
allow_files = [".egg"],
126-
),
127-
},
128-
)
41+
py_import = _py_import
12942

13043
# Re-exports of Starlark-defined symbols in @bazel_tools//tools/python.
13144

‎python/private/BUILD.bazel‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ filegroup(
3434
bzl_library(
3535
name = "reexports_bzl",
3636
srcs = ["reexports.bzl"],
37-
visibility = ["//python:__pkg__"],
37+
visibility = [
38+
"//docs:__pkg__",
39+
"//python:__pkg__",
40+
],
3841
deps = [":bazel_tools_bzl"],
3942
)
4043

‎python/py_binary.bzl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Public entry point for py_binary."""
16+
17+
load("//python/private:reexports.bzl", _py_binary = "py_binary")
18+
19+
py_binary = _py_binary

‎python/py_import.bzl‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Public entry point for py_import rule."""
16+
17+
load(":py_info.bzl", "PyInfo")
18+
19+
def _py_import_impl(ctx):
20+
# See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 .
21+
import_paths = [
22+
"/".join([ctx.workspace_name, x.short_path])
23+
for x in ctx.files.srcs
24+
]
25+
26+
return [
27+
DefaultInfo(
28+
default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True),
29+
),
30+
PyInfo(
31+
transitive_sources = depset(transitive = [
32+
dep[PyInfo].transitive_sources
33+
for dep in ctx.attr.deps
34+
]),
35+
imports = depset(direct = import_paths, transitive = [
36+
dep[PyInfo].imports
37+
for dep in ctx.attr.deps
38+
]),
39+
),
40+
]
41+
42+
py_import = rule(
43+
doc = """This rule allows the use of Python packages as dependencies.
44+
45+
It imports the given `.egg` file(s), which might be checked in source files,
46+
fetched externally as with `http_file`, or produced as outputs of other rules.
47+
48+
It may be used like a `py_library`, in the `deps` of other Python rules.
49+
50+
This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import).
51+
""",
52+
implementation = _py_import_impl,
53+
attrs = {
54+
"deps": attr.label_list(
55+
doc = "The list of other libraries to be linked in to the " +
56+
"binary target.",
57+
providers = [PyInfo],
58+
),
59+
"srcs": attr.label_list(
60+
doc = "The list of Python package files provided to Python targets " +
61+
"that depend on this target. Note that currently only the .egg " +
62+
"format is accepted. For .whl files, try the whl_library rule. " +
63+
"We accept contributions to extend py_import to handle .whl.",
64+
allow_files = [".egg"],
65+
),
66+
},
67+
)

‎python/py_info.bzl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Public entry point for PyInfo."""
16+
17+
load("//python/private:reexports.bzl", "internal_PyInfo")
18+
19+
PyInfo = internal_PyInfo

0 commit comments

Comments
 (0)