From 34cd8383ca07a596ab80d37fa6842ebb630f9a5e Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Mon, 31 Mar 2025 21:18:25 -0700 Subject: [PATCH 01/27] Add support for REPLs This patch adds three different ways to invoke a REPL for a given target, each with slightly unique use cases. Deployed binaries: Sometimes it's really useful to start the REPL for a binary that's already deployed in a docker container. You can do this with the `RULES_PYTHON_BOOTSTRAP_REPL` environment variable. For example: $ RULES_PYTHON_BOOTSTRAP_REPL=1 bazel run --//python/config_settings:bootstrap_impl=script //tools:wheelmaker Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. (InteractiveConsole) >>> import tools.wheelmaker >>> `py_{library,binary,test}` targets: These targets will now auto-generate additional `.repl` targets. $ bazel run --//python/config_settings:bootstrap_impl=script //python/runfiles:runfiles.repl Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. (InteractiveConsole) >>> import python.runfiles >>> Arbitrary `PyInfo` providers: Spawn a REPL for any target that provides `PyInfo` like this: $ bazel run --//python/config_settings:bootstrap_impl=script //python/bin:repl --//python/bin:repl_dep=//tools:wheelmaker Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux Type "help", "copyright", "credits" or "license" for more information. (InteractiveConsole) >>> import tools.wheelmaker >>> --- .bazelrc | 2 ++ python/bin/BUILD.bazel | 26 +++++++++++++++++ python/bin/repl.py | 28 +++++++++++++++++++ python/private/py_library_macro.bzl | 31 +++++++++++++++++++-- python/private/sentinel.bzl | 3 ++ python/private/stage1_bootstrap_template.sh | 4 +++ python/private/stage2_bootstrap_template.py | 11 +++++++- 7 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 python/bin/repl.py diff --git a/.bazelrc b/.bazelrc index ada5c5a0a7..748309f086 100644 --- a/.bazelrc +++ b/.bazelrc @@ -36,3 +36,5 @@ build:rtd --enable_bzlmod common --incompatible_python_disallow_native_rules build --lockfile_mode=update + +run:repl --//python/config_settings:bootstrap_impl=script //python/bin:repl diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 57bee34378..ec619c459c 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,5 @@ load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") +load("//python:py_binary.bzl", "py_binary") filegroup( name = "distribution", @@ -22,3 +23,28 @@ label_flag( name = "python_src", build_setting_default = "//python:none", ) + +py_binary( + name = "repl", + srcs = ["repl.py"], + deps = [ + ":repl_dep", + ":repl_lib_dep", + ], + visibility = ["//visibility:public"], +) + +# The user can modify this flag to make arbitrary libraries available for import +# on the REPL. Anything that exposes PyInfo can be used here. +label_flag( + name = "repl_dep", + build_setting_default = "//python:none", +) + +# The user can modify this flag to make additional libraries available +# specifically for the purpose of interacting with the REPL. For example, point +# this at ipython in your .bazelrc file. +label_flag( + name = "repl_lib_dep", + build_setting_default = "//python:none", +) diff --git a/python/bin/repl.py b/python/bin/repl.py new file mode 100644 index 0000000000..8a0e06e50e --- /dev/null +++ b/python/bin/repl.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path + +def start_repl(): + # Simulate Python's behavior when a valid startup script is defined by the + # PYTHONSTARTUP variable. If this file path fails to load, print the error + # and revert to the default behavior. + if (startup_file := os.getenv("PYTHONSTARTUP")): + try: + source_code = Path(startup_file).read_text() + except Exception as error: + print(f"{type(error).__name__}: {error}") + else: + compiled_code = compile(source_code, filename=startup_file, mode="exec") + eval(compiled_code, {}) + + try: + # If the user has made ipython available somehow (e.g. via + # `repl_lib_dep`), then use it. + import IPython + IPython.start_ipython() + except ModuleNotFoundError: + # Fall back to the default shell. + import code + code.interact(local=dict(globals(), **locals())) + +if __name__ == "__main__": + start_repl() diff --git a/python/private/py_library_macro.bzl b/python/private/py_library_macro.bzl index 981253d63a..b1648499ac 100644 --- a/python/private/py_library_macro.bzl +++ b/python/private/py_library_macro.bzl @@ -14,8 +14,35 @@ """Implementation of macro-half of py_library rule.""" load(":py_library_rule.bzl", py_library_rule = "py_library") +load(":py_binary_rule.bzl", py_binary_rule = "py_binary") + +# The py_library's attributes we don't want to forward to auto-generated +# targets. +_LIBRARY_ONLY_ATTRS = [ + "srcs", + "deps", + "data", + "imports", +] # A wrapper macro is used to avoid any user-observable changes between a # rule and macro. It also makes generator_function look as expected. -def py_library(**kwargs): - py_library_rule(**kwargs) +def py_library(name, **kwargs): + library_only_attrs = { + attr: kwargs.pop(attr, None) + for attr in _LIBRARY_ONLY_ATTRS + } + py_library_rule( + name = name, + **(library_only_attrs | kwargs) + ) + py_binary_rule( + name = "%s.repl" % name, + srcs = [], + main_module = "python.bin.repl", + deps = [ + ":%s" % name, + "@rules_python//python/bin:repl", + ], + **kwargs + ) diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 8b69682b49..047f09727e 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,6 +18,8 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ +load("//python:py_info.bzl", "PyInfo") + SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -29,6 +31,7 @@ def _sentinel_impl(ctx): SentinelInfo(), # Also output ToolchainInfo to allow it to be used for noop toolchains platform_common.ToolchainInfo(), + PyInfo(transitive_sources=depset()), ] sentinel = rule(implementation = _sentinel_impl) diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index c487624934..f5213ea8ec 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -238,6 +238,10 @@ command=( "$@" ) +# Point libedit/readline at the correct terminfo databases. +# https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl +export TERMINFO_DIRS=/etc/terminfo:/lib/terminfo:/usr/share/terminfo + # We use `exec` instead of a child process so that signals sent directly (e.g. # using `kill`) to this process (the PID seen by the calling process) are # received by the Python process. Otherwise, this process receives the signal diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index e8228edf3b..93a67be6d7 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -365,6 +365,16 @@ def main(): print_verbose("initial environ:", mapping=os.environ) print_verbose("initial sys.path:", values=sys.path) + if bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_REPL")): + global MAIN_PATH + global MAIN_MODULE + MAIN_PATH = "" + # TODO(philsc): Can we point at python.bin.repl instead? That would mean + # adding it as a dependency to all binaries. + MAIN_MODULE = "code" + # Prevent subprocesses from also entering the REPL. + del os.environ["RULES_PYTHON_BOOTSTRAP_REPL"] + main_rel_path = None # todo: things happen to work because find_runfiles_root # ends up using stage2_bootstrap, and ends up computing the proper @@ -438,7 +448,6 @@ def main(): _run_py_path(main_filename, args=sys.argv[1:]) else: _run_py_module(MAIN_MODULE) - sys.exit(0) main() From 06cef6723a5810584e1011ed545c47ded3b6381e Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 03:45:17 -0700 Subject: [PATCH 02/27] Remove the macro-generator --- .bazelrc | 2 -- python/bin/repl.py | 4 +++ python/private/py_library_macro.bzl | 31 ++------------------- python/private/sentinel.bzl | 3 -- python/private/stage1_bootstrap_template.sh | 4 --- python/private/stage2_bootstrap_template.py | 5 ++++ 6 files changed, 11 insertions(+), 38 deletions(-) diff --git a/.bazelrc b/.bazelrc index cd982a84ac..4e6f2fa187 100644 --- a/.bazelrc +++ b/.bazelrc @@ -36,5 +36,3 @@ build:rtd --enable_bzlmod common --incompatible_python_disallow_native_rules build --lockfile_mode=update - -run:repl --//python/config_settings:bootstrap_impl=script //python/bin:repl diff --git a/python/bin/repl.py b/python/bin/repl.py index 8a0e06e50e..57289c1e3e 100644 --- a/python/bin/repl.py +++ b/python/bin/repl.py @@ -2,6 +2,10 @@ from pathlib import Path def start_repl(): + # Point libedit/readline at the correct terminfo databases. + # https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl + os.environ["TERMINFO_DIRS"] = "/etc/terminfo:/lib/terminfo:/usr/share/terminfo" + # Simulate Python's behavior when a valid startup script is defined by the # PYTHONSTARTUP variable. If this file path fails to load, print the error # and revert to the default behavior. diff --git a/python/private/py_library_macro.bzl b/python/private/py_library_macro.bzl index b1648499ac..981253d63a 100644 --- a/python/private/py_library_macro.bzl +++ b/python/private/py_library_macro.bzl @@ -14,35 +14,8 @@ """Implementation of macro-half of py_library rule.""" load(":py_library_rule.bzl", py_library_rule = "py_library") -load(":py_binary_rule.bzl", py_binary_rule = "py_binary") - -# The py_library's attributes we don't want to forward to auto-generated -# targets. -_LIBRARY_ONLY_ATTRS = [ - "srcs", - "deps", - "data", - "imports", -] # A wrapper macro is used to avoid any user-observable changes between a # rule and macro. It also makes generator_function look as expected. -def py_library(name, **kwargs): - library_only_attrs = { - attr: kwargs.pop(attr, None) - for attr in _LIBRARY_ONLY_ATTRS - } - py_library_rule( - name = name, - **(library_only_attrs | kwargs) - ) - py_binary_rule( - name = "%s.repl" % name, - srcs = [], - main_module = "python.bin.repl", - deps = [ - ":%s" % name, - "@rules_python//python/bin:repl", - ], - **kwargs - ) +def py_library(**kwargs): + py_library_rule(**kwargs) diff --git a/python/private/sentinel.bzl b/python/private/sentinel.bzl index 047f09727e..8b69682b49 100644 --- a/python/private/sentinel.bzl +++ b/python/private/sentinel.bzl @@ -18,8 +18,6 @@ Label attributes with defaults cannot accept None, otherwise they fall back to using the default. A sentinel allows detecting an intended None value. """ -load("//python:py_info.bzl", "PyInfo") - SentinelInfo = provider( doc = "Indicates this was the sentinel target.", fields = [], @@ -31,7 +29,6 @@ def _sentinel_impl(ctx): SentinelInfo(), # Also output ToolchainInfo to allow it to be used for noop toolchains platform_common.ToolchainInfo(), - PyInfo(transitive_sources=depset()), ] sentinel = rule(implementation = _sentinel_impl) diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index f5213ea8ec..c487624934 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -238,10 +238,6 @@ command=( "$@" ) -# Point libedit/readline at the correct terminfo databases. -# https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl -export TERMINFO_DIRS=/etc/terminfo:/lib/terminfo:/usr/share/terminfo - # We use `exec` instead of a child process so that signals sent directly (e.g. # using `kill`) to this process (the PID seen by the calling process) are # received by the Python process. Otherwise, this process receives the signal diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 93a67be6d7..45d7d98512 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -372,9 +372,14 @@ def main(): # TODO(philsc): Can we point at python.bin.repl instead? That would mean # adding it as a dependency to all binaries. MAIN_MODULE = "code" + # Prevent subprocesses from also entering the REPL. del os.environ["RULES_PYTHON_BOOTSTRAP_REPL"] + # Point libedit/readline at the correct terminfo databases. + # https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl + os.environ["TERMINFO_DIRS"] = "/etc/terminfo:/lib/terminfo:/usr/share/terminfo" + main_rel_path = None # todo: things happen to work because find_runfiles_root # ends up using stage2_bootstrap, and ends up computing the proper From 6c2dd354836d7bdfff8ead6dccc95307751854eb Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 03:47:10 -0700 Subject: [PATCH 03/27] revert some unnecessary changes --- python/private/stage2_bootstrap_template.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 45d7d98512..9545cb9dc4 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -453,6 +453,7 @@ def main(): _run_py_path(main_filename, args=sys.argv[1:]) else: _run_py_module(MAIN_MODULE) + sys.exit(0) main() From fc9cdeb60e8bc02ac44fc5eb2c69eccef7e40899 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:28:13 -0700 Subject: [PATCH 04/27] Generate main.py so we can get independent of ipython --- python/bin/BUILD.bazel | 38 +++++++++++------- python/bin/repl_stub.py | 2 + python/private/BUILD.bazel | 11 ++++++ python/private/repl.bzl | 39 +++++++++++++++++++ python/private/repl_main_generator.py | 27 +++++++++++++ .../{bin/repl.py => private/repl_template.py} | 17 ++------ 6 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 python/bin/repl_stub.py create mode 100644 python/private/repl.bzl create mode 100644 python/private/repl_main_generator.py rename python/{bin/repl.py => private/repl_template.py} (50%) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index ec619c459c..16ebd6b9ba 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,4 +1,5 @@ load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") +load("//python/private:repl.bzl", "generate_repl_main") load("//python:py_binary.bzl", "py_binary") filegroup( @@ -24,27 +25,38 @@ label_flag( build_setting_default = "//python:none", ) +generate_repl_main( + name = "repl_py", + out = "repl.py", + src = "repl_stub.py", +) + +label_flag( + name = "repl_stub", + build_setting_default = "repl_stub.py", +) + +# The user can modify this flag to make an interpreter shell library available +# for the stub. E.g. if they switch the stub for an ipython-based on, then they +# can point this at their version of ipython. +label_flag( + name = "repl_stub_dep", + build_setting_default = "//python/private:empty", +) + py_binary( name = "repl", - srcs = ["repl.py"], + srcs = [":repl.py"], deps = [ + ":repl_stub_dep", ":repl_dep", - ":repl_lib_dep", ], visibility = ["//visibility:public"], ) -# The user can modify this flag to make arbitrary libraries available for import -# on the REPL. Anything that exposes PyInfo can be used here. +# The user can modify this flag to make arbitrary PyInfo targets available for +# import on the REPL. label_flag( name = "repl_dep", - build_setting_default = "//python:none", -) - -# The user can modify this flag to make additional libraries available -# specifically for the purpose of interacting with the REPL. For example, point -# this at ipython in your .bazelrc file. -label_flag( - name = "repl_lib_dep", - build_setting_default = "//python:none", + build_setting_default = "//python/private:empty", ) diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py new file mode 100644 index 0000000000..1f8d422c97 --- /dev/null +++ b/python/bin/repl_stub.py @@ -0,0 +1,2 @@ +import code +code.interact(local=dict(globals(), **locals())) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index ef4580e1ce..7648e154bf 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -789,6 +789,13 @@ py_library( ], ) +py_binary( + name = "repl_main_generator", + srcs = [ + "repl_main_generator.py", + ], +) + # The current toolchain's interpreter as an excutable, usable with # executable=True attributes. current_interpreter_executable( @@ -798,6 +805,10 @@ current_interpreter_executable( visibility = ["//visibility:public"], ) +py_library( + name = "empty", +) + sentinel( name = "sentinel", ) diff --git a/python/private/repl.bzl b/python/private/repl.bzl new file mode 100644 index 0000000000..6d4a30da42 --- /dev/null +++ b/python/private/repl.bzl @@ -0,0 +1,39 @@ +def _generate_repl_main_impl(ctx): + args = ctx.actions.args() + args.add_all([ + ctx.file._template, + ctx.file.src, + ctx.outputs.out, + ]) + + ctx.actions.run( + executable = ctx.executable._generator, + inputs = [ + ctx.file._template, + ctx.file.src, + ], + outputs = [ctx.outputs.out], + arguments = [args], + ) + +generate_repl_main = rule( + implementation = _generate_repl_main_impl, + attrs = { + "out": attr.output( + mandatory = True, + ), + "src": attr.label( + mandatory = True, + allow_single_file = True, + ), + "_template": attr.label( + default = "//python/private:repl_template.py", + allow_single_file = True, + ), + "_generator": attr.label( + default = "//python/private:repl_main_generator", + executable = True, + cfg = "exec", + ), + }, +) diff --git a/python/private/repl_main_generator.py b/python/private/repl_main_generator.py new file mode 100644 index 0000000000..8aae66a201 --- /dev/null +++ b/python/private/repl_main_generator.py @@ -0,0 +1,27 @@ +import sys +import textwrap +from pathlib import Path + +LINE_TO_REPLACE = """\ + pass # %REPLACE_WHOLE_LINE_WITH_STUB% +""" + +def main(argv): + template = Path(sys.argv[1]) + stub = Path(sys.argv[2]) + output = Path(sys.argv[3]) + + template_text = template.read_text() + stub_text = stub.read_text() + + indented_stub_text = textwrap.indent(stub_text, " " * 4) + + output_text = template_text.replace(LINE_TO_REPLACE, indented_stub_text) + if template_text == output_text: + raise ValueError("Failed to find the following in the template: {LINE_TO_REPLACE}") + + output.write_text(output_text) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/bin/repl.py b/python/private/repl_template.py similarity index 50% rename from python/bin/repl.py rename to python/private/repl_template.py index 57289c1e3e..2b0a332ee6 100644 --- a/python/bin/repl.py +++ b/python/private/repl_template.py @@ -1,11 +1,10 @@ import os from pathlib import Path -def start_repl(): - # Point libedit/readline at the correct terminfo databases. - # https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl - os.environ["TERMINFO_DIRS"] = "/etc/terminfo:/lib/terminfo:/usr/share/terminfo" +def repl_stub(): + pass # %REPLACE_WHOLE_LINE_WITH_STUB% +def start_repl(): # Simulate Python's behavior when a valid startup script is defined by the # PYTHONSTARTUP variable. If this file path fails to load, print the error # and revert to the default behavior. @@ -18,15 +17,7 @@ def start_repl(): compiled_code = compile(source_code, filename=startup_file, mode="exec") eval(compiled_code, {}) - try: - # If the user has made ipython available somehow (e.g. via - # `repl_lib_dep`), then use it. - import IPython - IPython.start_ipython() - except ModuleNotFoundError: - # Fall back to the default shell. - import code - code.interact(local=dict(globals(), **locals())) + repl_stub() if __name__ == "__main__": start_repl() From a2be313dfa3e199ebb84bf98ef7a573443e96efb Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:38:15 -0700 Subject: [PATCH 05/27] precommit --- python/bin/BUILD.bazel | 8 ++++---- python/bin/repl_stub.py | 1 + python/private/repl.bzl | 8 ++++---- python/private/repl_main_generator.py | 7 +++++-- python/private/repl_template.py | 7 +++++-- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 16ebd6b9ba..43d0a8684c 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,6 +1,6 @@ +load("//python:py_binary.bzl", "py_binary") load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") load("//python/private:repl.bzl", "generate_repl_main") -load("//python:py_binary.bzl", "py_binary") filegroup( name = "distribution", @@ -27,8 +27,8 @@ label_flag( generate_repl_main( name = "repl_py", - out = "repl.py", src = "repl_stub.py", + out = "repl.py", ) label_flag( @@ -47,11 +47,11 @@ label_flag( py_binary( name = "repl", srcs = [":repl.py"], + visibility = ["//visibility:public"], deps = [ - ":repl_stub_dep", ":repl_dep", + ":repl_stub_dep", ], - visibility = ["//visibility:public"], ) # The user can modify this flag to make arbitrary PyInfo targets available for diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py index 1f8d422c97..98a2a09f95 100644 --- a/python/bin/repl_stub.py +++ b/python/bin/repl_stub.py @@ -1,2 +1,3 @@ import code + code.interact(local=dict(globals(), **locals())) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 6d4a30da42..ed00b6adfc 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -26,14 +26,14 @@ generate_repl_main = rule( mandatory = True, allow_single_file = True, ), - "_template": attr.label( - default = "//python/private:repl_template.py", - allow_single_file = True, - ), "_generator": attr.label( default = "//python/private:repl_main_generator", executable = True, cfg = "exec", ), + "_template": attr.label( + default = "//python/private:repl_template.py", + allow_single_file = True, + ), }, ) diff --git a/python/private/repl_main_generator.py b/python/private/repl_main_generator.py index 8aae66a201..bfc0877c04 100644 --- a/python/private/repl_main_generator.py +++ b/python/private/repl_main_generator.py @@ -3,9 +3,10 @@ from pathlib import Path LINE_TO_REPLACE = """\ - pass # %REPLACE_WHOLE_LINE_WITH_STUB% + pass # %REPLACE_WHOLE_LINE_WITH_STUB% """ + def main(argv): template = Path(sys.argv[1]) stub = Path(sys.argv[2]) @@ -18,7 +19,9 @@ def main(argv): output_text = template_text.replace(LINE_TO_REPLACE, indented_stub_text) if template_text == output_text: - raise ValueError("Failed to find the following in the template: {LINE_TO_REPLACE}") + raise ValueError( + f"Failed to find the following in the template: {LINE_TO_REPLACE}" + ) output.write_text(output_text) diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 2b0a332ee6..509ea50643 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -1,14 +1,16 @@ import os from pathlib import Path + def repl_stub(): - pass # %REPLACE_WHOLE_LINE_WITH_STUB% + pass # %REPLACE_WHOLE_LINE_WITH_STUB% + def start_repl(): # Simulate Python's behavior when a valid startup script is defined by the # PYTHONSTARTUP variable. If this file path fails to load, print the error # and revert to the default behavior. - if (startup_file := os.getenv("PYTHONSTARTUP")): + if startup_file := os.getenv("PYTHONSTARTUP"): try: source_code = Path(startup_file).read_text() except Exception as error: @@ -19,5 +21,6 @@ def start_repl(): repl_stub() + if __name__ == "__main__": start_repl() From f6dc039902f7ef3bfe600266bc7b0df0940daacd Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:47:07 -0700 Subject: [PATCH 06/27] fix a couple of comments --- python/bin/BUILD.bazel | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 43d0a8684c..8cdec24821 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -31,13 +31,15 @@ generate_repl_main( out = "repl.py", ) +# The user can replace this with their own stub. E.g. they can use this to +# import ipython instead of the default shell. label_flag( name = "repl_stub", build_setting_default = "repl_stub.py", ) # The user can modify this flag to make an interpreter shell library available -# for the stub. E.g. if they switch the stub for an ipython-based on, then they +# for the stub. E.g. if they switch the stub for an ipython-based one, then they # can point this at their version of ipython. label_flag( name = "repl_stub_dep", From b0b83ab1c13e8533ecd3a0dca63cc8b166e7d219 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:50:06 -0700 Subject: [PATCH 07/27] Delete unnecessary code --- python/private/stage2_bootstrap_template.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 9545cb9dc4..6e4beaacc8 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -369,17 +369,11 @@ def main(): global MAIN_PATH global MAIN_MODULE MAIN_PATH = "" - # TODO(philsc): Can we point at python.bin.repl instead? That would mean - # adding it as a dependency to all binaries. MAIN_MODULE = "code" # Prevent subprocesses from also entering the REPL. del os.environ["RULES_PYTHON_BOOTSTRAP_REPL"] - # Point libedit/readline at the correct terminfo databases. - # https://github.com/astral-sh/python-build-standalone/blob/f0abfc9cb1f6a985fc5561cf5435f7f6e8a64e5b/docs/quirks.rst#backspace-key-doesnt-work-in-python-repl - os.environ["TERMINFO_DIRS"] = "/etc/terminfo:/lib/terminfo:/usr/share/terminfo" - main_rel_path = None # todo: things happen to work because find_runfiles_root # ends up using stage2_bootstrap, and ends up computing the proper From 402faf5ed40279d7d041220278843924b1cd33ee Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:50:25 -0700 Subject: [PATCH 08/27] Undo bootstrap hook for now --- python/private/stage2_bootstrap_template.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 6e4beaacc8..e8228edf3b 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -365,15 +365,6 @@ def main(): print_verbose("initial environ:", mapping=os.environ) print_verbose("initial sys.path:", values=sys.path) - if bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_REPL")): - global MAIN_PATH - global MAIN_MODULE - MAIN_PATH = "" - MAIN_MODULE = "code" - - # Prevent subprocesses from also entering the REPL. - del os.environ["RULES_PYTHON_BOOTSTRAP_REPL"] - main_rel_path = None # todo: things happen to work because find_runfiles_root # ends up using stage2_bootstrap, and ends up computing the proper From 66d216db9bb2304eeb23d182817a743c921d34e4 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 04:53:16 -0700 Subject: [PATCH 09/27] prevent import leaks --- python/bin/repl_stub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py index 98a2a09f95..3a4e702d88 100644 --- a/python/bin/repl_stub.py +++ b/python/bin/repl_stub.py @@ -1,3 +1,3 @@ import code -code.interact(local=dict(globals(), **locals())) +code.interact() From ac5d58ce0ba203106194f139a3c078145d7de13c Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:12:10 -0700 Subject: [PATCH 10/27] simplify design a bit --- python/bin/BUILD.bazel | 6 +++++- python/private/repl.bzl | 22 +++++++------------- python/private/repl_main_generator.py | 30 --------------------------- python/private/repl_template.py | 9 +++++--- 4 files changed, 19 insertions(+), 48 deletions(-) delete mode 100644 python/private/repl_main_generator.py diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 8cdec24821..229b75a6ac 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -27,7 +27,7 @@ label_flag( generate_repl_main( name = "repl_py", - src = "repl_stub.py", + src = "repl_stub", out = "repl.py", ) @@ -50,9 +50,13 @@ py_binary( name = "repl", srcs = [":repl.py"], visibility = ["//visibility:public"], + data = [ + ":repl_stub", + ], deps = [ ":repl_dep", ":repl_stub_dep", + "//python/runfiles", ], ) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index ed00b6adfc..ec8cfda76c 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,19 +1,13 @@ def _generate_repl_main_impl(ctx): - args = ctx.actions.args() - args.add_all([ - ctx.file._template, - ctx.file.src, - ctx.outputs.out, - ]) + stub_repo = ctx.attr.src.label.repo_name or ctx.workspace_name + stub_path = "/".join([stub_repo, ctx.file.src.short_path]) - ctx.actions.run( - executable = ctx.executable._generator, - inputs = [ - ctx.file._template, - ctx.file.src, - ], - outputs = [ctx.outputs.out], - arguments = [args], + ctx.actions.expand_template( + template = ctx.file._template, + output = ctx.outputs.out, + substitutions = { + "%stub_path%": stub_path, + }, ) generate_repl_main = rule( diff --git a/python/private/repl_main_generator.py b/python/private/repl_main_generator.py deleted file mode 100644 index bfc0877c04..0000000000 --- a/python/private/repl_main_generator.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys -import textwrap -from pathlib import Path - -LINE_TO_REPLACE = """\ - pass # %REPLACE_WHOLE_LINE_WITH_STUB% -""" - - -def main(argv): - template = Path(sys.argv[1]) - stub = Path(sys.argv[2]) - output = Path(sys.argv[3]) - - template_text = template.read_text() - stub_text = stub.read_text() - - indented_stub_text = textwrap.indent(stub_text, " " * 4) - - output_text = template_text.replace(LINE_TO_REPLACE, indented_stub_text) - if template_text == output_text: - raise ValueError( - f"Failed to find the following in the template: {LINE_TO_REPLACE}" - ) - - output.write_text(output_text) - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 509ea50643..43e4898870 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -1,9 +1,11 @@ import os +import runpy from pathlib import Path +from python.runfiles import runfiles -def repl_stub(): - pass # %REPLACE_WHOLE_LINE_WITH_STUB% + +STUB_PATH = "%stub_path%" def start_repl(): @@ -19,7 +21,8 @@ def start_repl(): compiled_code = compile(source_code, filename=startup_file, mode="exec") eval(compiled_code, {}) - repl_stub() + bazel_runfiles = runfiles.Create() + runpy.run_path(bazel_runfiles.Rlocation(STUB_PATH), run_name="__main__") if __name__ == "__main__": From 9c9c8b00a474a2cc3101d1785bc1a675a115d6c9 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:18:39 -0700 Subject: [PATCH 11/27] clean up some more --- python/bin/BUILD.bazel | 29 +++++++++-------------------- python/private/repl.bzl | 25 ++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 229b75a6ac..90bcc8de58 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -1,6 +1,5 @@ -load("//python:py_binary.bzl", "py_binary") load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") -load("//python/private:repl.bzl", "generate_repl_main") +load("//python/private:repl.bzl", "py_repl_binary") filegroup( name = "distribution", @@ -25,10 +24,14 @@ label_flag( build_setting_default = "//python:none", ) -generate_repl_main( - name = "repl_py", - src = "repl_stub", - out = "repl.py", +py_repl_binary( + name = "repl", + visibility = ["//visibility:public"], + stub = ":repl_stub", + deps = [ + ":repl_dep", + ":repl_stub_dep", + ], ) # The user can replace this with their own stub. E.g. they can use this to @@ -46,20 +49,6 @@ label_flag( build_setting_default = "//python/private:empty", ) -py_binary( - name = "repl", - srcs = [":repl.py"], - visibility = ["//visibility:public"], - data = [ - ":repl_stub", - ], - deps = [ - ":repl_dep", - ":repl_stub_dep", - "//python/runfiles", - ], -) - # The user can modify this flag to make arbitrary PyInfo targets available for # import on the REPL. label_flag( diff --git a/python/private/repl.bzl b/python/private/repl.bzl index ec8cfda76c..8ce45e7d53 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,3 +1,5 @@ +load("//python:py_binary.bzl", "py_binary") + def _generate_repl_main_impl(ctx): stub_repo = ctx.attr.src.label.repo_name or ctx.workspace_name stub_path = "/".join([stub_repo, ctx.file.src.short_path]) @@ -10,7 +12,7 @@ def _generate_repl_main_impl(ctx): }, ) -generate_repl_main = rule( +_generate_repl_main = rule( implementation = _generate_repl_main_impl, attrs = { "out": attr.output( @@ -31,3 +33,24 @@ generate_repl_main = rule( ), }, ) + +def py_repl_binary(name, stub, deps=[], data=[], **kwargs): + _generate_repl_main( + name = "%s_py" % name, + src = stub, + out = "%s.py" % name, + ) + + py_binary( + name = name, + srcs = [ + ":%s.py" % name, + ], + data = data + [ + stub, + ], + deps = deps + [ + "//python/runfiles", + ], + **kwargs, + ) From 1d3de79936e0541b507febcab723a0339691c348 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:19:51 -0700 Subject: [PATCH 12/27] run pre-commit --- python/bin/BUILD.bazel | 2 +- python/private/repl.bzl | 4 ++-- python/private/repl_template.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel index 90bcc8de58..30af7d1b9f 100644 --- a/python/bin/BUILD.bazel +++ b/python/bin/BUILD.bazel @@ -26,8 +26,8 @@ label_flag( py_repl_binary( name = "repl", - visibility = ["//visibility:public"], stub = ":repl_stub", + visibility = ["//visibility:public"], deps = [ ":repl_dep", ":repl_stub_dep", diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 8ce45e7d53..3699f43a2e 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -34,7 +34,7 @@ _generate_repl_main = rule( }, ) -def py_repl_binary(name, stub, deps=[], data=[], **kwargs): +def py_repl_binary(name, stub, deps = [], data = [], **kwargs): _generate_repl_main( name = "%s_py" % name, src = stub, @@ -52,5 +52,5 @@ def py_repl_binary(name, stub, deps=[], data=[], **kwargs): deps = deps + [ "//python/runfiles", ], - **kwargs, + **kwargs ) diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 43e4898870..32fd12ac9f 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -4,7 +4,6 @@ from python.runfiles import runfiles - STUB_PATH = "%stub_path%" From d8b11253223df2c0a67d7fdbf7c96ac8baa33f75 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:21:11 -0700 Subject: [PATCH 13/27] add docstring --- python/private/repl.bzl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 3699f43a2e..b1f1e51365 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,3 +1,19 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the rules to expose a REPL.""" + load("//python:py_binary.bzl", "py_binary") def _generate_repl_main_impl(ctx): From e05f9af473cc8afb8a97ab7b9f15ba1d8a6f9442 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:22:28 -0700 Subject: [PATCH 14/27] clean up naming a tad --- python/private/repl.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index b1f1e51365..2a4139efe1 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -17,8 +17,8 @@ load("//python:py_binary.bzl", "py_binary") def _generate_repl_main_impl(ctx): - stub_repo = ctx.attr.src.label.repo_name or ctx.workspace_name - stub_path = "/".join([stub_repo, ctx.file.src.short_path]) + stub_repo = ctx.attr.stub.label.repo_name or ctx.workspace_name + stub_path = "/".join([stub_repo, ctx.file.stub.short_path]) ctx.actions.expand_template( template = ctx.file._template, @@ -34,7 +34,7 @@ _generate_repl_main = rule( "out": attr.output( mandatory = True, ), - "src": attr.label( + "stub": attr.label( mandatory = True, allow_single_file = True, ), @@ -53,7 +53,7 @@ _generate_repl_main = rule( def py_repl_binary(name, stub, deps = [], data = [], **kwargs): _generate_repl_main( name = "%s_py" % name, - src = stub, + stub = stub, out = "%s.py" % name, ) From 9121abdc3917602712e4ae792fcc8df012b64608 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 8 Apr 2025 05:52:35 -0700 Subject: [PATCH 15/27] removed dead code --- python/private/BUILD.bazel | 7 ------- python/private/repl.bzl | 5 ----- 2 files changed, 12 deletions(-) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 7648e154bf..ac92acdeec 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -789,13 +789,6 @@ py_library( ], ) -py_binary( - name = "repl_main_generator", - srcs = [ - "repl_main_generator.py", - ], -) - # The current toolchain's interpreter as an excutable, usable with # executable=True attributes. current_interpreter_executable( diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 2a4139efe1..76a7004ff2 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -38,11 +38,6 @@ _generate_repl_main = rule( mandatory = True, allow_single_file = True, ), - "_generator": attr.label( - default = "//python/private:repl_main_generator", - executable = True, - cfg = "exec", - ), "_template": attr.label( default = "//python/private:repl_template.py", allow_single_file = True, From d48433b3f256b105efd625dbffeedec27e3b4d5a Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 18:31:07 -0700 Subject: [PATCH 16/27] Incorporate some of the feedback --- python/bin/repl_stub.py | 15 +++++++++++++++ python/private/repl.bzl | 19 +++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py index 3a4e702d88..ecf96fcfb5 100644 --- a/python/bin/repl_stub.py +++ b/python/bin/repl_stub.py @@ -1,3 +1,18 @@ +"""Simulates the REPL that Python spawns when invoking the binary with no arguments. + +The code module is responsible for the default shell. + +The import and `ocde.interact()` call here his is equivalent to doing: + + $ python3 -m code + Python 3.11.2 (main, Mar 13 2023, 12:18:29) [GCC 12.2.0] on linux + Type "help", "copyright", "credits" or "license" for more information. + (InteractiveConsole) + >>> + +The logic for PYTHONSTARTUP is handled in python/private/repl_template.py. +""" + import code code.interact() diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 76a7004ff2..ab3b904fe4 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,17 +1,3 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Implementation of the rules to expose a REPL.""" load("//python:py_binary.bzl", "py_binary") @@ -33,19 +19,24 @@ _generate_repl_main = rule( attrs = { "out": attr.output( mandatory = True, + doc = "The path to the file to generate.", ), "stub": attr.label( mandatory = True, allow_single_file = True, + doc = ("The stub responsible for actually invoking the final shell. " + + "See the \"Customizing the REPL\" docs for details."), ), "_template": attr.label( default = "//python/private:repl_template.py", allow_single_file = True, + doc = "The template to use for generating `out`.", ), }, ) def py_repl_binary(name, stub, deps = [], data = [], **kwargs): + """A _generate_repl_main( name = "%s_py" % name, stub = stub, From 2aa2a2729d36dd92f84a9797d9df8cc713cfc4ff Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 21:43:50 -0700 Subject: [PATCH 17/27] more feedback incorporated --- python/private/repl.bzl | 30 +++++++++++++++++++++++++++--- python/private/repl_template.py | 3 +++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index ab3b904fe4..ac0da982be 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,11 +1,12 @@ """Implementation of the rules to expose a REPL.""" -load("//python:py_binary.bzl", "py_binary") +load("//python:py_binary.bzl", _py_binary="py_binary") def _generate_repl_main_impl(ctx): stub_repo = ctx.attr.stub.label.repo_name or ctx.workspace_name stub_path = "/".join([stub_repo, ctx.file.stub.short_path]) + # Point the generated main file at the stub. ctx.actions.expand_template( template = ctx.file._template, output = ctx.outputs.out, @@ -33,17 +34,40 @@ _generate_repl_main = rule( doc = "The template to use for generating `out`.", ), }, + doc = """\ +Generates a "main" script for a py_binary target that starts a Python REPL. + +The template is designed to take care of the majority of the logic. The user +customizes the exact shell that will be started via the stub. The stub is a +simple shell script that imports the desired shell and then executes it. +""", ) def py_repl_binary(name, stub, deps = [], data = [], **kwargs): - """A + """A py_binary target that executes a REPL when run. + + The stub is the script that ultimately decides which shell the REPL will run. + It can be as simple as this: + + import code + code.interact() + + Or it can load something like IPython instead. + + Args: + name: Name of the generated py_binary target. + stub: The script that invokes the shell. + deps: The dependencies of the py_binary. + data: The runtime dependencies of the py_binary. + **kwargs: Forwarded to the py_binary. + """ _generate_repl_main( name = "%s_py" % name, stub = stub, out = "%s.py" % name, ) - py_binary( + _py_binary( name = name, srcs = [ ":%s.py" % name, diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 32fd12ac9f..5b1c9ac3a3 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -11,6 +11,9 @@ def start_repl(): # Simulate Python's behavior when a valid startup script is defined by the # PYTHONSTARTUP variable. If this file path fails to load, print the error # and revert to the default behavior. + # + # See upstream for more information: + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP if startup_file := os.getenv("PYTHONSTARTUP"): try: source_code = Path(startup_file).read_text() From 2c6f54460faa730c3b83ff5829a0b750584fa80d Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 23:31:56 -0700 Subject: [PATCH 18/27] add tests --- tests/support/sh_py_run_test.bzl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 9c8134ff40..f6ebc506cc 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -38,6 +38,8 @@ def _perform_transition_impl(input_settings, attr, base_impl): settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains if attr.python_src: settings["//python/bin:python_src"] = attr.python_src + if attr.repl_dep: + settings["//python/bin:repl_dep"] = attr.repl_dep if attr.venvs_use_declare_symlink: settings["//python/config_settings:venvs_use_declare_symlink"] = attr.venvs_use_declare_symlink if attr.venvs_site_packages: @@ -47,6 +49,7 @@ def _perform_transition_impl(input_settings, attr, base_impl): _RECONFIG_INPUTS = [ "//python/config_settings:bootstrap_impl", "//python/bin:python_src", + "//python/bin:repl_dep", "//command_line_option:extra_toolchains", "//python/config_settings:venvs_use_declare_symlink", "//python/config_settings:venvs_site_packages", @@ -70,6 +73,7 @@ toolchain. """, ), "python_src": attrb.Label(), + "repl_dep": attrb.Label(), "venvs_site_packages": attrb.String(), "venvs_use_declare_symlink": attrb.String(), } From 200e5e814d1c750924700254ac68fce2a67736ef Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 23:33:53 -0700 Subject: [PATCH 19/27] precommit --- python/private/repl.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index ac0da982be..b4b74d888c 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -1,6 +1,6 @@ """Implementation of the rules to expose a REPL.""" -load("//python:py_binary.bzl", _py_binary="py_binary") +load("//python:py_binary.bzl", _py_binary = "py_binary") def _generate_repl_main_impl(ctx): stub_repo = ctx.attr.stub.label.repo_name or ctx.workspace_name From 5f4643747ecfc55632192ee7b6ffd92bc26886d2 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 23:36:32 -0700 Subject: [PATCH 20/27] add tests --- tests/repl/BUILD.bazel | 44 ++++++++++++++++++++ tests/repl/helper/test_module.py | 4 ++ tests/repl/repl_test.py | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 tests/repl/BUILD.bazel create mode 100644 tests/repl/helper/test_module.py create mode 100644 tests/repl/repl_test.py diff --git a/tests/repl/BUILD.bazel b/tests/repl/BUILD.bazel new file mode 100644 index 0000000000..be9b4f17e9 --- /dev/null +++ b/tests/repl/BUILD.bazel @@ -0,0 +1,44 @@ +load("//python:py_library.bzl", "py_library") +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") + +# A library that adds a special import path only when this is specified as a +# dependency. This makes it easy for a dependency to have this import path +# available without the top-level target being able to import the module. +py_library( + name = "helper/test_module", + srcs = [ + "helper/test_module.py", + ], + imports = [ + "helper", + ], +) + +py_reconfig_test( + name = "repl_without_dep_test", + srcs = ["repl_test.py"], + main = "repl_test.py", + data = [ + "//python/bin:repl", + ], + env = { + # The helper/test_module should _not_ be importable for this test. + "EXPECT_TEST_MODULE_IMPORTABLE": "0", + }, + python_version = "3.12", +) + +py_reconfig_test( + name = "repl_with_dep_test", + srcs = ["repl_test.py"], + main = "repl_test.py", + data = [ + "//python/bin:repl", + ], + env = { + # The helper/test_module _should_ be importable for this test. + "EXPECT_TEST_MODULE_IMPORTABLE": "1", + }, + python_version = "3.12", + repl_dep = ":helper/test_module", +) diff --git a/tests/repl/helper/test_module.py b/tests/repl/helper/test_module.py new file mode 100644 index 0000000000..f3d4d96a81 --- /dev/null +++ b/tests/repl/helper/test_module.py @@ -0,0 +1,4 @@ +"""This is a file purely intended for validating //python/bin:repl.""" + +def print_hello(): + print("Hello World") diff --git a/tests/repl/repl_test.py b/tests/repl/repl_test.py new file mode 100644 index 0000000000..33bcb32a1a --- /dev/null +++ b/tests/repl/repl_test.py @@ -0,0 +1,70 @@ +import os +import subprocess +import sys +import unittest +from typing import Iterable + +from python import runfiles + +rfiles = runfiles.Create() + +# Signals the tests below whether we should be expecting the import of +# helpers/test_module.py on the REPL to work or not. +EXPECT_TEST_MODULE_IMPORTABLE = (os.environ["EXPECT_TEST_MODULE_IMPORTABLE"] == "1") + +class ReplTest(unittest.TestCase): + def setUp(self): + self.repl = rfiles.Rlocation("rules_python/python/bin/repl") + assert self.repl + + + def run_code_in_repl(self, lines: Iterable[str]) -> str: + """Runs the lines of code in the REPL and returns the text output.""" + return subprocess.check_output( + [self.repl], + text=True, + stderr=subprocess.STDOUT, + input="\n".join(lines), + ).strip() + + + + def test_repl_version(self): + """Validates that we can successfully execute arbitrary code on the REPL.""" + + result = self.run_code_in_repl([ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ]) + self.assertIn("version: 3.12", result) + + def test_cannot_import_test_module_directly(self): + """Validates that we cannot import helper/test_module.py since it's not a direct dep.""" + with self.assertRaises(ModuleNotFoundError): + import test_module + + + + + + + @unittest.skipIf(not EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set") + def test_import_test_module_success(self): + """Validates that we can import helper/test_module.py when repl_dep is set.""" + result = self.run_code_in_repl([ + "import test_module", + "test_module.print_hello()", + ]) + self.assertIn("Hello World", result) + @unittest.skipIf(EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set") + def test_import_test_module_failure(self): + """Validates that we cannot import helper/test_module.py when repl_dep isn't set.""" + result = self.run_code_in_repl([ + "import test_module", + ]) + self.assertIn("ModuleNotFoundError: No module named 'test_module'", result) + + +if __name__ == "__main__": + unittest.main() From 98064dcb3d7f041ac7d89be9229b9a7a800a4710 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 27 Apr 2025 23:37:02 -0700 Subject: [PATCH 21/27] precommit --- tests/repl/BUILD.bazel | 4 +-- tests/repl/helper/test_module.py | 1 + tests/repl/repl_test.py | 50 +++++++++++++++++--------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/tests/repl/BUILD.bazel b/tests/repl/BUILD.bazel index be9b4f17e9..62c7377d53 100644 --- a/tests/repl/BUILD.bazel +++ b/tests/repl/BUILD.bazel @@ -17,7 +17,6 @@ py_library( py_reconfig_test( name = "repl_without_dep_test", srcs = ["repl_test.py"], - main = "repl_test.py", data = [ "//python/bin:repl", ], @@ -25,13 +24,13 @@ py_reconfig_test( # The helper/test_module should _not_ be importable for this test. "EXPECT_TEST_MODULE_IMPORTABLE": "0", }, + main = "repl_test.py", python_version = "3.12", ) py_reconfig_test( name = "repl_with_dep_test", srcs = ["repl_test.py"], - main = "repl_test.py", data = [ "//python/bin:repl", ], @@ -39,6 +38,7 @@ py_reconfig_test( # The helper/test_module _should_ be importable for this test. "EXPECT_TEST_MODULE_IMPORTABLE": "1", }, + main = "repl_test.py", python_version = "3.12", repl_dep = ":helper/test_module", ) diff --git a/tests/repl/helper/test_module.py b/tests/repl/helper/test_module.py index f3d4d96a81..0c4a309b01 100644 --- a/tests/repl/helper/test_module.py +++ b/tests/repl/helper/test_module.py @@ -1,4 +1,5 @@ """This is a file purely intended for validating //python/bin:repl.""" + def print_hello(): print("Hello World") diff --git a/tests/repl/repl_test.py b/tests/repl/repl_test.py index 33bcb32a1a..51ca951110 100644 --- a/tests/repl/repl_test.py +++ b/tests/repl/repl_test.py @@ -10,14 +10,14 @@ # Signals the tests below whether we should be expecting the import of # helpers/test_module.py on the REPL to work or not. -EXPECT_TEST_MODULE_IMPORTABLE = (os.environ["EXPECT_TEST_MODULE_IMPORTABLE"] == "1") +EXPECT_TEST_MODULE_IMPORTABLE = os.environ["EXPECT_TEST_MODULE_IMPORTABLE"] == "1" + class ReplTest(unittest.TestCase): def setUp(self): self.repl = rfiles.Rlocation("rules_python/python/bin/repl") assert self.repl - def run_code_in_repl(self, lines: Iterable[str]) -> str: """Runs the lines of code in the REPL and returns the text output.""" return subprocess.check_output( @@ -27,16 +27,16 @@ def run_code_in_repl(self, lines: Iterable[str]) -> str: input="\n".join(lines), ).strip() - - def test_repl_version(self): """Validates that we can successfully execute arbitrary code on the REPL.""" - result = self.run_code_in_repl([ - "import sys", - "v = sys.version_info", - "print(f'version: {v.major}.{v.minor}')", - ]) + result = self.run_code_in_repl( + [ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ] + ) self.assertIn("version: 3.12", result) def test_cannot_import_test_module_directly(self): @@ -44,25 +44,29 @@ def test_cannot_import_test_module_directly(self): with self.assertRaises(ModuleNotFoundError): import test_module - - - - - - @unittest.skipIf(not EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set") + @unittest.skipIf( + not EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set" + ) def test_import_test_module_success(self): """Validates that we can import helper/test_module.py when repl_dep is set.""" - result = self.run_code_in_repl([ - "import test_module", - "test_module.print_hello()", - ]) + result = self.run_code_in_repl( + [ + "import test_module", + "test_module.print_hello()", + ] + ) self.assertIn("Hello World", result) - @unittest.skipIf(EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set") + + @unittest.skipIf( + EXPECT_TEST_MODULE_IMPORTABLE, "test only works without repl_dep set" + ) def test_import_test_module_failure(self): """Validates that we cannot import helper/test_module.py when repl_dep isn't set.""" - result = self.run_code_in_repl([ - "import test_module", - ]) + result = self.run_code_in_repl( + [ + "import test_module", + ] + ) self.assertIn("ModuleNotFoundError: No module named 'test_module'", result) From 32c436e2eda981b7d89befb3fac4babc0e6cc27f Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 4 May 2025 15:34:14 -0700 Subject: [PATCH 22/27] Fix the missing banner on startup --- python/bin/repl_stub.py | 13 ++++++++++++- python/private/repl.bzl | 16 +++++++++------- python/private/repl_template.py | 6 ++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py index ecf96fcfb5..86452aa869 100644 --- a/python/bin/repl_stub.py +++ b/python/bin/repl_stub.py @@ -14,5 +14,16 @@ """ import code +import sys -code.interact() +if sys.stdin.isatty(): + # Use the default options. + exitmsg = None +else: + # On a non-interactive console, we want to suppress the >>> and the exit message. + exitmsg = "" + sys.ps1 = "" + sys.ps2 = "" + +# We set the banner to an empty string because the repl_template.py file already prints the banner. +code.interact(banner="", exitmsg=exitmsg) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index b4b74d888c..911ab0c6a1 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -6,22 +6,22 @@ def _generate_repl_main_impl(ctx): stub_repo = ctx.attr.stub.label.repo_name or ctx.workspace_name stub_path = "/".join([stub_repo, ctx.file.stub.short_path]) + out = ctx.actions.declare_file(ctx.label.name + ".py") + # Point the generated main file at the stub. ctx.actions.expand_template( template = ctx.file._template, - output = ctx.outputs.out, + output = out, substitutions = { "%stub_path%": stub_path, }, ) + return [DefaultInfo(files=depset([out]))] + _generate_repl_main = rule( implementation = _generate_repl_main_impl, attrs = { - "out": attr.output( - mandatory = True, - doc = "The path to the file to generate.", - ), "stub": attr.label( mandatory = True, allow_single_file = True, @@ -40,6 +40,8 @@ Generates a "main" script for a py_binary target that starts a Python REPL. The template is designed to take care of the majority of the logic. The user customizes the exact shell that will be started via the stub. The stub is a simple shell script that imports the desired shell and then executes it. + +The target's name is used for the output filename (with a .py extension). """, ) @@ -64,14 +66,14 @@ def py_repl_binary(name, stub, deps = [], data = [], **kwargs): _generate_repl_main( name = "%s_py" % name, stub = stub, - out = "%s.py" % name, ) _py_binary( name = name, srcs = [ - ":%s.py" % name, + ":%s_py" % name, ], + main = "%s_py.py" % name, data = data + [ stub, ], diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 5b1c9ac3a3..0e058b23ae 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -1,5 +1,6 @@ import os import runpy +import sys from pathlib import Path from python.runfiles import runfiles @@ -8,6 +9,11 @@ def start_repl(): + if sys.stdin.isatty(): + # Print the banner similar to how python does it on startup when running interactively. + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' + sys.stderr.write("Python %s on %s\n%s\n" % (sys.version, sys.platform, cprt)) + # Simulate Python's behavior when a valid startup script is defined by the # PYTHONSTARTUP variable. If this file path fails to load, print the error # and revert to the default behavior. From 863e970decb08962197fe5a6135a44c0c1dd239b Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 4 May 2025 15:35:41 -0700 Subject: [PATCH 23/27] precommit --- python/private/repl.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/repl.bzl b/python/private/repl.bzl index 911ab0c6a1..838166a187 100644 --- a/python/private/repl.bzl +++ b/python/private/repl.bzl @@ -17,7 +17,7 @@ def _generate_repl_main_impl(ctx): }, ) - return [DefaultInfo(files=depset([out]))] + return [DefaultInfo(files = depset([out]))] _generate_repl_main = rule( implementation = _generate_repl_main_impl, From b769fb13a1c84aa8ab1638624bd626bc7e4dbf21 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 4 May 2025 15:41:37 -0700 Subject: [PATCH 24/27] add doc --- docs/repl.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/repl.md diff --git a/docs/repl.md b/docs/repl.md new file mode 100644 index 0000000000..8e4e267ce3 --- /dev/null +++ b/docs/repl.md @@ -0,0 +1,7 @@ +# Getting a REPL or Interactive Shell + +This document + +The goal with this PR here is to create a REPL in an environment identical to what you would see +when running your code in a py_binary. I.e. all the things that the first and second stage +bootstrap script do. The three that I can think of: From 44dda2fbaa363bcdcfaadd571bd3abc06d660a3a Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 4 May 2025 16:23:20 -0700 Subject: [PATCH 25/27] add some docs --- docs/repl.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/repl.md b/docs/repl.md index 8e4e267ce3..5339c760fc 100644 --- a/docs/repl.md +++ b/docs/repl.md @@ -1,7 +1,33 @@ # Getting a REPL or Interactive Shell -This document +rules_python provides a REPL to help with debugging and developing. The goal of +the REPL is to present an environment identical to what a `py_binary` creates +for your code. -The goal with this PR here is to create a REPL in an environment identical to what you would see -when running your code in a py_binary. I.e. all the things that the first and second stage -bootstrap script do. The three that I can think of: +## Usage + + + +## Customizing the shell + +By default, the `//python/bin:repl` target will invoke the shell from the `code` +module. It's possible to switch to another shell by writing a custom "stub" and +pointing the target at the necessary dependencies. + +For an IPython shell, create a file as follows. + +```python +import IPython +IPython.start_ipython() +``` + +Assuming the file is called `ipython_stub.py` and the `pip.parse` hub's name is +`my_deps`, then set this up in the .bazelrc file: +``` +# Allow the REPL stub to import ipython. In this case, @my_deps is the hub name +# of the pip.parse() call. +build --@rules_python//python/bin:repl_stub_dep=@my_deps//ipython + +# Point the REPL at the stub created above. +build --@rules_python//python/bin:repl_stub=//path/to:ipython_stub.py +``` From d9bed354f5bb803daa28956cd49132b83c6831b7 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Sun, 4 May 2025 22:41:49 -0700 Subject: [PATCH 26/27] flesh out docs a bit --- docs/index.md | 1 + docs/repl.md | 35 ++++++++++++++++++++++++++++++++++- docs/toolchains.md | 10 ++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index b10b445983..285b1cd66e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -101,6 +101,7 @@ pip coverage precompiling gazelle +REPL Extending Contributing support diff --git a/docs/repl.md b/docs/repl.md index 5339c760fc..f56ad56ffc 100644 --- a/docs/repl.md +++ b/docs/repl.md @@ -6,7 +6,38 @@ for your code. ## Usage +Start the REPL with the following command: +```console +$ bazel run @rules_python//python/bin:repl +Python 3.11.11 (main, Mar 17 2025, 21:02:09) [Clang 20.1.0 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +Settings like `//python/config_settings:python_version` will influence the exact +behaviour. +```console +$ bazel run @rules_python//python/bin:repl --@rules_python//python/config_settings:python_version=3.13 +Python 3.13.2 (main, Mar 17 2025, 21:02:54) [Clang 20.1.0 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +See [//python/config_settings](api/rules_python/python/config_settings/index) +and [Environment Variables](environment-variables) for more settings. +## Importing Python targets + +The `//python/bin:repl_dep` command line flag gives the REPL access to a target +that provides the `PyInfo` provider. + +```console +$ bazel run @rules_python//python/bin:repl --@rules_python//python/bin:repl_dep=@rules_python//tools:wheelmaker +Python 3.11.11 (main, Mar 17 2025, 21:02:09) [Clang 20.1.0 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import tools.wheelmaker +>>> +``` ## Customizing the shell @@ -14,6 +45,8 @@ By default, the `//python/bin:repl` target will invoke the shell from the `code` module. It's possible to switch to another shell by writing a custom "stub" and pointing the target at the necessary dependencies. +### IPython Example + For an IPython shell, create a file as follows. ```python @@ -22,7 +55,7 @@ IPython.start_ipython() ``` Assuming the file is called `ipython_stub.py` and the `pip.parse` hub's name is -`my_deps`, then set this up in the .bazelrc file: +`my_deps`, set this up in the .bazelrc file: ``` # Allow the REPL stub to import ipython. In this case, @my_deps is the hub name # of the pip.parse() call. diff --git a/docs/toolchains.md b/docs/toolchains.md index c8305e8f0d..121b398f7a 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -757,3 +757,13 @@ a fixed version. The `python` target does not provide access to any modules from `py_*` targets on its own. Please file a feature request if this is desired. ::: + +### Differences from `//python/bin:repl` + +The `//python/bin:python` target provides access to the underlying interpreter +without any hermeticity guarantees. + +The [`//python/bin:repl` target](repl) provides an environment indentical to +what `py_binary` provides. That means it handles things like the +[`PYTHONSAFEPATH`](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSAFEPATH) +environment variable automatically. The `//python/bin:python` target will not. From 520c98004ca36cef3e91c2769923cf0e905ebfc4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 15 May 2025 23:33:56 +0900 Subject: [PATCH 27/27] docs changes --- CHANGELOG.md | 2 ++ docs/repl.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94487219bc..a6ba65eb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,8 @@ END_UNRELEASED_TEMPLATE * (pypi) Starlark-based evaluation of environment markers (requirements.txt conditionals) available (not enabled by default) for improved multi-platform build support. Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it. +* (utils) Add a way to run a REPL for any `rules_python` target that returns + a `PyInfo` provider. {#v0-0-0-removed} ### Removed diff --git a/docs/repl.md b/docs/repl.md index f56ad56ffc..edcf37e811 100644 --- a/docs/repl.md +++ b/docs/repl.md @@ -1,7 +1,7 @@ # Getting a REPL or Interactive Shell rules_python provides a REPL to help with debugging and developing. The goal of -the REPL is to present an environment identical to what a `py_binary` creates +the REPL is to present an environment identical to what a {bzl:obj}`py_binary` creates for your code. ## Usage @@ -29,7 +29,7 @@ and [Environment Variables](environment-variables) for more settings. ## Importing Python targets The `//python/bin:repl_dep` command line flag gives the REPL access to a target -that provides the `PyInfo` provider. +that provides the {bzl:obj}`PyInfo` provider. ```console $ bazel run @rules_python//python/bin:repl --@rules_python//python/bin:repl_dep=@rules_python//tools:wheelmaker