docs: docgen python apis (bazel-contrib#2149)

Uses autodoc2 to generate Python documentation for runfiles and
sphinx_bzl.

This provides some basic API doc for our Python code. They don't look
particularly great,
yet, but we can work on how they look in another PR.

Also:
* Fixes position of license rule and extra space in license text
* Forces sphinx_rtd_theme >= 2.0. uv kept trying to downgrade it for
some reason.
* Use directives markup to document the sphinx_bzl directives
* Add `sphinx_run` rule to make it easier to run Sphinx interactively
for debugging

Author: rickeylev

docs/BUILD.bazel

@@ -19,10 +19,13 @@ load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable
 load("//python/uv/private:lock.bzl", "lock")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs")
+load("//sphinxdocs:sphinx_docs_library.bzl", "sphinx_docs_library")
 load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardoc", "sphinx_stardocs")
 
 package(default_visibility = ["//:__subpackages__"])
 
+licenses(["notice"])  # Apache 2.0
+
 # We only build for Linux and Mac because:
 # 1. The actual doc process only runs on Linux
 # 2. Mac is a common development platform, and is close enough to Linux
@@ -68,6 +71,7 @@ sphinx_docs(
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
     deps = [
         ":bzl_api_docs",
+        ":py_api_srcs",
         "//sphinxdocs/docs:docs_lib",
     ],
 )
@@ -97,15 +101,15 @@ sphinx_stardocs(
         # This depends on @pythons_hub, which is only created under bzlmod,
         "//python/extensions:pip_bzl",
     ] if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else []),
-    prefix = "api/",
+    prefix = "api/rules_python/",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
 )
 
 sphinx_stardoc(
     name = "py_cc_toolchain",
     src = "//python/private:py_cc_toolchain_rule.bzl",
-    prefix = "api/",
+    prefix = "api/rules_python/",
     public_load_path = "//python/cc:py_cc_toolchain.bzl",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
@@ -115,11 +119,20 @@ sphinx_stardoc(
 sphinx_stardoc(
     name = "py_runtime_pair",
     src = "//python/private:py_runtime_pair_rule_bzl",
+    prefix = "api/rules_python",
     public_load_path = "//python:py_runtime_pair.bzl",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
 )
 
+sphinx_docs_library(
+    name = "py_api_srcs",
+    srcs = [
+        "//python/runfiles",
+    ],
+    strip_prefix = "python/",
+)
+
 readthedocs_install(
     name = "readthedocs_install",
     docs = [":docs"],
@@ -135,6 +148,8 @@ sphinx_build_binary(
         requirement("myst_parser"),
         requirement("readthedocs_sphinx_ext"),
         requirement("typing_extensions"),
+        requirement("sphinx_autodoc2"),
+        requirement("sphinx_reredirects"),
         "//sphinxdocs/src/sphinx_bzl",
     ],
 )
@@ -147,8 +162,6 @@ lock(
     upgrade = True,
 )
 
-licenses(["notice"])  # Apache 2.0
-
 # Temporary compatibility aliases for some other projects depending on the old
 # bzl_library targets.
 alias(

docs/api/index.md

@@ -2,5 +2,6 @@
 
 ```{toctree}
 :glob:
-**
+*
+*/index
 ```

docs/api/rules_python/index.md

@@ -0,0 +1,8 @@
+# rules_python Bazel APIs
+
+API documentation for rules_python Bazel objects.
+
+```{toctree}
+:glob:
+**
+```

docs/api/python/cc/index.md renamed to docs/api/rules_python/python/cc/index.md

@@ -18,18 +18,81 @@
 # Any extensions here not built into Sphinx must also be added to
 # the dependencies of //docs:sphinx-builder
 extensions = [
-    "sphinx.ext.autodoc",
+    "autodoc2",
     "sphinx.ext.autosectionlabel",
-    "sphinx.ext.autosummary",
     "sphinx.ext.doctest",
     "sphinx.ext.duration",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
     "myst_parser",
     "sphinx_rtd_theme",  # Necessary to get jquery to make flyout work
     "sphinx_bzl.bzl",
+    "sphinx_reredirects",
 ]
 
+autodoc2_packages = [
+    "sphinx_bzl",
+    "runfiles",
+]
+
+autodoc2_output_dir = "api/py"
+autodoc2_sort_names = True
+autodoc2_class_docstring = "both"
+autodoc2_index_template = """
+Python APIs
+====================
+
+This page contains auto-generated API reference documentation [#f1]_.
+
+.. toctree::
+   :titlesonly:
+
+{% for package in top_level %}
+   {{ package }}
+{%- endfor %}
+
+.. [#f1] Created with `sphinx-autodoc2 <https://github.com/chrisjsewell/sphinx-autodoc2>`_
+
+"""
+
+
+autodoc2_docstring_parser_regexes = [
+    (".*", "myst"),
+]
+
+# NOTE: The redirects generation will clobber existing files.
+redirects = {
+    "api/tools/precompiler/index": "/api/rules_python/tools/precompiler/index.html",
+    "api/python/py_library": "/api/rules_python/python/py_library.html",
+    "api/python/py_binary": "/api/rules_python/python/py_binary.html",
+    "api/python/py_test": "/api/rules_python/python/py_test.html",
+    "api/python/defs": "/api/rules_python/python/defs.html",
+    "api/python/index": "/api/rules_python/python/index.html",
+    "api/python/py_runtime_info": "/api/rules_python/python/py_runtime_info.html",
+    "api/python/private/common/py_library_rule_bazel": "/api/rules_python/python/private/common/py_library_rule_bazel.html",
+    "api/python/private/common/py_test_rule_bazel": "/api/rules_python/python/private/common/py_test_rule_bazel.html",
+    "api/python/private/common/py_binary_rule_bazel": "/api/rules_python/python/private/common/py_binary_rule_bazel.html",
+    "api/python/private/common/py_runtime_rule": "/api/rules_python/python/private/common/py_runtime_rule.html",
+    "api/python/extensions/pip": "/api/rules_python/python/extensions/pip.html",
+    "api/python/extensions/python": "/api/rules_python/python/extensions/python.html",
+    "api/python/entry_points/py_console_script_binary": "/api/rules_python/python/entry_points/py_console_script_binary.html",
+    "api/python/cc/py_cc_toolchain_info": "/api/rules_python/python/cc/py_cc_toolchain_info.html",
+    "api/python/cc/index": "/api/rules_python/python/cc/index.html",
+    "api/python/py_cc_link_params_info": "/api/rules_python/python/py_cc_link_params_info.html",
+    "api/python/runtime_env_toolchains/index": "/api/rules_python/python/runtime_env_toolchains/index.html",
+    "api/python/pip": "/api/rules_python/python/pip.html",
+    "api/python/config_settings/index": "/api/rules_python/python/config_settings/index.html",
+    "api/python/packaging": "/api/rules_python/python/packaging.html",
+    "api/python/py_runtime": "/api/rules_python/python/py_runtime.html",
+    "api/sphinxdocs/sphinx": "/api/sphinxdocs/sphinxdocs/sphinx.html",
+    "api/sphinxdocs/sphinx_stardoc": "/api/sphinxdocs/sphinxdocs/sphinx_stardoc.html",
+    "api/sphinxdocs/readthedocs": "/api/sphinxdocs/sphinxdocs/readthedocs.html",
+    "api/sphinxdocs/index": "/api/sphinxdocs/sphinxdocs/index.html",
+    "api/sphinxdocs/private/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/private/sphinx_docs_library.html",
+    "api/sphinxdocs/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/sphinx_docs_library.html",
+    "api/sphinxdocs/inventories/index": "/api/sphinxdocs/sphinxdocs/inventories/index.html",
+}
+
 # Adapted from the template code:
 # https://github.com/readthedocs/readthedocs.org/blob/main/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl
 if os.environ.get("READTHEDOCS") == "True":

docs/api/python/config_settings/index.md renamed to docs/api/rules_python/python/config_settings/index.md

@@ -5,10 +5,12 @@ version = "0.0.0"
 dependencies = [
     # NOTE: This is only used as input to create the resolved requirements.txt
     # file, which is what builds, both Bazel and Readthedocs, both use.
+    "sphinx-autodoc2",
     "sphinx",
     "myst-parser",
-    "sphinx_rtd_theme",
+    "sphinx_rtd_theme >=2.0", # uv insists on downgrading for some reason
     "readthedocs-sphinx-ext",
     "absl-py",
-    "typing-extensions"
+    "typing-extensions",
+    "sphinx-reredirects"
 ]