[WIP] protodoc: Don't use aspect

.bazelrc

@@ -31,6 +31,8 @@ build --action_env=PATH --host_action_env=PATH
 build --enable_platform_specific_config
 build --test_summary=terse
 
+build --experimental_proto_descriptor_sets_include_source_info
+
 # Allow tags to influence execution requirements
 common --experimental_allow_tags_propagation
 

api/BUILD

@@ -1,6 +1,6 @@
 # DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
 
-load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_descriptor_set", "proto_library")
 
 licenses(["notice"])  # Apache 2
 
@@ -322,3 +322,12 @@ filegroup(
     ],
     visibility = ["//visibility:public"],
 )
+
+proto_descriptor_set(
+    name = "v3_proto_set",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":v3_protos",
+        ":xds_protos",
+    ],
+)

docs/BUILD

@@ -4,6 +4,7 @@ load(
 )
 load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
 load("@rules_pkg//pkg:pkg.bzl", "pkg_tar")
+load("//tools/base:envoy_python.bzl", "envoy_genparallel", "envoy_pkg_filter")
 
 licenses(["notice"])  # Apache 2
 
@@ -43,6 +44,33 @@ filegroup(
     srcs = glob(["root/**/*.pb"]),
 )
 
+envoy_genparallel(
+    name = "protos_rst",
+    srcs = [":proto_names"],
+    outs = ["protos_rst.tar"],
+    args = [
+        "$(location @com_google_protobuf//:protoc)",
+        "--descriptor_set_in=$$(realpath $(location @envoy_api//:v3_proto_set))",
+        "--plugin=protoc-gen-api_proto_plugin=$(location //tools/protodoc)",
+        "--api_proto_plugin_out=$$OUTDIR",
+    ],
+    tools = [
+        "//tools/protodoc",
+        "@com_google_protobuf//:protoc",
+        "@envoy_api//:v3_proto_set",
+    ],
+    verbosity = "warn",
+)
+
+envoy_pkg_filter(
+    name = "api_rst",
+    srcs = [":protos_rst"],
+    matching = "*.rst",
+    merge_paths = {"contrib/envoy/extensions": "api-v3"},
+    remap_paths = {"envoy": "api-v3"},
+    strip_empty = True,
+)
+
 genrule(
     name = "extensions_security_rst",
     srcs = [
@@ -85,42 +113,26 @@ genquery(
 )
 
 genrule(
-    name = "empty_protos_rst",
-    srcs = [":empty_extensions.json"],
-    outs = ["empty_protos_rst.tar"],
-    cmd = """
-    $(location //tools/protodoc:generate_empty) \\
-        $(location empty_extensions.json) $@
-    """,
-    tools = ["//tools/protodoc:generate_empty"],
-)
-
-genrule(
-    name = "api_rst",
+    name = "proto_names",
     srcs = [
-        "//tools/protodoc:api_v3_protodoc",
         ":v3_proto_srcs",
+        ":xds_proto_srcs",
     ],
-    outs = ["api_rst.tar"],
+    outs = ["proto_names.txt"],
     cmd = """
-    $(location //tools/docs:generate_api_rst) \\
-        $(location v3_proto_srcs) $(locations //tools/protodoc:api_v3_protodoc) $@
+    cat $(SRCS) | cut -d/ -f3- | tr ':' '/' > $@
     """,
-    tools = ["//tools/docs:generate_api_rst"],
 )
 
 genrule(
-    name = "xds_rst",
-    srcs = [
-        "//tools/protodoc:xds_protodoc",
-        ":xds_proto_srcs",
-    ],
-    outs = ["xds_rst.tar"],
+    name = "empty_protos_rst",
+    srcs = [":empty_extensions.json"],
+    outs = ["empty_protos_rst.tar"],
     cmd = """
-    $(location //tools/docs:generate_api_rst) \\
-        $(location xds_proto_srcs) $(locations //tools/protodoc:xds_protodoc) $@
+    $(location //tools/protodoc:generate_empty) \\
+        $(location empty_extensions.json) $@
     """,
-    tools = ["//tools/docs:generate_api_rst"],
+    tools = ["//tools/protodoc:generate_empty"],
 )
 
 pkg_files(
@@ -206,7 +218,6 @@ pkg_tar(
         ":extensions_security_rst",
         ":external_deps_rst",
         ":version_history_rst",
-        ":xds_rst",
     ],
 )
 

tools/base/BUILD

@@ -1,4 +1,5 @@
 load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("//tools/base:envoy_python.bzl", "envoy_entry_point")
 
 licenses(["notice"])  # Apache 2
 
@@ -7,3 +8,9 @@ envoy_package()
 exports_files([
     "entry_point.py",
 ])
+
+envoy_entry_point(
+    name = "parallel",
+    pkg = "envoy.base.utils",
+    script = "envoy.parallel",
+)

tools/base/envoy_python.bzl

@@ -245,6 +245,135 @@ def envoy_genjson(name, srcs = [], yaml_srcs = [], filter = None, args = None):
         filter = filter,
     )
 
+def envoy_genparallel(name, args, srcs, outs, verbosity = "info", tools = [], parallel = "//tools/base:parallel"):
+    """Run batched tasks in parallel.
+
+    This is not dissimilar to how aspects work, except that the jobs are not batched one process
+    per task, and are instead batched according to the number of available cpus.
+
+    This is useful for tasks that have a high startup overhead which makes them unsuitable
+    for running in an aspect (for example `protoc`).
+
+    `args` are the command and any cli args for the command that should be run on every batch
+
+    `srcs` are labels pointing to text files containing a list of targets
+
+    `tools` are passed to the `genrule`. You can refer to these eg with `$(location x)` in `args`.
+
+    `verbosity` should be one of `info`, `warn`, `error`
+
+    `$$OUTDIR` is available when constructing args to pass your tool for output generation.
+
+    Example:
+
+    ```starlark
+
+    envoy_genparallel(
+        name = "protos_rst",
+        outs = ["protos_rst.tar"],
+        args = [
+            "$(location @com_google_protobuf//:protoc)",
+            "--descriptor_set_in=$$(realpath $(location @envoy_api//:v3_proto_set))",
+            "--plugin=protoc-gen-api_proto_plugin=$(location //tools/protodoc)",
+            "--api_proto_plugin_out=$$OUTDIR",
+        ],
+        srcs = [":proto_names"],
+        tools = [
+            "//tools/protodoc",
+            "@com_google_protobuf//:protoc",
+            "@envoy_api//:v3_proto_set",
+        ],
+        verbosity = "warn",
+    )
+    ```
+    """
+    native.genrule(
+        name = name,
+        srcs = srcs,
+        outs = outs,
+        cmd = """
+        OUTDIR=$$(mktemp -d) \
+        && PARALLEL_TARGETS=$$(cat $(SRCS)) \
+        && $(location %s) -v %s "%s" \
+            $$PARALLEL_TARGETS \
+        && tar cf $@ -C $$OUTDIR . \
+        && rm -rf $$OUTDIR
+        """ % (parallel, verbosity, " ".join(args)),
+        tools = [parallel] + tools,
+    )
+
+def envoy_pkg_filter(
+        name,
+        srcs,
+        matching = "",
+        prune = "",
+        remap_paths = {},
+        merge_paths = {},
+        strip_files = False,
+        strip_dirs = False,
+        strip_empty = False):
+    """Generate a tarball from other tarballs mangling the resulting files and directories.
+
+    `srcs` should be labels pointing to tarball files.
+
+    `matching` is a match string that is passed to `find -name "$match"`. Only matching files
+        are included in the output tarball.
+
+    `prune` is the reverse of `matching`
+
+    `remap_paths` is a mapping of `src` to `target` files or directories to `mv`.
+
+    `merge_paths` instead copies and then deletes the `src`
+
+    `strip*` removes empty `files`, `dirs` or both if `strip_empty` is set.
+
+    Example:
+
+    ```starlark
+
+    envoy_pkg_filter(
+       name = "mypkg",
+       srcs = [":some_source.tar",],
+       matching = "*.proto",
+       remap_paths = {"path1": "path3"},
+       merge_paths = {"path2/sub/dir": "path3"},
+    )
+
+    ```
+
+    """
+    strip_files = strip_empty if strip_empty else strip_files
+    strip_dirs = strip_empty if strip_empty else strip_dirs
+    commands = []
+    deletable = []
+    if matching:
+        deletable.append('-type f ! -name "%s"' % matching)
+    if prune:
+        deletable.append('-name "%s"' % prune)
+    if strip_dirs:
+        deletable.append("-type d -empty")
+    if strip_files:
+        deletable.append("-type f -empty")
+    if deletable:
+        commands.append("find $$OUTDIR %s" % (" -o ".join(["%s -delete" % d for d in deletable])))
+    for src, target in remap_paths.items():
+        commands.append("mv $$OUTDIR/%s $$OUTDIR/%s" % (src, target))
+    for src, target in merge_paths.items():
+        commands.append("cp -a $$OUTDIR/%s $$OUTDIR/%s && rm -rf $$OUTDIR/%s" % (src, target, src))
+    commands = "&& ".join(commands)
+    native.genrule(
+        name = name,
+        srcs = srcs,
+        outs = ["%s.tar" % name],
+        cmd = """
+        OUTDIR=$$(mktemp -d) \
+        && tar xf $(SRCS) -C $$OUTDIR \
+        && %s \
+        && tar cf $@ -C $$OUTDIR . \
+        && rm -rf $$OUTDIR
+        """ % commands,
+    )
+
 def envoy_py_data(name, src, format = None, entry_point = base_entry_point):
     """Preload JSON/YAML data as a python lib.
 

tools/base/requirements.in

@@ -3,7 +3,7 @@ aio.api.bazel
 cffi>=1.15.0
 colorama
 coloredlogs
-envoy.base.utils>=0.3.4
+envoy.base.utils>=0.3.5
 envoy.code.check>=0.2.1
 envoy.dependency.check>=0.1.1
 envoy.dependency.pip_check>=0.1.2

tools/docs/BUILD

@@ -24,13 +24,6 @@ py_binary(
     data = ["//bazel:all_repository_locations"],
 )
 
-py_binary(
-    name = "generate_api_rst",
-    srcs = [
-        "generate_api_rst.py",
-    ],
-)
-
 # The upstream lib is maintained here:
 #
 #    https://github.com/envoyproxy/pytooling/tree/main/envoy.docs.sphinx_runner