fix: remove dependency on pandoc to improve performance

Author: parthea

.github/workflows/tests.yaml

@@ -126,7 +126,7 @@ jobs:
       - name: Install system dependencies.
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl pandoc unzip
+          sudo apt-get install -y curl unzip
       - name: Install showcase
         run: |
           sudo mkdir -p /usr/src/showcase
@@ -174,7 +174,7 @@ jobs:
       - name: Install system dependencies.
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl pandoc unzip
+          sudo apt-get install -y curl unzip
       - name: Install nox.
         run: python -m pip install nox
       - name: Install protoc.
@@ -220,7 +220,7 @@ jobs:
       - name: Install system dependencies.
         run: |
             sudo apt-get update
-            sudo apt-get install -y curl pandoc unzip
+            sudo apt-get install -y curl unzip
       - name: Install protoc.
         run: |
             sudo mkdir -p /usr/src/protoc/
@@ -248,7 +248,7 @@ jobs:
       - name: Install system dependencies.
         run: |
             sudo apt-get update
-            sudo apt-get install -y curl pandoc unzip
+            sudo apt-get install -y curl unzip
       - name: Install protoc.
         run: |
             sudo mkdir -p /usr/src/protoc/
@@ -276,7 +276,7 @@ jobs:
       - name: Install system dependencies.
         run: |
             sudo apt-get update
-            sudo apt-get install -y curl pandoc unzip
+            sudo apt-get install -y curl unzip
       - name: Install protoc.
         run: |
             sudo mkdir -p /usr/src/protoc/
@@ -302,7 +302,7 @@ jobs:
       - name: Install system dependencies.
         run: |
             sudo apt-get update
-            sudo apt-get install -y curl pandoc unzip
+            sudo apt-get install -y curl unzip
       - name: Install nox.
         run: python -m pip install nox
       - name: Check autogenerated snippets.
@@ -323,10 +323,6 @@ jobs:
           python-version: "${{ matrix.python }}"
           cache: 'pip'
           allow-prereleases: true
-      - name: Install pandoc
-        run: |
-            sudo apt-get update
-            sudo apt-get install -y pandoc gcc git
       - name: Install nox.
         run: |
             python -m pip install nox
@@ -349,10 +345,6 @@ jobs:
           python-version: "${{ matrix.python }}"
           cache: 'pip'
           allow-prereleases: true
-      - name: Install pandoc
-        run: |
-            sudo apt-get update
-            sudo apt-get install -y pandoc gcc git
       - name: Install nox.
         run: |
             python -m pip install nox

BUILD.bazel

@@ -1,56 +1,6 @@
-load("//:gapic_generator_python.bzl", "pandoc_binary", "pandoc_toolchain")
 load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement")
 load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
 
-toolchain_type(
-    name = "pandoc_toolchain_type",
-    visibility = ["//visibility:public"],
-)
-
-pandoc_toolchain(
-    exec_compatible_with = [
-        "@platforms//os:linux",
-        "@platforms//cpu:arm64",
-    ],
-    platform = "linux_arm_64",
-)
-
-pandoc_toolchain(
-    exec_compatible_with = [
-        "@platforms//os:linux",
-        "@platforms//cpu:x86_64",
-    ],
-    platform = "linux_x86_64",
-)
-
-pandoc_toolchain(
-    exec_compatible_with = [
-        "@platforms//os:osx",
-        "@platforms//cpu:arm64",
-    ],
-    platform = "macOS_arm_64",
-)
-
-pandoc_toolchain(
-    exec_compatible_with = [
-        "@platforms//os:osx",
-        "@platforms//cpu:x86_64",
-    ],
-    platform = "macOS_x86_64",
-)
-
-pandoc_toolchain(
-    exec_compatible_with = [
-        "@platforms//os:windows",
-        "@platforms//cpu:x86_64",
-    ],
-    platform = "windows_x86_64",
-)
-
-pandoc_binary(
-    name = "pandoc_binary",
-)
-
 py_runtime(
     name = "pyenv3_runtime",
     interpreter = ":pyenv3wrapper.sh",
@@ -71,11 +21,11 @@ toolchain(
 py_binary(
     name = "gapic_plugin",
     srcs = glob(["gapic/**/*.py"]),
-    data = [":pandoc_binary"] + glob([
+    data = glob([
         "gapic/**/*.j2",
         "gapic/**/.*.j2",
     ]),
-    main = "gapic/cli/generate_with_pandoc.py",
+    main = "gapic/cli/generate_gapic.py",
     python_version = "PY3",
     visibility = ["//visibility:public"],
     deps = [
@@ -86,7 +36,6 @@ py_binary(
         requirement("googleapis-common-protos"),
         requirement("jinja2"),
         requirement("MarkupSafe"),
-        requirement("pypandoc"),
         requirement("PyYAML"),
         requirement("grpc-google-iam-v1"),
     ],

WORKSPACE

@@ -53,12 +53,10 @@ install_deps()
 load(
     "//:repositories.bzl",
     "gapic_generator_python",
-    "gapic_generator_register_toolchains",
 )
 
 gapic_generator_python()
 
-gapic_generator_register_toolchains()
 
 _grpc_version = "1.71.0"
 
@@ -123,3 +121,9 @@ switched_rules_by_language(
     gapic = True,
     grpc = True,
 )
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+python_register_toolchains(
+    name = "python311",
+    python_version = "3.11",
+)
+

docs/getting-started/local.rst

@@ -48,19 +48,6 @@ To ensure it is installed propertly:
   libprotoc 3.6.0
 
 
-pandoc
-~~~~~~
-
-This generator relies on `pandoc`_ to convert from Markdown (the *lingua
-franca* for documentation in protocol buffers) into ReStructured Text (the
-*lingua franca* for documentation in Python).
-
-Install this using an appropriate mechanism for your operating system.
-Multiple installation paths are documented on the `pandoc installation page`_.
-
-.. _pandoc: https://pandoc.org/
-.. _pandoc installation page: https://pandoc.org/installing.html
-
 
 API Generator for Python
 ~~~~~~~~~~~~~~~~~~~~~~~~

gapic/cli/generate_with_pandoc.py renamed to gapic/cli/generate_gapic.py

@@ -2,9 +2,6 @@
 import sys
 
 if __name__ == "__main__":
-    os.environ["PYPANDOC_PANDOC"] = os.path.join(
-        os.path.abspath(__file__).rsplit("gapic", 1)[0], "pandoc"
-    )
     os.environ["LC_ALL"] = "C.UTF-8"
     os.environ["PYTHONNOUSERSITE"] = "True"
 

gapic/utils/rst.py

@@ -15,8 +15,6 @@
 import re
 from typing import Optional
 
-import pypandoc  # type: ignore
-
 from gapic.utils.lines import wrap
 
 
@@ -43,29 +41,12 @@ def rst(
     Returns:
         str: The same text, in RST format.
     """
-    # Quick check: If the text block does not appear to have any formatting,
-    # do not convert it.
-    # (This makes code generation significantly faster; calling out to pandoc
-    # is by far the most expensive thing we do.)
-    if not re.search(r"[|*`_[\]]", text):
-        answer = wrap(
-            text,
-            indent=indent,
-            offset=indent + 3,
-            width=width - indent,
-        )
-    else:
-        # Convert from CommonMark to ReStructured Text.
-        answer = (
-            pypandoc.convert_text(
-                text,
-                "rst",
-                format=source_format,
-                extra_args=["--columns=%d" % (width - indent)],
-            )
-            .strip()
-            .replace("\n", f"\n{' ' * indent}")
-        )
+    answer = wrap(
+        text,
+        indent=indent,
+        offset=indent + 3,
+        width=width - indent,
+    )
 
     # Add a newline to the end of the document if any line breaks are
     # already present.

gapic_generator_python.bzl

@@ -1,12 +1,5 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-_PANDOC_BUILD_FILE = """
-filegroup(
-    name = "pandoc",
-    srcs = ["bin/pandoc"],
-    visibility = ["//visibility:public"],
-)"""
-
 def gapic_generator_python():
 
     _maybe(
@@ -25,47 +18,6 @@ def gapic_generator_python():
         strip_prefix = "grpc-{}".format(_grpc_version),
         url = "https://github.com/grpc/grpc/archive/v{}.zip".format(_grpc_version),
     )
-
-    _maybe(
-        http_archive,
-        name = "pandoc_linux_arm_64",
-        build_file_content = _PANDOC_BUILD_FILE,
-        strip_prefix = "pandoc-3.7.0.2",
-        url = "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-arm64.tar.gz",
-    )
-
-    _maybe(
-        http_archive,
-        name = "pandoc_linux_x86_64",
-        build_file_content = _PANDOC_BUILD_FILE,
-        strip_prefix = "pandoc-3.7.0.2",
-        url = "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-amd64.tar.gz",
-    )
-
-    _maybe(
-        http_archive,
-        name = "pandoc_macOS_arm_64",
-        build_file_content = _PANDOC_BUILD_FILE,
-        strip_prefix = "pandoc-3.7.0.2",
-        url = "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-arm64-macOS.zip",
-    )
-
-    _maybe(
-        http_archive,
-        name = "pandoc_macOS_x86_64",
-        build_file_content = _PANDOC_BUILD_FILE,
-        strip_prefix = "pandoc-3.7.0.2",
-        url = "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-x86_64-macOS.zip",
-    )
-
-    _maybe(
-        http_archive,
-        name = "pandoc_windows_x86_64",
-        build_file_content = _PANDOC_BUILD_FILE,
-        strip_prefix = "pandoc-3.7.0.2",
-        url = "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-windows-x86_64.zip",
-    )
-
     _rules_gapic_version = "0.5.4"
     _maybe(
         http_archive,
@@ -81,14 +33,6 @@ def gapic_generator_python():
         urls = ["https://github.com/googleapis/googleapis/archive/{}.zip".format(_commit_sha)],
     )
 
-def gapic_generator_register_toolchains():
-    native.register_toolchains(
-        "@gapic_generator_python//:pandoc_toolchain_linux_arm_64",
-        "@gapic_generator_python//:pandoc_toolchain_linux_x86_64",
-        "@gapic_generator_python//:pandoc_toolchain_macOS_arm_64",
-        "@gapic_generator_python//:pandoc_toolchain_macOS_x86_64",
-        "@gapic_generator_python//:pandoc_toolchain_windows_x86_64",
-    )
 
 def _maybe(repo_rule, name, strip_repo_prefix = "", **kwargs):
     if not name.startswith(strip_repo_prefix):

repositories.bzl

@@ -4,7 +4,6 @@ googleapis-common-protos
 jinja2
 MarkupSafe
 protobuf
-pypandoc
 PyYAML
 grpc-google-iam-v1
 proto-plus