Implement code generation for gRPC (#9)

* Add grpcio-tools as a dependency
* Add mypy-protobuf as a dependency
* Use dataclass 'GenerationSpec' to hold gRPC details
* Detect proto files using a basepath and subpath
* Generate the protobuf descriptor set in the same call that generates the package
- Add utility methods to GenerationSpec for inspecting a generated protobuf package
* Add acceptance tests

Author: jfriedri-ni

tools/grpc_generator/README.md

@@ -3,3 +3,92 @@
 `grpc_generator` is a Python tool that generates gRPC stubs from proto files.
 
 It supports emitting stubs into namespace packages, transforming them as necessary from submodules to subpackages.
+
+## Setup
+
+```pwsh
+# Initialize the tool
+cd tools\grpc_generator
+poetry install
+```
+
+## Generate
+
+```pwsh
+# PowerShell
+poetry run grpc-generator `
+   --proto-basepath ..\..\third_party\ni-apis `
+   --proto-subpath ni\protobuf\types `
+   --output-basepath ..\..\packages\ni.protobuf.types\src `
+   --output-format submodule
+```
+
+## Generate into a namespace package
+
+```pwsh
+# PowerShell
+poetry run grpc-generator `
+   --proto-basepath ..\..\third_party\ni-apis `
+   --proto-subpath ni\measurementlink\pinmap\v1 `
+   --output-basepath ..\..\packages\ni.measurementlink.pinmap.v1\src `
+   --output-format subpackage
+```
+
+## Options
+
+**`grpc-generator --help`**
+```
+Usage: grpc-generator [OPTIONS]
+
+  Generate gRPC Python stubs from proto files.
+
+  Specifying input and output locations
+
+    This script uses the protobuf files from the folder specified by --proto-
+    basepath and --proto-subpath and emits Python files into the folder
+    specified by --output-basepath:
+
+    {proto-basepath}/{proto-subpath}  -->  {output-basepath}/{proto-subpath}
+
+    The script resolves gRPC imports from --proto-basepath by default. Include
+    additional paths by using --proto-include-path for each required folder.
+
+  Specifying output format
+
+    The script supports generating gRPC packages as either subpackages or
+    submodules with --output-format.
+
+    When generating submodules, the script creates Python files with names
+    that match the source protobuf files:
+
+    waveform.proto  -->  waveform_pb2.py
+
+    When generating subpackages, the script creates folders with names that
+    match the source protobuf files:
+
+    waveform.proto  -->  waveform_pb2/__init__.py
+
+    Clients use the same "import waveform_pb2" syntax.
+
+Options:
+  --output-basepath PATH          Emit the generated gRPC files to PATH
+                                  [required]
+  --output-format [submodule|subpackage]
+                                  Generate a Python submodule or subpackage
+                                  [required]
+  --proto-basepath PATH           Use PATH as the base for --proto-subpath
+                                  [default: C:\dev\ni\git\github\ni-apis-
+                                  python\third_party\ni-apis]
+  --proto-include-path PATH       Add PATH to the import search list, can be
+                                  used more than once  [default:
+                                  C:\dev\ni\git\github\ni-apis-
+                                  python\third_party\ni-apis]
+  --proto-subpath PATH            Use the proto files under PATH as input
+                                  [required]
+  --help                          Show this message and exit.
+  --version                       Show the version and exit.
+
+  Example:
+
+  grpc-generator  --proto-subpath ni/protobuf/types  --output-basepath ../../packages/ni.protobuf.types/src  --output-format submodule
+```

tools/grpc_generator/poetry.lock

@@ -13,6 +13,14 @@ grpc-generator = "grpc_generator.__main__:cli"
 [tool.poetry.dependencies]
 python = "^3.11"
 click = "^8.2.1"
+# When updating the grpcio-tools version, also update the minimum grpcio version
+# and regenerate gRPC stubs.
+grpcio-tools = [
+  { version = "1.49.1", python = ">=3.9,<3.12" },
+  { version = "1.59.0", python = ">=3.12,<3.13" },
+  { version = "1.67.0", python = "^3.13" },
+]
+mypy-protobuf = ">=3.6"
 
 [tool.poetry.group.lint.dependencies]
 bandit = { version = ">=1.7", extras = ["toml"] }

tools/grpc_generator/pyproject.toml

@@ -1 +1,9 @@
 """grpc_generator package."""
+
+import warnings
+
+warnings.filterwarnings(
+    action="ignore",
+    category=UserWarning,
+    module=r"^grpc_tools",
+)  # grpc_tools\protoc.py:21: UserWarning: pkg_resources is deprecated as an API

tools/grpc_generator/src/grpc_generator/__init__.py

@@ -1,21 +1,105 @@
 """grpc_generator entry points."""
 
-import logging
+import pathlib
 
 import click
 
 from . import generator
 
 
-_logger = logging.getLogger(__name__)
-_logger.addHandler(logging.NullHandler())
+REPO_ROOT = next(
+    (p for p in pathlib.Path(__file__).parents if (p / "third_party").exists()), pathlib.Path(".")
+)
 
 
-@click.command()
+@click.command(epilog=generator.USAGE_EXAMPLE)
+@click.option(
+    "--output-basepath",
+    metavar="PATH",
+    type=click.Path(file_okay=False, writable=True, path_type=pathlib.Path),
+    required=True,
+    help="Emit the generated gRPC files to PATH",
+)
+@click.option(
+    "--output-format",
+    type=click.Choice(choices=[entry.value for entry in generator.OutputFormat]),
+    required=True,
+    help="Generate a Python submodule or subpackage",
+)
+@click.option(
+    "--proto-basepath",
+    metavar="PATH",
+    type=click.Path(file_okay=False, path_type=pathlib.Path),
+    default=REPO_ROOT.joinpath("third_party/ni-apis"),
+    show_default=True,
+    help="Use PATH as the base for --proto-subpath",
+)
+@click.option(
+    "--proto-include-path",
+    metavar="PATH",
+    multiple=True,
+    type=click.Path(file_okay=False, path_type=pathlib.Path),
+    default=[REPO_ROOT.joinpath("third_party/ni-apis")],
+    show_default=True,
+    help="Add PATH to the import search list, can be used more than once",
+)
+@click.option(
+    "--proto-subpath",
+    metavar="PATH",
+    type=click.Path(file_okay=False, path_type=pathlib.Path),
+    required=True,
+    help="Use the proto files under PATH as input",
+)
 @click.help_option()
-def cli() -> None:
-    """Generate gRPC Python stubs from proto files."""
-    generator.handle_cli()
+@click.version_option()
+def cli(
+    proto_basepath: pathlib.Path,
+    proto_subpath: pathlib.Path,
+    proto_include_path: list[pathlib.Path],
+    output_basepath: pathlib.Path,
+    output_format: str,
+) -> None:
+    """Generate gRPC Python stubs from proto files.
+
+    Specifying input and output locations
+
+      This script uses the protobuf files from the folder specified by
+    --proto-basepath and --proto-subpath and emits Python files into the
+    folder specified by --output-basepath:
+
+    \b
+      {proto-basepath}/{proto-subpath}  -->  {output-basepath}/{proto-subpath}
+
+      The script resolves gRPC imports from --proto-basepath by default. Include
+    additional paths by using --proto-include-path for each required folder.
+
+    Specifying output format
+
+      The script supports generating gRPC packages as either subpackages
+    or submodules with --output-format.
+
+      When generating submodules, the script creates Python files with names
+    that match the source protobuf files:
+
+    \b
+      waveform.proto  -->  waveform_pb2.py
+
+      When generating subpackages, the script creates folders with names
+    that match the source protobuf files:
+
+    \b
+      waveform.proto  -->  waveform_pb2/__init__.py
+
+      Clients use the same "import waveform_pb2" syntax.
+
+    """  # noqa: D301 - Use r""" if any backslashes in a docstring
+    generator.handle_cli(
+        proto_basepath=proto_basepath,
+        proto_subpath=proto_subpath,
+        proto_include_paths=proto_include_path,
+        output_basepath=output_basepath,
+        output_format=generator.OutputFormat(output_format),
+    )
 
 
 if __name__ == "__main__":