Merge remote-tracking branch 'daniel/master' into pr/wrapper-as-output

Author: boukeversteegh

.gitignore

@@ -4,12 +4,9 @@
 .pytest_cache
 .python-version
 build/
-betterproto/tests/*.bin
-betterproto/tests/*_pb2.py
-betterproto/tests/*.py
-!betterproto/tests/generate.py
-!betterproto/tests/test_*.py
+betterproto/tests/output_*
 **/__pycache__
 dist
 **/*.egg-info
 output
+.idea

Pipfile

@@ -8,6 +8,7 @@ flake8 = "*"
 mypy = "*"
 isort = "*"
 pytest = "*"
+pytest-asyncio = "*"
 rope = "*"
 v = {editable = true,version = "*"}
 

Pipfile.lock

@@ -311,18 +311,34 @@ $ pip install -e .
 
 There are two types of tests:
 
-1. Manually-written tests for some behavior of the library
-2. Proto files and JSON inputs for automated tests
+1. Standard tests
+2. Custom tests
 
-For #2, you can add a new `*.proto` file into the `betterproto/tests` directory along with a sample `*.json` input and it will get automatically picked up.
+#### Standard tests
+
+Adding a standard test case is easy.
+
+- Create a new directory `betterproto/tests/inputs/<name>`
+  - add `<name>.proto`  with a message called `Test`
+  - add `<name>.json` with some test data
+
+It will be picked up automatically when you run the tests.
+
+- See also: [Standard Tests Development Guide](betterproto/tests/README.md) 
+
+#### Custom tests
+
+Custom tests are found in `tests/test_*.py` and are run with pytest.
+
+#### Running
 
 Here's how to run the tests.
 
 ```sh
 # Generate assets from sample .proto files
 $ pipenv run generate
 
-# Run the tests
+# Run all tests
 $ pipenv run test
 ```
 
@@ -340,6 +356,9 @@ $ pipenv run test
 - [x] Refs to nested types
 - [x] Imports in proto files
 - [x] Well-known Google types
+  - [ ] Support as request input
+  - [ ] Support as response output
+    - [ ] Automatically wrap/unwrap responses
 - [x] OneOf support
   - [x] Basic support on the wire
   - [x] Check which was set from the group

README.md

@@ -11,10 +11,12 @@
     Any,
     AsyncGenerator,
     Callable,
+    Collection,
     Dict,
     Generator,
     Iterable,
     List,
+    Mapping,
     Optional,
     SupportsBytes,
     Tuple,
@@ -1000,32 +1002,80 @@ def _get_wrapper(proto_type: str) -> Type:
     }[proto_type]
 
 
+_Value = Union[str, bytes]
+_MetadataLike = Union[Mapping[str, _Value], Collection[Tuple[str, _Value]]]
+
+
 class ServiceStub(ABC):
     """
     Base class for async gRPC service stubs.
     """
 
-    def __init__(self, channel: grpclib.client.Channel) -> None:
+    def __init__(
+        self,
+        channel: grpclib.client.Channel,
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional[grpclib.metadata.Deadline] = None,
+        metadata: Optional[_MetadataLike] = None,
+    ) -> None:
         self.channel = channel
+        self.timeout = timeout
+        self.deadline = deadline
+        self.metadata = metadata
+
+    def __resolve_request_kwargs(
+        self,
+        timeout: Optional[float],
+        deadline: Optional[grpclib.metadata.Deadline],
+        metadata: Optional[_MetadataLike],
+    ):
+        return {
+            "timeout": self.timeout if timeout is None else timeout,
+            "deadline": self.deadline if deadline is None else deadline,
+            "metadata": self.metadata if metadata is None else metadata,
+        }
 
     async def _unary_unary(
-        self, route: str, request: "IProtoMessage", response_type: Type[T]
+        self,
+        route: str,
+        request: "IProtoMessage",
+        response_type: Type[T],
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional[grpclib.metadata.Deadline] = None,
+        metadata: Optional[_MetadataLike] = None,
     ) -> T:
         """Make a unary request and return the response."""
         async with self.channel.request(
-            route, grpclib.const.Cardinality.UNARY_UNARY, type(request), response_type
+            route,
+            grpclib.const.Cardinality.UNARY_UNARY,
+            type(request),
+            response_type,
+            **self.__resolve_request_kwargs(timeout, deadline, metadata),
         ) as stream:
             await stream.send_message(request, end=True)
             response = await stream.recv_message()
             assert response is not None
             return response
 
     async def _unary_stream(
-        self, route: str, request: "IProtoMessage", response_type: Type[T]
+        self,
+        route: str,
+        request: "IProtoMessage",
+        response_type: Type[T],
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional[grpclib.metadata.Deadline] = None,
+        metadata: Optional[_MetadataLike] = None,
     ) -> AsyncGenerator[T, None]:
         """Make a unary request and return the stream response iterator."""
         async with self.channel.request(
-            route, grpclib.const.Cardinality.UNARY_STREAM, type(request), response_type
+            route,
+            grpclib.const.Cardinality.UNARY_STREAM,
+            type(request),
+            response_type,
+            **self.__resolve_request_kwargs(timeout, deadline, metadata),
         ) as stream:
             await stream.send_message(request, end=True)
             async for message in stream:

betterproto/__init__.py

@@ -0,0 +1,2 @@
+@SET plugin_dir=%~dp0
+@python %plugin_dir%/plugin.py %*

betterproto/plugin.bat

@@ -138,7 +138,7 @@ def get_py_zero(type_num: int) -> str:
 
 
 def traverse(proto_file):
-    def _traverse(path, items, prefix = ''):
+    def _traverse(path, items, prefix=""):
         for i, item in enumerate(items):
             # Adjust the name since we flatten the heirarchy.
             item.name = next_prefix = prefix + item.name

betterproto/plugin.py

@@ -0,0 +1,75 @@
+# Standard Tests Development Guide
+
+Standard test cases are found in [betterproto/tests/inputs](inputs), where each subdirectory represents a testcase, that is verified in isolation.
+
+```
+inputs/
+   bool/
+   double/
+   int32/
+   ...
+```
+
+## Test case directory structure
+
+Each testcase has a `<name>.proto` file with a message called `Test`, a matching `.json` file and optionally a custom test file called `test_*.py`.
+
+```bash
+bool/
+  bool.proto
+  bool.json
+  test_bool.py  # optional
+```
+
+### proto
+
+`<name>.proto` &mdash; *The protobuf message to test*
+
+```protobuf
+syntax = "proto3";
+
+message Test {
+    bool value = 1;
+}
+```
+
+You can add multiple `.proto` files to the test case, as long as one file matches the directory name. 
+
+### json
+
+`<name>.json` &mdash; *Test-data to validate the message with*
+
+```json
+{
+  "value": true
+}
+```
+
+### pytest
+
+`test_<name>.py` &mdash; *Custom test to validate specific aspects of the generated class*
+
+```python
+from betterproto.tests.output_betterproto.bool.bool import Test
+
+def test_value():
+    message = Test()
+    assert not message.value, "Boolean is False by default"
+```
+
+## Standard tests
+
+The following tests are automatically executed for all cases:
+
+- [x] Can the generated python code imported?
+- [x] Can the generated message class be instantiated?
+- [x] Is the generated code compatible with the Google's `grpc_tools.protoc` implementation?
+
+## Running the tests
+
+- `pipenv run generate`
+  This generates
+  - `betterproto/tests/output_betterproto` &mdash; *the plugin generated python classes*
+  - `betterproto/tests/output_reference` &mdash; *reference implementation classes*
+- `pipenv run test`
+

betterproto/tests/README.md

@@ -1,84 +1,74 @@
 #!/usr/bin/env python
 import os
+import sys
+from typing import Set
+
+from betterproto.tests.util import (
+    get_directories,
+    inputs_path,
+    output_path_betterproto,
+    output_path_reference,
+    protoc_plugin,
+    protoc_reference,
+)
 
 # Force pure-python implementation instead of C++, otherwise imports
 # break things because we can't properly reset the symbol database.
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 
-import importlib
-import json
-import subprocess
-import sys
-from typing import Generator, Tuple
-
-from google.protobuf import symbol_database
-from google.protobuf.descriptor_pool import DescriptorPool
-from google.protobuf.json_format import MessageToJson, Parse
-
-
-root = os.path.dirname(os.path.realpath(__file__))
 
+def generate(whitelist: Set[str]):
+    path_whitelist = {os.path.realpath(e) for e in whitelist if os.path.exists(e)}
+    name_whitelist = {e for e in whitelist if not os.path.exists(e)}
 
-def get_files(end: str) -> Generator[str, None, None]:
-    for r, dirs, files in os.walk(root):
-        for filename in [f for f in files if f.endswith(end)]:
-            yield os.path.join(r, filename)
+    test_case_names = set(get_directories(inputs_path))
 
+    for test_case_name in sorted(test_case_names):
+        test_case_path = os.path.realpath(os.path.join(inputs_path, test_case_name))
 
-def get_base(filename: str) -> str:
-    return os.path.splitext(os.path.basename(filename))[0]
+        if (
+            whitelist
+            and test_case_path not in path_whitelist
+            and test_case_name not in name_whitelist
+        ):
+            continue
 
+        case_output_dir_reference = os.path.join(output_path_reference, test_case_name)
+        case_output_dir_betterproto = os.path.join(
+            output_path_betterproto, test_case_name
+        )
 
-def ensure_ext(filename: str, ext: str) -> str:
-    if not filename.endswith(ext):
-        return filename + ext
-    return filename
+        print(f"Generating output for {test_case_name}")
+        os.makedirs(case_output_dir_reference, exist_ok=True)
+        os.makedirs(case_output_dir_betterproto, exist_ok=True)
 
+        protoc_reference(test_case_path, case_output_dir_reference)
+        protoc_plugin(test_case_path, case_output_dir_betterproto)
 
-if __name__ == "__main__":
-    os.chdir(root)
-
-    if len(sys.argv) > 1:
-        proto_files = [ensure_ext(f, ".proto") for f in sys.argv[1:]]
-        bases = {get_base(f) for f in proto_files}
-        json_files = [
-            f for f in get_files(".json") if get_base(f).split("-")[0] in bases
-        ]
-    else:
-        proto_files = get_files(".proto")
-        json_files = get_files(".json")
-
-    for filename in proto_files:
-        print(f"Generating code for {os.path.basename(filename)}")
-        subprocess.run(
-            f"protoc --python_out=. {os.path.basename(filename)}", shell=True
-        )
-        subprocess.run(
-            f"protoc --plugin=protoc-gen-custom=../plugin.py --custom_out=. {os.path.basename(filename)}",
-            shell=True,
-        )
 
-    for filename in json_files:
-        # Reset the internal symbol database so we can import the `Test` message
-        # multiple times. Ugh.
-        sym = symbol_database.Default()
-        sym.pool = DescriptorPool()
+HELP = "\n".join(
+    [
+        "Usage: python generate.py",
+        "       python generate.py [DIRECTORIES or NAMES]",
+        "Generate python classes for standard tests.",
+        "",
+        "DIRECTORIES    One or more relative or absolute directories of test-cases to generate classes for.",
+        "               python generate.py inputs/bool inputs/double inputs/enum",
+        "",
+        "NAMES          One or more test-case names to generate classes for.",
+        "               python generate.py bool double enums",
+    ]
+)
 
-        parts = get_base(filename).split("-")
-        out = filename.replace(".json", ".bin")
-        print(f"Using {parts[0]}_pb2 to generate {os.path.basename(out)}")
 
-        imported = importlib.import_module(f"{parts[0]}_pb2")
-        input_json = open(filename).read()
-        parsed = Parse(input_json, imported.Test())
-        serialized = parsed.SerializeToString()
-        preserve = "casing" not in filename
-        serialized_json = MessageToJson(parsed, preserving_proto_field_name=preserve)
+def main():
+    if set(sys.argv).intersection({"-h", "--help"}):
+        print(HELP)
+        return
+    whitelist = set(sys.argv[1:])
 
-        s_loaded = json.loads(serialized_json)
-        in_loaded = json.loads(input_json)
+    generate(whitelist)
 
-        if s_loaded != in_loaded:
-            raise AssertionError("Expected JSON to be equal:", s_loaded, in_loaded)
 
-        open(out, "wb").write(serialized)
+if __name__ == "__main__":
+    main()