Skip to content

Commit f45935a

Browse files
authored
feat: add client debug logging support for async gRPC (#2291)
1 parent dddf797 commit f45935a

File tree

17 files changed

+867
-203
lines changed

17 files changed

+867
-203
lines changed

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_mixins.py.j2

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
# gRPC handles serialization and deserialization, so we just need
3333
# to pass in the functions for each.
3434
if "delete_operation" not in self._stubs:
35-
self._stubs["delete_operation"] = self.grpc_channel.unary_unary(
35+
self._stubs["delete_operation"] = self._logged_channel.unary_unary(
3636
"/google.longrunning.Operations/DeleteOperation",
3737
request_serializer=operations_pb2.DeleteOperationRequest.SerializeToString,
3838
response_deserializer=None,
@@ -52,7 +52,7 @@
5252
# gRPC handles serialization and deserialization, so we just need
5353
# to pass in the functions for each.
5454
if "cancel_operation" not in self._stubs:
55-
self._stubs["cancel_operation"] = self.grpc_channel.unary_unary(
55+
self._stubs["cancel_operation"] = self._logged_channel.unary_unary(
5656
"/google.longrunning.Operations/CancelOperation",
5757
request_serializer=operations_pb2.CancelOperationRequest.SerializeToString,
5858
response_deserializer=None,
@@ -72,7 +72,7 @@
7272
# gRPC handles serialization and deserialization, so we just need
7373
# to pass in the functions for each.
7474
if "wait_operation" not in self._stubs:
75-
self._stubs["wait_operation"] = self.grpc_channel.unary_unary(
75+
self._stubs["wait_operation"] = self._logged_channel.unary_unary(
7676
"/google.longrunning.Operations/WaitOperation",
7777
request_serializer=operations_pb2.WaitOperationRequest.SerializeToString,
7878
response_deserializer=None,
@@ -92,7 +92,7 @@
9292
# gRPC handles serialization and deserialization, so we just need
9393
# to pass in the functions for each.
9494
if "get_operation" not in self._stubs:
95-
self._stubs["get_operation"] = self.grpc_channel.unary_unary(
95+
self._stubs["get_operation"] = self._logged_channel.unary_unary(
9696
"/google.longrunning.Operations/GetOperation",
9797
request_serializer=operations_pb2.GetOperationRequest.SerializeToString,
9898
response_deserializer=operations_pb2.Operation.FromString,
@@ -112,7 +112,7 @@
112112
# gRPC handles serialization and deserialization, so we just need
113113
# to pass in the functions for each.
114114
if "list_operations" not in self._stubs:
115-
self._stubs["list_operations"] = self.grpc_channel.unary_unary(
115+
self._stubs["list_operations"] = self._logged_channel.unary_unary(
116116
"/google.longrunning.Operations/ListOperations",
117117
request_serializer=operations_pb2.ListOperationsRequest.SerializeToString,
118118
response_deserializer=operations_pb2.ListOperationsResponse.FromString,
@@ -136,7 +136,7 @@
136136
# gRPC handles serialization and deserialization, so we just need
137137
# to pass in the functions for each.
138138
if "list_locations" not in self._stubs:
139-
self._stubs["list_locations"] = self.grpc_channel.unary_unary(
139+
self._stubs["list_locations"] = self._logged_channel.unary_unary(
140140
"/google.cloud.location.Locations/ListLocations",
141141
request_serializer=locations_pb2.ListLocationsRequest.SerializeToString,
142142
response_deserializer=locations_pb2.ListLocationsResponse.FromString,
@@ -156,7 +156,7 @@
156156
# gRPC handles serialization and deserialization, so we just need
157157
# to pass in the functions for each.
158158
if "get_location" not in self._stubs:
159-
self._stubs["get_location"] = self.grpc_channel.unary_unary(
159+
self._stubs["get_location"] = self._logged_channel.unary_unary(
160160
"/google.cloud.location.Locations/GetLocation",
161161
request_serializer=locations_pb2.GetLocationRequest.SerializeToString,
162162
response_deserializer=locations_pb2.Location.FromString,
@@ -188,7 +188,7 @@
188188
# gRPC handles serialization and deserialization, so we just need
189189
# to pass in the functions for each.
190190
if "set_iam_policy" not in self._stubs:
191-
self._stubs["set_iam_policy"] = self.grpc_channel.unary_unary(
191+
self._stubs["set_iam_policy"] = self._logged_channel.unary_unary(
192192
"/google.iam.v1.IAMPolicy/SetIamPolicy",
193193
request_serializer=iam_policy_pb2.SetIamPolicyRequest.SerializeToString,
194194
response_deserializer=policy_pb2.Policy.FromString,
@@ -216,7 +216,7 @@
216216
# gRPC handles serialization and deserialization, so we just need
217217
# to pass in the functions for each.
218218
if "get_iam_policy" not in self._stubs:
219-
self._stubs["get_iam_policy"] = self.grpc_channel.unary_unary(
219+
self._stubs["get_iam_policy"] = self._logged_channel.unary_unary(
220220
"/google.iam.v1.IAMPolicy/GetIamPolicy",
221221
request_serializer=iam_policy_pb2.GetIamPolicyRequest.SerializeToString,
222222
response_deserializer=policy_pb2.Policy.FromString,
@@ -246,7 +246,7 @@
246246
# gRPC handles serialization and deserialization, so we just need
247247
# to pass in the functions for each.
248248
if "test_iam_permissions" not in self._stubs:
249-
self._stubs["test_iam_permissions"] = self.grpc_channel.unary_unary(
249+
self._stubs["test_iam_permissions"] = self._logged_channel.unary_unary(
250250
"/google.iam.v1.IAMPolicy/TestIamPermissions",
251251
request_serializer=iam_policy_pb2.TestIamPermissionsRequest.SerializeToString,
252252
response_deserializer=iam_policy_pb2.TestIamPermissionsResponse.FromString,

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
{% block content %}
44

5+
import json
56
import logging as std_logging
67
import pickle
78
import warnings
@@ -69,7 +70,12 @@ class _LoggingClientInterceptor(grpc.UnaryUnaryClientInterceptor): # pragma: NO
6970
elif isinstance(request, google.protobuf.message.Message):
7071
request_payload = MessageToJson(request)
7172
else:
72-
request_payload = f"{type(result).__name__}: {pickle.dumps(request)}"
73+
request_payload = f"{type(request).__name__}: {pickle.dumps(request)}"
74+
75+
request_metadata = {
76+
key: value.decode("utf-8") if isinstance(value, bytes) else value
77+
for key, value in request_metadata
78+
}
7379
grpc_request = {
7480
"payload": request_payload,
7581
"requestMethod": "grpc",
@@ -90,7 +96,7 @@ class _LoggingClientInterceptor(grpc.UnaryUnaryClientInterceptor): # pragma: NO
9096
if logging_enabled: # pragma: NO COVER
9197
response_metadata = response.trailing_metadata()
9298
# Convert gRPC metadata `<class 'grpc.aio._metadata.Metadata'>` to list of tuples
93-
metadata = dict([(k, v) for k, v in response_metadata]) if response_metadata else None
99+
metadata = dict([(k, str(v)) for k, v in response_metadata]) if response_metadata else None
94100
result = response.result()
95101
if isinstance(result, proto.Message):
96102
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2293): Investigate if we can improve this logic

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2

Lines changed: 92 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
{% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
55

66
import inspect
7+
import json
8+
import pickle
9+
import logging as std_logging
710
import warnings
811
from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union
912

@@ -16,8 +19,11 @@ from google.api_core import operations_v1
1619
{% endif %}
1720
from google.auth import credentials as ga_credentials # type: ignore
1821
from google.auth.transport.grpc import SslCredentials # type: ignore
22+
from google.protobuf.json_format import MessageToJson
23+
import google.protobuf.message
1924

2025
import grpc # type: ignore
26+
import proto # type: ignore
2127
from grpc.experimental import aio # type: ignore
2228

2329
{% filter sort_lines %}
@@ -47,6 +53,81 @@ from google.longrunning import operations_pb2 # type: ignore
4753
from .base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO
4854
from .grpc import {{ service.name }}GrpcTransport
4955

56+
try:
57+
from google.api_core import client_logging # type: ignore
58+
CLIENT_LOGGING_SUPPORTED = True # pragma: NO COVER
59+
except ImportError: # pragma: NO COVER
60+
CLIENT_LOGGING_SUPPORTED = False
61+
62+
_LOGGER = std_logging.getLogger(__name__)
63+
64+
65+
class _LoggingClientAIOInterceptor(grpc.aio.UnaryUnaryClientInterceptor): # pragma: NO COVER
66+
async def intercept_unary_unary(self, continuation, client_call_details, request):
67+
logging_enabled = CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(std_logging.DEBUG)
68+
if logging_enabled: # pragma: NO COVER
69+
request_metadata = client_call_details.metadata
70+
if isinstance(request, proto.Message):
71+
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2293): Investigate if we can improve this logic
72+
or wait for next gen protobuf.
73+
#}
74+
request_payload = type(request).to_json(request)
75+
elif isinstance(request, google.protobuf.message.Message):
76+
request_payload = MessageToJson(request)
77+
else:
78+
request_payload = f"{type(request).__name__}: {pickle.dumps(request)}"
79+
80+
request_metadata = {
81+
key: value.decode("utf-8") if isinstance(value, bytes) else value
82+
for key, value in request_metadata
83+
}
84+
grpc_request = {
85+
"payload": request_payload,
86+
"requestMethod": "grpc",
87+
"metadata": dict(request_metadata),
88+
}
89+
_LOGGER.debug(
90+
f"Sending request for {client_call_details.method}",
91+
extra = {
92+
"serviceName": "{{ service.meta.address.proto }}",
93+
"rpcName": str(client_call_details.method),
94+
"request": grpc_request,
95+
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2275): logging `metadata` seems repetitive and may need to be cleaned up. We're including it within "request" for consistency with REST transport.' #}
96+
"metadata": grpc_request["metadata"],
97+
},
98+
)
99+
response = await continuation(client_call_details, request)
100+
if logging_enabled: # pragma: NO COVER
101+
response_metadata = await response.trailing_metadata()
102+
# Convert gRPC metadata `<class 'grpc.aio._metadata.Metadata'>` to list of tuples
103+
metadata = dict([(k, str(v)) for k, v in response_metadata]) if response_metadata else None
104+
result = await response
105+
if isinstance(result, proto.Message):
106+
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2293): Investigate if we can improve this logic
107+
or wait for next gen protobuf.
108+
#}
109+
response_payload = type(result).to_json(result)
110+
elif isinstance(result, google.protobuf.message.Message):
111+
response_payload = MessageToJson(result)
112+
else:
113+
response_payload = f"{type(result).__name__}: {pickle.dumps(result)}"
114+
grpc_response = {
115+
"payload": response_payload,
116+
"metadata": metadata,
117+
"status": "OK",
118+
}
119+
_LOGGER.debug(
120+
f"Received response to rpc {client_call_details.method}.",
121+
extra = {
122+
"serviceName": "{{ service.meta.address.proto }}",
123+
"rpcName": str(client_call_details.method),
124+
"response": grpc_response,
125+
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2275): logging `metadata` seems repetitive and may need to be cleaned up. We're including it within "request" for consistency with REST transport.' #}
126+
"metadata": grpc_response["metadata"],
127+
},
128+
)
129+
return response
130+
50131

51132
class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
52133
"""gRPC AsyncIO backend transport for {{ service.name }}.
@@ -242,8 +323,11 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
242323
],
243324
)
244325

245-
# Wrap messages. This must be done after self._grpc_channel exists
326+
self._interceptor = _LoggingClientAIOInterceptor()
327+
self._grpc_channel._unary_unary_interceptors.append(self._interceptor)
328+
self._logged_channel = self._grpc_channel
246329
self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters
330+
# Wrap messages. This must be done after self._logged_channel exists
247331
self._prep_wrapped_messages(client_info)
248332

249333
@property
@@ -267,7 +351,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
267351
# Quick check: Only create a new client if we do not already have one.
268352
if self._operations_client is None:
269353
self._operations_client = operations_v1.OperationsAsyncClient(
270-
self.grpc_channel
354+
self._logged_channel
271355
)
272356

273357
# Return the client from cache.
@@ -297,7 +381,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
297381
# gRPC handles serialization and deserialization, so we just need
298382
# to pass in the functions for each.
299383
if '{{ method.transport_safe_name|snake_case }}' not in self._stubs:
300-
self._stubs['{{ method.transport_safe_name|snake_case }}'] = self.grpc_channel.{{ method.grpc_stub_type }}(
384+
self._stubs['{{ method.transport_safe_name|snake_case }}'] = self._logged_channel.{{ method.grpc_stub_type }}(
301385
'/{{ '.'.join(method.meta.address.package) }}.{{ service.name }}/{{ method.name }}',
302386
request_serializer={{ method.input.ident }}.{% if method.input.ident.python_import.module.endswith('_pb2') %}SerializeToString{% else %}serialize{% endif %},
303387
response_deserializer={{ method.output.ident }}.{% if method.output.ident.python_import.module.endswith('_pb2') %}FromString{% else %}deserialize{% endif %},
@@ -325,7 +409,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
325409
# gRPC handles serialization and deserialization, so we just need
326410
# to pass in the functions for each.
327411
if "set_iam_policy" not in self._stubs:
328-
self._stubs["set_iam_policy"] = self.grpc_channel.unary_unary(
412+
self._stubs["set_iam_policy"] = self._logged_channel.unary_unary(
329413
"/google.iam.v1.IAMPolicy/SetIamPolicy",
330414
request_serializer=iam_policy_pb2.SetIamPolicyRequest.SerializeToString,
331415
response_deserializer=policy_pb2.Policy.FromString,
@@ -351,7 +435,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
351435
# gRPC handles serialization and deserialization, so we just need
352436
# to pass in the functions for each.
353437
if "get_iam_policy" not in self._stubs:
354-
self._stubs["get_iam_policy"] = self.grpc_channel.unary_unary(
438+
self._stubs["get_iam_policy"] = self._logged_channel.unary_unary(
355439
"/google.iam.v1.IAMPolicy/GetIamPolicy",
356440
request_serializer=iam_policy_pb2.GetIamPolicyRequest.SerializeToString,
357441
response_deserializer=policy_pb2.Policy.FromString,
@@ -380,7 +464,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
380464
# gRPC handles serialization and deserialization, so we just need
381465
# to pass in the functions for each.
382466
if "test_iam_permissions" not in self._stubs:
383-
self._stubs["test_iam_permissions"] = self.grpc_channel.unary_unary(
467+
self._stubs["test_iam_permissions"] = self._logged_channel.unary_unary(
384468
"/google.iam.v1.IAMPolicy/TestIamPermissions",
385469
request_serializer=iam_policy_pb2.TestIamPermissionsRequest.SerializeToString,
386470
response_deserializer=iam_policy_pb2.TestIamPermissionsResponse.FromString,
@@ -393,7 +477,7 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
393477
{{ shared_macros.wrap_async_method_macro()|indent(4) }}
394478

395479
def close(self):
396-
return self.grpc_channel.close()
480+
return self._logged_channel.close()
397481

398482
@property
399483
def kind(self) -> str:
@@ -405,4 +489,4 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
405489
__all__ = (
406490
'{{ service.name }}GrpcAsyncIOTransport',
407491
)
408-
{% endblock %}
492+
{% endblock %}

tests/integration/goldens/asset/google/cloud/asset_v1/services/asset_service/transports/grpc.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
#
16+
import json
1617
import logging as std_logging
1718
import pickle
1819
import warnings
@@ -54,7 +55,12 @@ def intercept_unary_unary(self, continuation, client_call_details, request):
5455
elif isinstance(request, google.protobuf.message.Message):
5556
request_payload = MessageToJson(request)
5657
else:
57-
request_payload = f"{type(result).__name__}: {pickle.dumps(request)}"
58+
request_payload = f"{type(request).__name__}: {pickle.dumps(request)}"
59+
60+
request_metadata = {
61+
key: value.decode("utf-8") if isinstance(value, bytes) else value
62+
for key, value in request_metadata
63+
}
5864
grpc_request = {
5965
"payload": request_payload,
6066
"requestMethod": "grpc",
@@ -74,7 +80,7 @@ def intercept_unary_unary(self, continuation, client_call_details, request):
7480
if logging_enabled: # pragma: NO COVER
7581
response_metadata = response.trailing_metadata()
7682
# Convert gRPC metadata `<class 'grpc.aio._metadata.Metadata'>` to list of tuples
77-
metadata = dict([(k, v) for k, v in response_metadata]) if response_metadata else None
83+
metadata = dict([(k, str(v)) for k, v in response_metadata]) if response_metadata else None
7884
result = response.result()
7985
if isinstance(result, proto.Message):
8086
response_payload = type(result).to_json(result)
@@ -1019,7 +1025,7 @@ def get_operation(
10191025
# gRPC handles serialization and deserialization, so we just need
10201026
# to pass in the functions for each.
10211027
if "get_operation" not in self._stubs:
1022-
self._stubs["get_operation"] = self.grpc_channel.unary_unary(
1028+
self._stubs["get_operation"] = self._logged_channel.unary_unary(
10231029
"/google.longrunning.Operations/GetOperation",
10241030
request_serializer=operations_pb2.GetOperationRequest.SerializeToString,
10251031
response_deserializer=operations_pb2.Operation.FromString,

0 commit comments

Comments
 (0)