Revert "Migrate the client code to use betterproto"

This partially reverts commits:

* Migrate the client code to use betterproto
  (e159417)
* Fix component tests to use betterproto
  (b5b52d5)
* Fix client test to use betterproto
  (b3efe71)

It preserves the changes to tests to use mocks instead of a fake but
full gRPC server, and adapts the new code to use the grpcio library and
the generated code from the traditional google protobuf compiler.

Since the `client-base` library currently being used doesn't support
grpcio, we also need to update that library too, but we do that in
a separate commit.

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

src/frequenz/client/microgrid/_client.py

@@ -5,19 +5,34 @@
 
 import asyncio
 import logging
-from collections.abc import Callable, Iterable, Set
-from typing import Any, TypeVar
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Set
+from typing import Any, TypeVar, cast
+
+import grpc.aio
+
+# pylint: disable=no-name-in-module
+from frequenz.api.common.components_pb2 import ComponentCategory as PbComponentCategory
+from frequenz.api.common.metrics_pb2 import Bounds as PbBounds
+from frequenz.api.microgrid.microgrid_pb2 import ComponentData as PbComponentData
+from frequenz.api.microgrid.microgrid_pb2 import ComponentFilter as PbComponentFilter
+from frequenz.api.microgrid.microgrid_pb2 import ComponentIdParam as PbComponentIdParam
+from frequenz.api.microgrid.microgrid_pb2 import ComponentList as PbComponentList
+from frequenz.api.microgrid.microgrid_pb2 import ConnectionFilter as PbConnectionFilter
+from frequenz.api.microgrid.microgrid_pb2 import ConnectionList as PbConnectionList
+from frequenz.api.microgrid.microgrid_pb2 import (
+    MicrogridMetadata as PbMicrogridMetadata,
+)
+from frequenz.api.microgrid.microgrid_pb2 import SetBoundsParam as PbSetBoundsParam
+from frequenz.api.microgrid.microgrid_pb2 import (
+    SetPowerActiveParam as PbSetPowerActiveParam,
+)
+from frequenz.api.microgrid.microgrid_pb2_grpc import MicrogridStub
 
-import grpclib
-import grpclib.client
-from betterproto.lib.google import protobuf as pb_google
+# pylint: enable=no-name-in-module
 from frequenz.channels import Receiver
 from frequenz.client.base import channel, retry, streaming
-from frequenz.microgrid.betterproto.frequenz.api import microgrid as pb_microgrid
-from frequenz.microgrid.betterproto.frequenz.api.common import (
-    components as pb_components,
-)
-from frequenz.microgrid.betterproto.frequenz.api.common import metrics as pb_metrics
+from google.protobuf.empty_pb2 import Empty  # pylint: disable=no-name-in-module
+from google.protobuf.timestamp_pb2 import Timestamp  # pylint: disable=no-name-in-module
 
 from ._component import (
     Component,
@@ -72,7 +87,7 @@ def __init__(
         self._server_url = server_url
         """The location of the microgrid API server as a URL."""
 
-        self.api = pb_microgrid.MicrogridStub(channel.parse_grpc_uri(server_url))
+        self.api = MicrogridStub(channel.parse_grpc_uri(server_url))
         """The gRPC stub for the microgrid API."""
 
         self._broadcasters: dict[int, streaming.GrpcStreamBroadcaster[Any, Any]] = {}
@@ -95,20 +110,24 @@ async def components(self) -> Iterable[Component]:
                 [GrpcError][frequenz.client.microgrid.GrpcError].
         """
         try:
-            component_list = await self.api.list_components(
-                pb_microgrid.ComponentFilter(),
-                timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+            # grpc.aio is missing types and mypy thinks this is not awaitable,
+            # but it is
+            component_list = await cast(
+                Awaitable[PbComponentList],
+                self.api.ListComponents(
+                    PbComponentFilter(),
+                    timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+                ),
             )
-        except grpclib.GRPCError as grpc_error:
+        except grpc.aio.AioRpcError as grpc_error:
             raise ClientError.from_grpc_error(
                 server_url=self._server_url,
-                operation="list_components",
+                operation="ListComponents",
                 grpc_error=grpc_error,
             ) from grpc_error
 
         components_only = filter(
-            lambda c: c.category
-            is not pb_components.ComponentCategory.COMPONENT_CATEGORY_SENSOR,
+            lambda c: c.category is not PbComponentCategory.COMPONENT_CATEGORY_SENSOR,
             component_list.components,
         )
         result: Iterable[Component] = map(
@@ -132,13 +151,16 @@ async def metadata(self) -> Metadata:
         Returns:
             the microgrid metadata.
         """
-        microgrid_metadata: pb_microgrid.MicrogridMetadata | None = None
+        microgrid_metadata: PbMicrogridMetadata | None = None
         try:
-            microgrid_metadata = await self.api.get_microgrid_metadata(
-                pb_google.Empty(),
-                timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+            microgrid_metadata = await cast(
+                Awaitable[PbMicrogridMetadata],
+                self.api.GetMicrogridMetadata(
+                    Empty(),
+                    timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+                ),
             )
-        except grpclib.GRPCError:
+        except grpc.aio.AioRpcError:
             _logger.exception("The microgrid metadata is not available.")
 
         if not microgrid_metadata:
@@ -174,21 +196,24 @@ async def connections(
                 most likely a subclass of
                 [GrpcError][frequenz.client.microgrid.GrpcError].
         """
-        connection_filter = pb_microgrid.ConnectionFilter(
-            starts=list(starts), ends=list(ends)
-        )
+        connection_filter = PbConnectionFilter(starts=starts, ends=ends)
         try:
             valid_components, all_connections = await asyncio.gather(
                 self.components(),
-                self.api.list_connections(
-                    connection_filter,
-                    timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+                # grpc.aio is missing types and mypy thinks this is not
+                # awaitable, but it is
+                cast(
+                    Awaitable[PbConnectionList],
+                    self.api.ListConnections(
+                        connection_filter,
+                        timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
+                    ),
                 ),
             )
-        except grpclib.GRPCError as grpc_error:
+        except grpc.aio.AioRpcError as grpc_error:
             raise ClientError.from_grpc_error(
                 server_url=self._server_url,
-                operation="list_connections",
+                operation="ListConnections",
                 grpc_error=grpc_error,
             ) from grpc_error
         # Filter out the components filtered in `components` method.
@@ -212,7 +237,7 @@ async def _new_component_data_receiver(
         *,
         component_id: int,
         expected_category: ComponentCategory,
-        transform: Callable[[pb_microgrid.ComponentData], _ComponentDataT],
+        transform: Callable[[PbComponentData], _ComponentDataT],
         maxsize: int,
     ) -> Receiver[_ComponentDataT]:
         """Return a new broadcaster receiver for a given `component_id`.
@@ -239,8 +264,13 @@ async def _new_component_data_receiver(
         if broadcaster is None:
             broadcaster = streaming.GrpcStreamBroadcaster(
                 f"raw-component-data-{component_id}",
-                lambda: self.api.stream_component_data(
-                    pb_microgrid.ComponentIdParam(id=component_id)
+                # We need to cast here because grpc says StreamComponentData is
+                # a grpc.CallIterator[PbComponentData] which is not an AsyncIterator,
+                # but it is a grpc.aio.UnaryStreamCall[..., PbComponentData], which it
+                # is.
+                lambda: cast(
+                    AsyncIterator[PbComponentData],
+                    self.api.StreamComponentData(PbComponentIdParam(id=component_id)),
                 ),
                 transform,
                 retry_strategy=self._retry_strategy,
@@ -394,16 +424,17 @@ async def set_power(self, component_id: int, power_w: float) -> None:
                 [GrpcError][frequenz.client.microgrid.GrpcError].
         """
         try:
-            await self.api.set_power_active(
-                pb_microgrid.SetPowerActiveParam(
-                    component_id=component_id, power=power_w
+            await cast(
+                Awaitable[Empty],
+                self.api.SetPowerActive(
+                    PbSetPowerActiveParam(component_id=component_id, power=power_w),
+                    timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
                 ),
-                timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
             )
-        except grpclib.GRPCError as grpc_error:
+        except grpc.aio.AioRpcError as grpc_error:
             raise ClientError.from_grpc_error(
                 server_url=self._server_url,
-                operation="set_power_active",
+                operation="SetPowerActive",
                 grpc_error=grpc_error,
             ) from grpc_error
 
@@ -413,7 +444,7 @@ async def set_bounds(
         lower: float,
         upper: float,
     ) -> None:
-        """Send `SetBoundsParam`s received from a channel to the Microgrid service.
+        """Send `PbSetBoundsParam`s received from a channel to the Microgrid service.
 
         Args:
             component_id: ID of the component to set bounds for.
@@ -432,21 +463,22 @@ async def set_bounds(
         if lower > 0:
             raise ValueError(f"Lower bound {lower} must be less than or equal to 0.")
 
-        target_metric = (
-            pb_microgrid.SetBoundsParamTargetMetric.TARGET_METRIC_POWER_ACTIVE
-        )
+        target_metric = PbSetBoundsParam.TargetMetric.TARGET_METRIC_POWER_ACTIVE
         try:
-            await self.api.add_inclusion_bounds(
-                pb_microgrid.SetBoundsParam(
-                    component_id=component_id,
-                    target_metric=target_metric,
-                    bounds=pb_metrics.Bounds(lower=lower, upper=upper),
+            await cast(
+                Awaitable[Timestamp],
+                self.api.AddInclusionBounds(
+                    PbSetBoundsParam(
+                        component_id=component_id,
+                        target_metric=target_metric,
+                        bounds=PbBounds(lower=lower, upper=upper),
+                    ),
+                    timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
                 ),
-                timeout=int(DEFAULT_GRPC_CALL_TIMEOUT),
             )
-        except grpclib.GRPCError as grpc_error:
+        except grpc.aio.AioRpcError as grpc_error:
             raise ClientError.from_grpc_error(
                 server_url=self._server_url,
-                operation="add_inclusion_bounds",
+                operation="AddInclusionBounds",
                 grpc_error=grpc_error,
             ) from grpc_error

src/frequenz/client/microgrid/_component.py

@@ -6,8 +6,13 @@
 from dataclasses import dataclass
 from enum import Enum
 
-from frequenz.microgrid.betterproto.frequenz.api.common import components
-from frequenz.microgrid.betterproto.frequenz.api.microgrid import grid, inverter
+# pylint: disable=no-name-in-module
+from frequenz.api.common.components_pb2 import ComponentCategory as PbComponentCategory
+from frequenz.api.microgrid.grid_pb2 import Metadata as PbGridMetadata
+from frequenz.api.microgrid.inverter_pb2 import Metadata as PbInverterMetadata
+from frequenz.api.microgrid.inverter_pb2 import Type as PbInverterType
+
+# pylint: enable=no-name-in-module
 
 
 class ComponentType(Enum):
@@ -17,22 +22,22 @@ class ComponentType(Enum):
 class InverterType(ComponentType):
     """Enum representing inverter types."""
 
-    NONE = inverter.Type.TYPE_UNSPECIFIED
+    NONE = PbInverterType.TYPE_UNSPECIFIED
     """Unspecified inverter type."""
 
-    BATTERY = inverter.Type.TYPE_BATTERY
+    BATTERY = PbInverterType.TYPE_BATTERY
     """Battery inverter."""
 
-    SOLAR = inverter.Type.TYPE_SOLAR
+    SOLAR = PbInverterType.TYPE_SOLAR
     """Solar inverter."""
 
-    HYBRID = inverter.Type.TYPE_HYBRID
+    HYBRID = PbInverterType.TYPE_HYBRID
     """Hybrid inverter."""
 
 
 def component_type_from_protobuf(
-    component_category: components.ComponentCategory,
-    component_metadata: inverter.Metadata,
+    component_category: PbComponentCategory.ValueType,
+    component_metadata: PbInverterMetadata,
 ) -> ComponentType | None:
     """Convert a protobuf InverterType message to Component enum.
 
@@ -48,7 +53,7 @@ def component_type_from_protobuf(
     # ComponentType values in the protobuf definition are not unique across categories
     # as of v0.11.0, so we need to check the component category first, before doing any
     # component type checks.
-    if component_category == components.ComponentCategory.COMPONENT_CATEGORY_INVERTER:
+    if component_category == PbComponentCategory.COMPONENT_CATEGORY_INVERTER:
         if not any(int(t.value) == int(component_metadata.type) for t in InverterType):
             return None
 
@@ -60,30 +65,30 @@ def component_type_from_protobuf(
 class ComponentCategory(Enum):
     """Possible types of microgrid component."""
 
-    NONE = components.ComponentCategory.COMPONENT_CATEGORY_UNSPECIFIED
+    NONE = PbComponentCategory.COMPONENT_CATEGORY_UNSPECIFIED
     """Unspecified component category."""
 
-    GRID = components.ComponentCategory.COMPONENT_CATEGORY_GRID
+    GRID = PbComponentCategory.COMPONENT_CATEGORY_GRID
     """Grid component."""
 
-    METER = components.ComponentCategory.COMPONENT_CATEGORY_METER
+    METER = PbComponentCategory.COMPONENT_CATEGORY_METER
     """Meter component."""
 
-    INVERTER = components.ComponentCategory.COMPONENT_CATEGORY_INVERTER
+    INVERTER = PbComponentCategory.COMPONENT_CATEGORY_INVERTER
     """Inverter component."""
 
-    BATTERY = components.ComponentCategory.COMPONENT_CATEGORY_BATTERY
+    BATTERY = PbComponentCategory.COMPONENT_CATEGORY_BATTERY
     """Battery component."""
 
-    EV_CHARGER = components.ComponentCategory.COMPONENT_CATEGORY_EV_CHARGER
+    EV_CHARGER = PbComponentCategory.COMPONENT_CATEGORY_EV_CHARGER
     """EV charger component."""
 
-    CHP = components.ComponentCategory.COMPONENT_CATEGORY_CHP
+    CHP = PbComponentCategory.COMPONENT_CATEGORY_CHP
     """CHP component."""
 
 
 def component_category_from_protobuf(
-    component_category: components.ComponentCategory,
+    component_category: PbComponentCategory.ValueType,
 ) -> ComponentCategory:
     """Convert a protobuf ComponentCategory message to ComponentCategory enum.
 
@@ -100,7 +105,7 @@ def component_category_from_protobuf(
             a valid component category as it does not form part of the
             microgrid itself)
     """
-    if component_category == components.ComponentCategory.COMPONENT_CATEGORY_SENSOR:
+    if component_category == PbComponentCategory.COMPONENT_CATEGORY_SENSOR:
         raise ValueError("Cannot create a component from a sensor!")
 
     if not any(t.value == component_category for t in ComponentCategory):
@@ -131,8 +136,8 @@ class GridMetadata(ComponentMetadata):
 
 
 def component_metadata_from_protobuf(
-    component_category: components.ComponentCategory,
-    component_metadata: grid.Metadata,
+    component_category: PbComponentCategory.ValueType,
+    component_metadata: PbGridMetadata,
 ) -> GridMetadata | None:
     """Convert a protobuf GridMetadata message to GridMetadata class.
 
@@ -145,7 +150,7 @@ def component_metadata_from_protobuf(
     Returns:
         GridMetadata instance corresponding to the protobuf message.
     """
-    if component_category == components.ComponentCategory.COMPONENT_CATEGORY_GRID:
+    if component_category == PbComponentCategory.COMPONENT_CATEGORY_GRID:
         max_current = component_metadata.rated_fuse_current
         fuse = Fuse(max_current)
         return GridMetadata(fuse)