[WORKFLOW] AI update based on proto changes from commit f0044e0

Author: viambot

src/viam/app/data_client.py

@@ -67,6 +67,7 @@
     TabularDataSourceType,
     TagsByFilterRequest,
     TagsByFilterResponse,
+    UpdateBoundingBoxRequest,
 )
 from viam.proto.app.datapipelines import (
     CreateDataPipelineRequest,
@@ -1131,6 +1132,7 @@ async def add_bounding_box_to_image_by_id(
         y_min_normalized: float,
         x_max_normalized: float,
         y_max_normalized: float,
+        confidence: Optional[float] = None,
     ) -> str:
         """Add a bounding box to an image.
 
@@ -1156,6 +1158,7 @@ async def add_bounding_box_to_image_by_id(
             y_min_normalized (float): Min Y value of the bounding box normalized from 0 to 1.
             x_max_normalized (float): Max X value of the bounding box normalized from 0 to 1.
             y_max_normalized (float): Max Y value of the bounding box normalized from 0 to 1.
+            confidence (Optional[float]): The confidence score for the bounding box.
 
         Raises:
             GRPCError: If the X or Y values are outside of the [0, 1] range.
@@ -1174,6 +1177,7 @@ async def add_bounding_box_to_image_by_id(
                 x_min_normalized=x_min_normalized,
                 y_max_normalized=y_max_normalized,
                 y_min_normalized=y_min_normalized,
+                confidence=confidence,
             )
         else:
             request = AddBoundingBoxToImageByIDRequest(
@@ -1183,6 +1187,7 @@ async def add_bounding_box_to_image_by_id(
                 x_min_normalized=x_min_normalized,
                 y_max_normalized=y_max_normalized,
                 y_min_normalized=y_min_normalized,
+                confidence=confidence,
             )
         response: AddBoundingBoxToImageByIDResponse = await self._data_client.AddBoundingBoxToImageByID(request, metadata=self._metadata)
         return response.bbox_id
@@ -1535,6 +1540,7 @@ async def binary_data_capture_upload(
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
         data_request_times: Optional[Tuple[datetime, datetime]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload binary sensor data.
 
@@ -1573,6 +1579,7 @@ async def binary_data_capture_upload(
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
             data_request_times (Optional[Tuple[datetime.datetime, datetime.datetime]]): Optional tuple containing datetime objects
                 denoting the times this data was requested ``[0]`` by the robot and received ``[1]`` from the appropriate sensor.
+            mime_type (Optional[str]): The MIME type of the file.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1603,6 +1610,7 @@ async def binary_data_capture_upload(
             method_parameters=method_parameters,
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type if mime_type else "",
         )
         if file_extension:
             metadata.file_extension = file_extension if file_extension[0] == "." else f".{file_extension}"
@@ -1619,6 +1627,7 @@ async def tabular_data_capture_upload(
         data_request_times: List[Tuple[datetime, datetime]],
         method_parameters: Optional[Mapping[str, Any]] = None,
         tags: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload tabular sensor data.
 
@@ -1658,6 +1667,7 @@ async def tabular_data_capture_upload(
                 Pass a list of tabular data and timestamps with length ``n > 1`` to upload ``n`` datapoints, all with the same metadata.
             method_parameters (Optional[Mapping[str, Any]]): Optional dictionary of method parameters. No longer in active use.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based data filtering when retrieving data.
+            mime_type (Optional[str]): The MIME type of the file.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1700,6 +1710,7 @@ async def tabular_data_capture_upload(
             type=DataType.DATA_TYPE_TABULAR_SENSOR,
             method_parameters=method_parameters,
             tags=tags,
+            mime_type=mime_type if mime_type else "",
         )
         response = await self._data_capture_upload(metadata=metadata, sensor_contents=sensor_contents)
         return response.file_id
@@ -1721,6 +1732,7 @@ async def streaming_data_capture_upload(
         data_request_times: Optional[Tuple[datetime, datetime]] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Uploads the metadata and contents of streaming binary data.
 
@@ -1753,6 +1765,7 @@ async def streaming_data_capture_upload(
                 denoting the times this data was requested ``[0]`` by the robot and received ``[1]`` from the appropriate sensor.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): The MIME type of the file.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1773,6 +1786,7 @@ async def streaming_data_capture_upload(
             file_extension=file_ext if file_ext[0] == "." else f".{file_ext}",
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type if mime_type else "",
         )
         sensor_metadata = SensorMetadata(
             time_requested=datetime_to_timestamp(data_request_times[0]) if data_request_times else None,
@@ -1802,6 +1816,7 @@ async def file_upload(
         file_extension: Optional[str] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload arbitrary file data.
 
@@ -1832,6 +1847,7 @@ async def file_upload(
                 isn't provided. Files with a ``.jpeg``, ``.jpg``, or ``.png`` extension will be saved to the **Images** tab.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): The MIME type of the file.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1852,6 +1868,7 @@ async def file_upload(
             file_extension=file_extension if file_extension else "",
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type if mime_type else "",
         )
         response: FileUploadResponse = await self._file_upload(metadata=metadata, file_contents=FileData(data=data))
         return response.binary_data_id
@@ -1866,6 +1883,7 @@ async def file_upload_from_path(
         method_parameters: Optional[Mapping[str, Any]] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload arbitrary file data.
 
@@ -1890,6 +1908,7 @@ async def file_upload_from_path(
             method_parameters (Optional[str]): Optional dictionary of the method parameters. No longer in active use.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): The MIME type of the file.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1917,6 +1936,7 @@ async def file_upload_from_path(
             file_extension=file_extension if file_extension else "",
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type if mime_type else "",
         )
         response: FileUploadResponse = await self._file_upload(metadata=metadata, file_contents=FileData(data=data if data else bytes()))
         return response.binary_data_id
@@ -2229,3 +2249,67 @@ def create_filter(
             bbox_labels,
             dataset_id,
         )
+
+    async def update_bounding_box(
+        self,
+        binary_id: Union[BinaryID, str],
+        bbox_id: str,
+        label: Optional[str] = None,
+        x_min_normalized: Optional[float] = None,
+        y_min_normalized: Optional[float] = None,
+        x_max_normalized: Optional[float] = None,
+        y_max_normalized: Optional[float] = None,
+        confidence: Optional[float] = None,
+    ) -> None:
+        """Updates a bounding box in an image.
+
+        ::
+
+            await data_client.update_bounding_box(
+                binary_id="<YOUR-BINARY-DATA-ID>",
+                bbox_id="your-bounding-box-id-to-update",
+                label="new-label",
+                x_min_normalized=0.1,
+                y_min_normalized=0.1,
+                x_max_normalized=0.3,
+                y_max_normalized=0.3,
+                confidence=0.9
+            )
+
+        Args:
+            binary_id (Union[~viam.proto.app.data.BinaryID, str]): The binary data ID or :class:`BinaryID` of the image containing the bounding
+                box to update. *DEPRECATED:* :class:`BinaryID` *is deprecated and will be removed in a future release. Instead, pass binary data IDs as a
+                list of strings.*
+            bbox_id (str): The ID of the bounding box to update.
+            label (Optional[str]): The new label for the bounding box.
+            x_min_normalized (Optional[float]): New min X value of the bounding box normalized from 0 to 1.
+            y_min_normalized (Optional[float]): New min Y value of the bounding box normalized from 0 to 1.
+            x_max_normalized (Optional[float]): New max X value of the bounding box normalized from 0 to 1.
+            y_max_normalized (Optional[float]): New max Y value of the bounding box normalized from 0 to 1.
+            confidence (Optional[float]): The new confidence score for the bounding box.
+
+        Raises:
+            GRPCError: If the X or Y values are outside of the [0, 1] range.
+
+        For more information, see `Data Client API <https://docs.viam.com/dev/reference/apis/data-client/#updateboundingbox>`_.
+        """
+        request = UpdateBoundingBoxRequest(bbox_id=bbox_id)
+        if isinstance(binary_id, str):
+            request.binary_data_id = binary_id
+        else:
+            request.binary_id = binary_id
+
+        if label is not None:
+            request.label = label
+        if x_min_normalized is not None:
+            request.x_min_normalized = x_min_normalized
+        if y_min_normalized is not None:
+            request.y_min_normalized = y_min_normalized
+        if x_max_normalized is not None:
+            request.x_max_normalized = x_max_normalized
+        if y_max_normalized is not None:
+            request.y_max_normalized = y_max_normalized
+        if confidence is not None:
+            request.confidence = confidence
+
+        await self._data_client.UpdateBoundingBox(request, metadata=self._metadata)

src/viam/components/arm/arm.py

@@ -1,10 +1,11 @@
 import abc
-from typing import Any, Dict, Final, Optional, Tuple
+from typing import Any, Dict, Final, Mapping, Optional, Tuple
 
 from viam.resource.types import API, RESOURCE_NAMESPACE_RDK, RESOURCE_TYPE_COMPONENT
 
 from ..component_base import ComponentBase
 from . import JointPositions, KinematicsFileFormat, Pose
+from .mesh import Mesh
 
 
 class Arm(ComponentBase):
@@ -195,7 +196,7 @@ async def is_moving(self) -> bool:
     @abc.abstractmethod
     async def get_kinematics(
         self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs
-    ) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    ) -> Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]:
         """
         Get the kinematics information associated with the arm.
 
@@ -212,11 +213,14 @@ async def get_kinematics(
             # Get the byte contents of the file.
             k_bytes = kinematics[1]
 
+            # Get the map of meshes by URDF filepath.
+            meshes_by_urdf_filepath = kinematics[2]
+
         Returns:
-            Tuple[KinematicsFileFormat.ValueType, bytes]: A tuple containing two values; the first [0] value represents the format of the
+            Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]: A tuple containing three values; the first [0] value represents the format of the
             file, either in URDF format (``KinematicsFileFormat.KINEMATICS_FILE_FORMAT_URDF``) or
             Viam's kinematic parameter format (spatial vector algebra) (``KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA``),
-            and the second [1] value represents the byte contents of the file.
+            the second [1] value represents the byte contents of the file, and the third [2] value is a map of meshes by URDF filepath.
 
         For more information, see `Arm component <https://docs.viam.com/dev/reference/apis/components/arm/#getkinematics>`_.
         """

src/viam/components/arm/client.py

@@ -2,7 +2,7 @@
 
 from grpclib.client import Channel
 
-from viam.proto.common import DoCommandRequest, DoCommandResponse, Geometry, GetKinematicsRequest, GetKinematicsResponse
+from viam.proto.common import DoCommandRequest, DoCommandResponse, Geometry, GetKinematicsRequest, GetKinematicsResponse, Mesh
 from viam.proto.component.arm import (
     ArmServiceStub,
     GetEndPositionRequest,
@@ -113,11 +113,11 @@ async def do_command(
 
     async def get_kinematics(
         self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs
-    ) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    ) -> Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]:
         md = kwargs.get("metadata", self.Metadata()).proto
         request = GetKinematicsRequest(name=self.name, extra=dict_to_struct(extra))
         response: GetKinematicsResponse = await self.client.GetKinematics(request, timeout=timeout, metadata=md)
-        return (response.format, response.kinematics_data)
+        return (response.format, response.kinematics_data, response.meshes_by_urdf_filepath)
 
     async def get_geometries(self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs) -> List[Geometry]:
         md = kwargs.get("metadata", self.Metadata())

src/viam/components/arm/service.py

@@ -7,6 +7,7 @@
     GetGeometriesResponse,
     GetKinematicsRequest,
     GetKinematicsResponse,
+    Mesh,
 )
 from viam.proto.component.arm import (
     GetEndPositionRequest,
@@ -109,8 +110,8 @@ async def GetKinematics(self, stream: Stream[GetKinematicsRequest, GetKinematics
         assert request is not None
         arm = self.get_resource(request.name)
         timeout = stream.deadline.time_remaining() if stream.deadline else None
-        format, kinematics_data = await arm.get_kinematics(extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata)
-        response = GetKinematicsResponse(format=format, kinematics_data=kinematics_data)
+        format, kinematics_data, meshes_by_urdf_filepath = await arm.get_kinematics(extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata)
+        response = GetKinematicsResponse(format=format, kinematics_data=kinematics_data, meshes_by_urdf_filepath=meshes_by_urdf_filepath)
         await stream.send_message(response)
 
     async def GetGeometries(self, stream: Stream[GetGeometriesRequest, GetGeometriesResponse]) -> None:

src/viam/components/audio_in/audio_in.py

@@ -2,8 +2,7 @@
 import sys
 from typing import Final, Optional
 
-from viam.proto.common import GetPropertiesResponse
-from viam.proto.component.audioin import GetAudioResponse
+from viam.proto.common import AudioInfo, GetPropertiesResponse
 from viam.resource.types import API, RESOURCE_NAMESPACE_RDK, RESOURCE_TYPE_COMPONENT
 from viam.streams import Stream
 
@@ -28,35 +27,7 @@ class AudioIn(ComponentBase):
     )
 
     Properties: "TypeAlias" = GetPropertiesResponse
-    AudioResponse: "TypeAlias" = GetAudioResponse
-    AudioStream = Stream[AudioResponse]
-
-    @abc.abstractmethod
-    async def get_audio(
-        self, codec: str, duration_seconds: float, previous_timestamp_ns: int, *, timeout: Optional[float] = None, **kwargs
-    ) -> AudioStream:
-        """
-        Get a stream of audio from the device
-
-        ::
-
-            my_audio_in = AudioIn.from_robot(robot=machine, name="my_audio_in")
-
-            stream = await my_audio_in.get_audio(
-                codec=AudioCodec.PCM16,
-                duration_seconds=10.0,
-                previous_timestamp_ns=0
-            )
-
-        Args:
-            codec (str): The desired codec of the returned audio data
-            duration_seconds (float): duration of the stream. 0 = indefinite stream
-            previous_timestamp_ns (int): starting timestamp in nanoseconds for recording continuity.
-            Set to 0 to begin recording from the current time.
-        Returns:
-            AudioStream: stream of audio chunks.
-        ...
-        """
+    AudioInfo: "TypeAlias" = AudioInfo
 
     @abc.abstractmethod
     async def get_properties(self, *, timeout: Optional[float] = None, **kwargs) -> Properties: