Add frame by frame compression and test this compression - Nikita Ostapenko

python/client/GrpcClient.py

@@ -0,0 +1,76 @@
+import grpc
+from typing import Awaitable, Iterable
+
+from cakelab.arflow_grpc.v1.ar_frame_pb2 import ARFrame 
+
+from cakelab.arflow_grpc.v1.arflow_service_pb2_grpc import ARFlowServiceStub
+from cakelab.arflow_grpc.v1.session_pb2 import SessionUuid, SessionMetadata
+from cakelab.arflow_grpc.v1.device_pb2 import Device
+
+from cakelab.arflow_grpc.v1.create_session_request_pb2 import CreateSessionRequest
+from cakelab.arflow_grpc.v1.create_session_response_pb2 import CreateSessionResponse
+from cakelab.arflow_grpc.v1.delete_session_request_pb2 import DeleteSessionRequest
+from cakelab.arflow_grpc.v1.delete_session_response_pb2 import DeleteSessionResponse
+from cakelab.arflow_grpc.v1.get_session_request_pb2 import GetSessionRequest
+from cakelab.arflow_grpc.v1.get_session_response_pb2 import GetSessionResponse
+from cakelab.arflow_grpc.v1.join_session_request_pb2 import JoinSessionRequest
+from cakelab.arflow_grpc.v1.join_session_response_pb2 import JoinSessionResponse
+from cakelab.arflow_grpc.v1.list_sessions_request_pb2 import ListSessionsRequest
+from cakelab.arflow_grpc.v1.list_sessions_response_pb2 import ListSessionsResponse
+from cakelab.arflow_grpc.v1.leave_session_request_pb2 import LeaveSessionRequest
+from cakelab.arflow_grpc.v1.leave_session_response_pb2 import LeaveSessionResponse
+from cakelab.arflow_grpc.v1.save_ar_frames_request_pb2 import SaveARFramesRequest
+from cakelab.arflow_grpc.v1.save_ar_frames_response_pb2 import SaveARFramesResponse
+
+class GrpcClient:
+    def __init__(self, url):
+        self.channel = grpc.insecure_channel(url)
+    async def create_session_async(self, name: str, device: Device, save_path: str = "") -> CreateSessionResponse:
+        request = CreateSessionRequest(
+            session_metadata=SessionMetadata(name=name, save_path=save_path),
+            device=device
+        )
+        response: Awaitable[CreateSessionResponse] = ARFlowServiceStub(self.channel).CreateSession(request)
+        return response
+    async def delete_session_async(self, session_id: str) -> DeleteSessionResponse:
+        request = DeleteSessionRequest(
+            session_id=SessionUuid(value = session_id)
+        )
+        response: Awaitable[DeleteSessionResponse] = ARFlowServiceStub(self.channel).DeleteSession(request)
+        return response
+    async def get_session_async(self, session_id: str) -> GetSessionResponse:
+        request = GetSessionRequest(
+            session_id=SessionUuid(value=session_id)
+        )
+        response: Awaitable[GetSessionResponse] = ARFlowServiceStub(self.channel).GetSession(request)
+        return response
+    async def join_session_async(self, session_id: str, device: Device) -> JoinSessionResponse:
+        request = JoinSessionRequest(
+            session_id=SessionUuid(value=session_id),
+            device=device
+        )
+        response: Awaitable[JoinSessionResponse] = ARFlowServiceStub(self.channel).JoinSession(request)
+        return response
+    async def list_sessions_async(self) -> ListSessionsResponse:
+        request = ListSessionsRequest()
+        response: Awaitable[ListSessionsResponse] = ARFlowServiceStub(self.channel).ListSessions(request)
+        return response
+    async def leave_session_async(self, session_id: str, device: Device) -> LeaveSessionResponse:
+        request = LeaveSessionRequest(
+            session_id=SessionUuid(value=session_id),
+            device=device
+        )
+        response: Awaitable[LeaveSessionResponse] = ARFlowServiceStub(self.channel).LeaveSession(request)
+        return response
+    async def save_ar_frames_async(self, session_id: str, ar_frames: Iterable[ARFrame], device: Device) -> SaveARFramesResponse:
+        request = SaveARFramesRequest(
+            session_id=SessionUuid(value=session_id),
+            frames=ar_frames,
+            device=device
+        )
+        response: Awaitable[SaveARFramesResponse] = ARFlowServiceStub(self.channel).SaveARFrames(request)
+        return response
+
+    def close(self):
+        self.channel.close()
+

python/client/clients/cli.py

@@ -0,0 +1,136 @@
+
+
+from GrpcClient import GrpcClient
+from util.GetDeviceInfo import GetDeviceInfo
+from util.SessionRunner import SessionRunner
+
+from cakelab.arflow_grpc.v1.device_pb2 import Device
+from cakelab.arflow_grpc.v1.session_pb2 import Session
+from cakelab.arflow_grpc.v1.ar_frame_pb2 import ARFrame
+
+from cakelab.arflow_grpc.v1.create_session_response_pb2 import CreateSessionResponse
+from cakelab.arflow_grpc.v1.list_sessions_response_pb2 import ListSessionsResponse
+
+import asyncio
+from threading import Thread
+from threading import Event
+from time import sleep
+
+class CLIClient:
+    session: Session | None = None
+    running: Thread | None = None
+    stop_event: Event | None = None
+    def __init__(self):
+        host = input("Enter hostname: ")
+        port = input("Enter port: ")
+        self.client = GrpcClient(f"{host}:{port}")
+        asyncio.run(self.__manage_sessions())
+
+    async def __manage_sessions(self):
+        while True:
+            print("Available Sessions:")
+            session_list: list[Session] = list((await self.client.list_sessions_async()).sessions)
+            for session in session_list:
+                print(f"   Name: {session.metadata.name}, # of Devices: {len(session.devices)}")
+            print("Options:")
+            print("1. Create and Join  Session")
+            print("2. Join Session")
+            print("3. Delete Session")
+            print("4. Refresh")
+            print("5. Exit")
+            choice: str = input("Choose an option: ")
+            match choice:
+                case "1":
+                    name: str = input("Enter session name: ")
+                    if any(session.metadata.name == name for session in session_list):
+                        name = input(f"Session with name '{name}' already exists. Please choose a different name:")
+                    device: Device = GetDeviceInfo.get_device_info()
+                    try:
+                        self.session = (await self.client.create_session_async(name, device)).session
+                        print("Created Session")
+                        await self.__join_session()
+                    except Exception as e:
+                        print(f"Failed to create session: {e}")
+                case "2":
+                    name: str = input("Enter session name to join: ")
+                    target_session: Session | None = next((session for session in session_list if session.metadata.name == name), None)
+                    if target_session is None:
+                        print(f"No session found with name '{name}'")
+                        continue
+                    try:
+                        self.session = (await self.client.join_session_async(target_session.id.value, GetDeviceInfo.get_device_info())).session
+                        print("Joined Session")
+                        await self.__join_session()
+                    except Exception as e:
+                        print(f"Failed to join session: {e}")
+                case "3":
+                    name: str = input("Enter session name to delete: ")
+                    target_session: Session | None = next((session for session in session_list if session.metadata.name == name), None)
+                    if target_session is None:
+                        print(f"No session found with name '{name}'")
+                        continue
+                    try:
+                        await self.client.delete_session_async(target_session.id.value)
+                        print(f"Session '{name}' deleted successfully.")
+                    except Exception as e:
+                        print(f"Failed to delete session: {e}")
+                case "4":
+                    continue
+                case "5":
+                    return
+                case _:
+                    print("Invalid Option")
+
+    async def __join_session(self):
+        if self.session is None: return
+        runner = SessionRunner(self.session, GetDeviceInfo.get_device_info(), self.__on_frame)
+        print("Currently only able to record camera frames")
+        while True:
+            print("Options:")
+            if self.running:
+                print("1. Stop Recording")
+            else:
+                print("1. Start Recording")
+            print("2. Leave Session")
+            choice: str = input("Choose an option: ")
+            match choice:
+                case "1":
+                    if self.running:
+                        self.stop_event.set()
+                        self.running.join()
+                        self.running = None
+                        self.stop_event = None
+                        print("Stopping Recording")
+                    else:
+                        self.stop_event = Event()
+                        self.running = Thread(target=self.__record_frame, args=(runner,))
+                        self.running.start()
+                        print("Starting Recording")
+                case "2":
+                    await self.client.leave_session_async(self.session.id.value, GetDeviceInfo.get_device_info())
+                    print("Leaving Session")
+                    return
+                case _:
+                    print("Invalid Option")
+
+    def __record_frame(self, runner: SessionRunner):
+        while not self.stop_event.is_set():
+            asyncio.run(runner.gather_camera_frame_async())
+            sleep(0.25)
+
+
+    async def __on_frame(self, session: Session, frame: ARFrame , device: Device):
+        if self.session is None:
+            return
+        await self.client.save_ar_frames_async(
+            session_id=self.session.id.value,
+            ar_frames=[frame],
+            device=GetDeviceInfo.get_device_info()
+        )
+
+def main():
+    CLIClient()
+
+
+if __name__ == "__main__":
+    main()

python/client/util/GetDeviceInfo.py

@@ -0,0 +1,16 @@
+from cakelab.arflow_grpc.v1.device_pb2 import Device
+import platform
+import uuid
+
+class GetDeviceInfo:
+    @staticmethod
+    def get_device_info() -> Device:
+        name = platform.node()
+        #not sure what model is, im just gonna leave it as system name combined with version, change this later
+        model = platform.system() + platform.version()
+        return Device(
+            name=name,
+            model=model,
+            type=3, #for now only functions on desktops
+            uid= str(uuid.uuid3(uuid.NAMESPACE_DNS, name + model))
+        )

python/client/util/SessionRunner.py

@@ -0,0 +1,65 @@
+from cakelab.arflow_grpc.v1.session_pb2 import Session
+from cakelab.arflow_grpc.v1.device_pb2 import Device
+
+from cakelab.arflow_grpc.v1.ar_frame_pb2 import ARFrame
+from cakelab.arflow_grpc.v1.color_frame_pb2 import ColorFrame
+from cakelab.arflow_grpc.v1.xr_cpu_image_pb2 import XRCpuImage
+from cakelab.arflow_grpc.v1.vector2_int_pb2 import Vector2Int
+from google.protobuf.timestamp_pb2 import Timestamp
+import cv2
+import time
+from typing import Callable, Coroutine, Any
+
+class SessionRunner:
+    camera : cv2.VideoCapture | None = None
+    session: Session | None = None
+    device: Device | None = None
+    onARFrame: Callable[[Session, ARFrame, Device], Coroutine[Any, Any, None]] | None = None
+    def __init__(self, session: Session, device: Device, onARFrame: Callable[[Session, ARFrame, Device], Coroutine[Any, Any, None]]):
+        self.camera = cv2.VideoCapture(0)
+        self.onARFrame = onARFrame
+        self.session = session
+        self.device = device
+    def __del__(self):
+        if self.camera is not None:
+            self.camera.release()
+            self.camera = None
+    async def gather_camera_frame_async(self) -> None:
+        if self.camera is None:
+            return
+        ret, frame = self.camera.read()
+        if not ret:
+            return
+        height, width = frame.shape[:2]
+        yuv = (cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)).flatten()
+        y_size = width * height
+        uv_size = y_size // 4
+        Y: XRCpuImage.Plane = XRCpuImage.Plane(data = (yuv[:y_size].reshape((height, width))).tobytes(), row_stride = width, pixel_stride=1)
+        U: XRCpuImage.Plane = XRCpuImage.Plane(data = (yuv[y_size:y_size + uv_size].reshape((height // 2, width // 2))).tobytes(), row_stride = width // 2, pixel_stride=1)
+        V: XRCpuImage.Plane = XRCpuImage.Plane(data = (yuv[y_size + uv_size:].reshape((height // 2, width // 2))).tobytes(), row_stride = width // 2, pixel_stride=1)
+        # Trim the U and V planes because ARFlow adds an extra byte as it is a bug with the android format
+        U.data = U.data[:-1]
+        V.data = V.data[:-1]
+        now = time.time()
+        timestamp = Timestamp()
+        nanos = int(now * 1e9)
+        Timestamp.FromNanoseconds(timestamp, nanos)
+        xrcpu_image: XRCpuImage = XRCpuImage(
+            dimensions= Vector2Int(x=width, y=height),
+            format= XRCpuImage.FORMAT_ANDROID_YUV_420_888,
+            timestamp=now,
+            planes=[Y, U, V]
+        )
+        color_frame = ColorFrame(
+            image=xrcpu_image,
+            device_timestamp=timestamp
+        )
+        ar_frame = ARFrame(
+            color_frame=color_frame
+        )
+        if self.onARFrame and self.session and self.device:
+            await self.onARFrame(self.session, ar_frame, self.device)
+
+
+
+

python/examples/simple/simple.py

@@ -5,19 +5,28 @@
 
 import arflow
 
-
 class CustomService(arflow.ARFlowServicer):
-    def on_register(self, request: arflow.RegisterClientRequest):
-        """Called when a client registers."""
-        print("Client registered!")
-
-    def on_frame_received(self, decoded_data_frame: arflow.DecodedDataFrame):
-        """Called when a frame is received."""
-        print("Frame received!")
-
+    def on_save_ar_frames(self, frames, session_stream, device):
+        print("AR frame received")
+    def on_save_transform_frames(self, frames, session_stream, device):
+        print("Transform frame received")
+    def on_save_color_frames(self, frames, session_stream, device):
+        print("Color frame received")
+    def on_save_depth_frames(self, frames, session_stream, device):
+        print("Depth frame received")
+    def on_save_gyroscope_frames(self, frames, session_stream, device):
+        print("Gyroscope frame received")
+    def on_save_audio_frames(self, frames, session_stream, device):
+        print("Audio frame received")
+    def on_save_plane_detection_frames(self, frames, session_stream, device):
+        print("Plane detection frame received")
+    def on_save_point_cloud_frames(self, frames, session_stream, device):
+        print("Point cloud frame received")
+    def on_save_mesh_detection_frames(self, frames, session_stream, device):
+        print("Mesh detection frame received")
 
 def main():
-    arflow.run_server(CustomService, port=8500, save_dir=Path("./"))
+    arflow.run_server(CustomService, spawn_viewer = True, port=8500)
 
 
 if __name__ == "__main__":