RSDK-12152 RSDK-12153 AudioIn and AudioOut component wrappers (#655)

Author: oliviamiller

src/audio-common.ts

@@ -0,0 +1,22 @@
+export interface AudioProperties {
+  /** List of audio codecs supported by the device */
+  supportedCodecs: string[];
+  /** Current sample rate in Hz */
+  sampleRateHz: number;
+  /** Maximum number of audio channels supported (e.g., 1 for mono, 2 for stereo) */
+  numChannels: number;
+}
+
+/** Common audio codec constants */
+export const AudioCodec = {
+  MP3: 'mp3',
+  PCM16: 'pcm16',
+  PCM32: 'pcm32',
+  PCM32_FLOAT: 'pcm32float',
+  AAC: 'aac',
+  OPUS: 'opus',
+  FLAC: 'flac',
+  WAV: 'wav',
+} as const;
+
+export type AudioCodecType = (typeof AudioCodec)[keyof typeof AudioCodec];

src/components/audio-in/audio-in.ts

@@ -0,0 +1,45 @@
+import type { Resource, Struct } from '../../types';
+import type { AudioInfo } from '../../gen/common/v1/common_pb';
+import type { AudioProperties } from '../../audio-common';
+
+export interface AudioChunk {
+  audioData: Uint8Array;
+  audioInfo?: AudioInfo;
+  startTimeNs: bigint;
+  endTimeNs: bigint;
+  sequence: number;
+  requestID: string;
+}
+
+/** Represents a device that takes audio input. */
+
+export interface AudioIn extends Resource {
+  /**
+   * Stream audio from the device.
+   *
+   * @example
+   *
+   * ```ts
+   * const audioIn = new VIAM.AudioInClient(machine, 'my_audio_in');
+   * const stream = audioIn.getAudio(VIAM.AudioCodec.PCM16, 3, 0n, {});
+   * ```
+   */
+  getAudio(
+    codec: string,
+    durationSeconds: number,
+    previousTimestamp?: bigint,
+    extra?: Struct
+  ): AsyncIterable<AudioChunk>;
+
+  /**
+   * Return the audio input properties.
+   *
+   * @example
+   *
+   * ```ts
+   * const audioIn = new VIAM.AudioInClient(machine, 'my_audio_in');
+   * const properties = await audioIn.getProperties();
+   * ```
+   */
+  getProperties: () => Promise<AudioProperties>;
+}

src/components/audio-in/client.spec.ts

@@ -0,0 +1,129 @@
+// @vitest-environment happy-dom
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { GetAudioResponse } from '../../gen/component/audioin/v1/audioin_pb';
+import { GetPropertiesResponse } from '../../gen/common/v1/common_pb';
+import { RobotClient } from '../../robot';
+import { type AudioChunk } from './audio-in';
+import { AudioInClient } from './client';
+import { AudioCodec } from '../../audio-common';
+vi.mock('../../robot');
+
+import type { PartialMessage } from '@bufbuild/protobuf';
+import { createClient, createRouterTransport } from '@connectrpc/connect';
+import {
+  createWritableIterable,
+  type WritableIterable,
+} from '@connectrpc/connect/protocol';
+import { AudioInService } from '../../gen/component/audioin/v1/audioin_connect';
+
+let audioin: AudioInClient;
+
+let testAudioStream: WritableIterable<PartialMessage<GetAudioResponse>>;
+
+const testProperties = new GetPropertiesResponse({
+  supportedCodecs: [AudioCodec.PCM16, AudioCodec.MP3, AudioCodec.PCM32_FLOAT],
+  sampleRateHz: 48_000,
+  numChannels: 2,
+});
+
+describe('AudioInClient tests', () => {
+  beforeEach(() => {
+    testAudioStream =
+      createWritableIterable<PartialMessage<GetAudioResponse>>();
+
+    const mockTransport = createRouterTransport(({ service }) => {
+      service(AudioInService, {
+        getAudio: () => {
+          return testAudioStream;
+        },
+        getProperties: () => {
+          return testProperties;
+        },
+      });
+    });
+
+    RobotClient.prototype.createServiceClient = vi
+      .fn()
+      .mockImplementation(() => createClient(AudioInService, mockTransport));
+
+    audioin = new AudioInClient(new RobotClient('host'), 'test-audio-in');
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('getAudio tests', () => {
+    it('getAudio streams audio chunks', async () => {
+      const audioChunks: AudioChunk[] = [];
+
+      const streamProm = (async () => {
+        for await (const chunk of audioin.getAudio(
+          AudioCodec.PCM16,
+          1.1,
+          BigInt(0)
+        )) {
+          audioChunks.push(chunk);
+        }
+      })();
+
+      await testAudioStream.write({
+        audio: {
+          audioData: new Uint8Array([4, 5, 6]),
+          audioInfo: {
+            codec: AudioCodec.PCM16,
+            sampleRateHz: 48_000,
+            numChannels: 2,
+          },
+          startTimestampNanoseconds: BigInt(1000),
+          endTimestampNanoseconds: BigInt(2000),
+          sequence: 1,
+        },
+        requestId: 'test-request-1',
+      });
+
+      await testAudioStream.write({
+        audio: {
+          audioData: new Uint8Array([7, 8, 9]),
+          audioInfo: {
+            codec: AudioCodec.PCM16,
+            sampleRateHz: 48_000,
+            numChannels: 2,
+          },
+          startTimestampNanoseconds: BigInt(2000),
+          endTimestampNanoseconds: BigInt(3000),
+          sequence: 2,
+        },
+        requestId: 'test-request-1',
+      });
+
+      testAudioStream.close();
+      await streamProm;
+
+      expect(audioChunks.length).toEqual(2);
+
+      const chunk1 = audioChunks[0]!;
+      expect(chunk1.audioData).toEqual(new Uint8Array([4, 5, 6]));
+      expect(chunk1.sequence).toEqual(1);
+
+      const chunk2 = audioChunks[1]!;
+      expect(chunk2.audioData).toEqual(new Uint8Array([7, 8, 9]));
+      expect(chunk2.sequence).toEqual(2);
+    });
+  });
+
+  describe('getProperties tests', () => {
+    it('getProperties returns audio properties', async () => {
+      const properties = await audioin.getProperties();
+
+      expect(properties.supportedCodecs).toEqual([
+        AudioCodec.PCM16,
+        AudioCodec.MP3,
+        AudioCodec.PCM32_FLOAT,
+      ]);
+      expect(properties.sampleRateHz).toEqual(48_000);
+      expect(properties.numChannels).toEqual(2);
+    });
+  });
+});

src/components/audio-in/client.ts

@@ -0,0 +1,92 @@
+import type { RobotClient } from '../../robot';
+import type { Options } from '../../types';
+
+import { Struct, type JsonValue } from '@bufbuild/protobuf';
+import type { CallOptions, Client } from '@connectrpc/connect';
+import { AudioInService } from '../../gen/component/audioin/v1/audioin_connect';
+import { GetAudioRequest } from '../../gen/component/audioin/v1/audioin_pb';
+import { GetPropertiesRequest } from '../../gen/common/v1/common_pb';
+import { type AudioIn, type AudioChunk } from './audio-in';
+import { doCommandFromClient } from '../../utils';
+
+/*
+ * A gRPC-web client for the AudioIn component.
+ *
+ * @group Clients
+ */
+export class AudioInClient implements AudioIn {
+  private client: Client<typeof AudioInService>;
+  public readonly name: string;
+  private readonly options: Options;
+  public callOptions: CallOptions = { headers: {} as Record<string, string> };
+
+  constructor(client: RobotClient, name: string, options: Options = {}) {
+    this.client = client.createServiceClient(AudioInService);
+    this.name = name;
+    this.options = options;
+  }
+
+  async *getAudio(
+    codec: string,
+    durationSeconds: number,
+    previousTimestamp = 0n,
+    extra = {},
+    callOptions = this.callOptions
+  ): AsyncIterable<AudioChunk> {
+    const request = new GetAudioRequest({
+      name: this.name,
+      codec,
+      durationSeconds,
+      previousTimestampNanoseconds: previousTimestamp,
+      extra: Struct.fromJson(extra),
+    });
+
+    this.options.requestLogger?.(request);
+
+    const stream = this.client.getAudio(request, callOptions);
+
+    // Yield chunks as they arrive
+    for await (const resp of stream) {
+      if (!resp.audio) {
+        continue;
+      }
+      yield {
+        audioData: resp.audio.audioData,
+        audioInfo: resp.audio.audioInfo,
+        startTimeNs: resp.audio.startTimestampNanoseconds,
+        endTimeNs: resp.audio.endTimestampNanoseconds,
+        sequence: resp.audio.sequence,
+        requestID: resp.requestId,
+      };
+    }
+  }
+
+  async getProperties(callOptions = this.callOptions) {
+    const request = new GetPropertiesRequest({
+      name: this.name,
+    });
+
+    this.options.requestLogger?.(request);
+
+    const response = await this.client.getProperties(request, callOptions);
+
+    return {
+      supportedCodecs: response.supportedCodecs,
+      sampleRateHz: response.sampleRateHz,
+      numChannels: response.numChannels,
+    };
+  }
+
+  async doCommand(
+    command: Struct,
+    callOptions = this.callOptions
+  ): Promise<JsonValue> {
+    return doCommandFromClient(
+      this.client.doCommand,
+      this.name,
+      command,
+      this.options,
+      callOptions
+    );
+  }
+}

src/components/audio-out/audio-out.ts

@@ -0,0 +1,40 @@
+import type { Resource, Struct } from '../../types';
+import type { AudioInfo } from '../../gen/common/v1/common_pb';
+import type { AudioProperties } from '../../audio-common';
+
+/** Represents a device that outputs audio. */
+export interface AudioOut extends Resource {
+  /**
+   * Play audio on the device.
+   *
+   * @example
+   *
+   * ```ts
+   * const audioOut = new VIAM.AudioOutClient(machine, 'my_audio_out');
+   * const audioData = new Uint8Array([...]); // Your audio data
+   * const audioInfo = { codec: 'pcm16', sampleRateHz: 48000, numChannels: 2 };
+   * await audioOut.play(audioData, audioInfo);
+   * ```
+   *
+   * @param audioData - The audio data to play
+   * @param audioInfo - Information about the audio format (optional, required
+   *   for raw pcm data)
+   */
+  play: (
+    audioData: Uint8Array,
+    audioInfo?: AudioInfo,
+    extra?: Struct
+  ) => Promise<void>;
+
+  /**
+   * Return the audio output properties.
+   *
+   * @example
+   *
+   * ```ts
+   * const audioOut = new VIAM.AudioOutClient(machine, 'my_audio_out');
+   * const properties = await audioOut.getProperties();
+   * ```
+   */
+  getProperties: () => Promise<AudioProperties>;
+}