Add support for noiseCancellation frameProcessors (#966)

Author: lukasIO

.changeset/large-cars-pull.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+Add support for noiseCancellation frameProcessors

agents/src/voice/room_io/_input.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioFrame, FrameProcessor } from '@livekit/rtc-node';
 import {
   AudioStream,
   type NoiseCancellationOptions,
@@ -22,6 +22,7 @@ export class ParticipantAudioInputStream extends AudioInput {
   private sampleRate: number;
   private numChannels: number;
   private noiseCancellation?: NoiseCancellationOptions;
+  private frameProcessor?: FrameProcessor<AudioFrame>;
   private publication: RemoteTrackPublication | null = null;
   private participantIdentity: string | null = null;
   private logger = log();
@@ -34,16 +35,21 @@ export class ParticipantAudioInputStream extends AudioInput {
     room: Room;
     sampleRate: number;
     numChannels: number;
-    noiseCancellation?: NoiseCancellationOptions;
+    noiseCancellation?: NoiseCancellationOptions | FrameProcessor<AudioFrame>;
   }) {
     super();
     this.room = room;
     this.sampleRate = sampleRate;
     this.numChannels = numChannels;
-    this.noiseCancellation = noiseCancellation;
+    if (noiseCancellation instanceof FrameProcessor) {
+      this.frameProcessor = noiseCancellation;
+    } else {
+      this.noiseCancellation = noiseCancellation;
+    }
 
     this.room.on(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
     this.room.on(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+    this.room.on(RoomEvent.TokenRefreshed, this.onTokenRefreshed);
   }
 
   setParticipant(participant: RemoteParticipant | string | null) {
@@ -116,6 +122,9 @@ export class ParticipantAudioInputStream extends AudioInput {
     if (this.deferredStream.isSourceSet) {
       this.deferredStream.detachSource();
     }
+
+    this.frameProcessor?.close();
+
     this.publication = null;
   }
 
@@ -140,21 +149,40 @@ export class ParticipantAudioInputStream extends AudioInput {
         outputRate: this.sampleRate,
       }),
     );
+    this.frameProcessor?.onStreamInfoUpdated({
+      participantIdentity: participant.identity,
+      roomName: this.room.name!,
+      publicationSid: publication.sid!,
+    });
+    this.frameProcessor?.onCredentialsUpdated({
+      token: this.room.token!,
+      url: this.room.serverUrl!,
+    });
     return true;
   };
 
+  private onTokenRefreshed = () => {
+    if (this.room.token && this.room.serverUrl) {
+      this.frameProcessor?.onCredentialsUpdated({
+        token: this.room.token,
+        url: this.room.serverUrl,
+      });
+    }
+  };
+
   private createStream(track: RemoteTrack): ReadableStream<AudioFrame> {
     return new AudioStream(track, {
       sampleRate: this.sampleRate,
       numChannels: this.numChannels,
-      noiseCancellation: this.noiseCancellation,
+      noiseCancellation: this.frameProcessor || this.noiseCancellation,
       // TODO(AJS-269): resolve compatibility issue with node-sdk to remove the forced type casting
     }) as unknown as ReadableStream<AudioFrame>;
   }
 
   async close() {
     this.room.off(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
     this.room.off(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+    this.room.off(RoomEvent.TokenRefreshed, this.onTokenRefreshed);
     this.closeStream();
     // Ignore errors - stream may be locked by RecorderIO or already cancelled
     await this.deferredStream.stream.cancel().catch(() => {});

agents/src/voice/room_io/room_io.ts

@@ -2,8 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import {
+  type AudioFrame,
   ConnectionState,
   DisconnectReason,
+  type FrameProcessor,
   type NoiseCancellationOptions,
   type Participant,
   ParticipantKind,
@@ -75,7 +77,7 @@ export interface RoomInputOptions {
     Can be overridden by the `participant` argument of RoomIO constructor or `set_participant`.
   */
   participantIdentity?: string;
-  noiseCancellation?: NoiseCancellationOptions;
+  noiseCancellation?: NoiseCancellationOptions | FrameProcessor<AudioFrame>;
   textInputCallback?: TextInputCallback;
   /** Participant kinds accepted for auto subscription. If not provided,
     accept `DEFAULT_PARTICIPANT_KINDS`

examples/package.json

@@ -40,6 +40,7 @@
     "@livekit/agents-plugin-silero": "workspace:*",
     "@livekit/agents-plugin-xai": "workspace:*",
     "@livekit/noise-cancellation-node": "^0.1.9",
+    "@livekit/plugins-ai-coustics": "0.1.7",
     "@livekit/rtc-node": "catalog:",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/api-logs": "^0.54.0",

examples/src/basic_agent.ts

@@ -13,7 +13,8 @@ import {
 } from '@livekit/agents';
 import * as livekit from '@livekit/agents-plugin-livekit';
 import * as silero from '@livekit/agents-plugin-silero';
-import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node';
+// import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node';
+import * as aic from '@livekit/plugins-ai-coustics';
 import { fileURLToPath } from 'node:url';
 import { z } from 'zod';
 
@@ -82,7 +83,9 @@ export default defineAgent({
       agent,
       room: ctx.room,
       inputOptions: {
-        noiseCancellation: BackgroundVoiceCancellation(),
+        noiseCancellation: aic.audioEnhancement(),
+        // or for krisp use
+        // noiseCancellation: BackgroundVoiceCancellation(),
       },
     });