Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion core/meetings/modules/mixed-capture-core/src/csrc-poll.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,11 @@ const receiver = (sources: () => ContributingSourceLike[]) => ({
check('past inactiveMs the deactivation is SYNTHESIZED (the transport never sends this edge)',
out.length === 2 && out[1].active === false && out[1].csrc === 7 && out[1].tMs === t,
JSON.stringify(out));
check('the default inactivity window is 400ms', CSRC_INACTIVE_MS === 400);
// The module's own default spans a PACKET gap (jitter, DTX), not a speech PAUSE. It is only a
// default: the measured production window is longer and is passed in by the composition root,
// so what this file owns is the SEAM, not the number production runs on.
check('the built-in window is the packet-gap default, and nothing more than a default',
CSRC_INACTIVE_MS === 400, String(CSRC_INACTIVE_MS));

// Speaking again is a NEW activation — turns are edges, not a level.
speaking = true; lastSpoke = t; poll.poll();
Expand All @@ -100,6 +104,59 @@ const receiver = (sources: () => ContributingSourceLike[]) => ({
check('a poll after destroy() emits nothing', out.length === after);
}

// ── the inactivity window is a PARAMETER, not a constant ─────────────────────────────────
// A caller with a MEASURED window must be able to hand it over instead of forking this file —
// the seam, not a copied constant, is the contract.
{
let t = 1_900_000_000_000;
const out: CsrcTransition[] = [];
let speaking = true;
let lastSpoke = t;
const poll = createCsrcPoll({
onTransition: (x) => out.push(x),
now: () => t,
timeOrigin: () => 0,
inactiveMs: 800,
receivers: () => [receiver(() => [{ source: 9, timestamp: speaking ? t : lastSpoke, audioLevel: 0.4 }])],
});
poll.poll();
check('override: the source opens a turn', out.length === 1 && out[0].active === true, JSON.stringify(out));
speaking = false;
t += 500; poll.poll();
check('override: 500ms of silence — past the 400ms default, inside the caller\'s 800ms — holds the turn OPEN',
out.length === 1, JSON.stringify(out));
t += 400; poll.poll(); // 900ms > 800ms
check('override: past the caller\'s window the deactivation is synthesized',
out.length === 2 && out[1].active === false, JSON.stringify(out));
poll.destroy();
}

// ── an unusable window never reaches the comparisons ────────────────────────────────────────
// NaN makes both liveness comparisons false: every source would re-activate and re-deactivate on
// every poll. The sensor is the point of introduction, so it is the sensor that refuses it.
{
for (const bad of [Number.NaN, 50, Number.POSITIVE_INFINITY]) {
let t = 1_900_000_000_000;
const out: CsrcTransition[] = [];
const logs: string[] = [];
const poll = createCsrcPoll({
onTransition: (x) => out.push(x),
now: () => t,
timeOrigin: () => 0,
inactiveMs: bad,
log: (m) => logs.push(m),
// always 200 ms stale: inside the 400 ms default, so a healthy sensor holds ONE activation
receivers: () => [receiver(() => [{ source: 11, timestamp: t - 200, audioLevel: 0.4 }])],
});
poll.poll(); t += 100; poll.poll(); t += 100; poll.poll();
check(`unusable inactiveMs=${String(bad)}: falls back to the default — one activation, no flip-flop`,
out.length === 1 && out[0].active === true, JSON.stringify(out));
check(`unusable inactiveMs=${String(bad)}: is said through log`,
logs.some((m) => m.includes('not a usable window')), JSON.stringify(logs));
poll.destroy();
}
}

// ── two concurrent sources ──────────────────────────────────────────────────────────────────────
{
let t = 1_900_000_000_000;
Expand Down
12 changes: 10 additions & 2 deletions core/meetings/modules/mixed-capture-core/src/csrc-poll.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,10 @@ export interface CsrcPollOptions {
receivers?: () => CsrcReceiverLike[];
/** Poll cadence in ms. Default 100 — the granularity the transport itself updates at. */
pollMs?: number;
/** How long a source stays active after its last observed contribution. Default 400 ms. */
/** How long a source stays active after its last observed contribution. Default 400 ms — a
* packet-gap window; a caller with a measured speech-pause window passes its own. A value that
* is not finite or is shorter than one poll is unusable (NaN makes both liveness comparisons
* false and flips every source on every poll) and is replaced by the default, said via `log`. */
inactiveMs?: number;
/** Epoch-ms clock (injectable for tests). */
now?: () => number;
Expand Down Expand Up @@ -182,12 +185,17 @@ const isAudioReceiver = (r: CsrcReceiverLike): boolean =>
*/
export function createCsrcPoll(opts: CsrcPollOptions): CsrcPoll {
const pollMs = opts.pollMs ?? CSRC_POLL_MS;
const inactiveMs = opts.inactiveMs ?? CSRC_INACTIVE_MS;
const requestedInactiveMs = opts.inactiveMs;
const inactiveMs = typeof requestedInactiveMs === 'number' && Number.isFinite(requestedInactiveMs)
&& requestedInactiveMs >= pollMs ? requestedInactiveMs : CSRC_INACTIVE_MS;
const now = opts.now ?? (() => Date.now());
const timeOrigin = opts.timeOrigin
?? (() => Number((globalThis as unknown as { performance?: { timeOrigin?: number } }).performance?.timeOrigin ?? 0));
const receivers = opts.receivers ?? hookedReceivers;
const log = opts.log ?? (() => { /* silent */ });
if (requestedInactiveMs !== undefined && inactiveMs !== requestedInactiveMs) {
log(`inactiveMs=${String(requestedInactiveMs)} is not a usable window (finite, >= the ${pollMs}ms poll) — using ${CSRC_INACTIVE_MS}ms`);
}

/** csrc → the last moment (epoch ms) it was observed contributing, plus its last carried levels. */
const live = new Map<number, { lastSeen: number; audioLevel?: number; rtpTimestamp?: number }>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -651,8 +651,8 @@ export class ChunkedTranscriber {
if (!mine()) return;
if (this.turn && ev.tracks) this.turn.spanTracks = ev.tracks;
// A segmenter's speech-end lands a little early and needs a trailing STT pad; a transport
// deactivation already carries the sensor's own 400ms inactivity window, so padding it
// again would only feed Whisper more silence.
// deactivation already carries the sensor's inactivity window (800 ms as the bot composes
// it), so padding it again would only feed Whisper more silence.
this.closeTurn(ev.t1, ev.reason === 'silence' ? SILENCE_CLOSE_CONTEXT_MS : 0);
},
};
Expand Down
8 changes: 5 additions & 3 deletions core/meetings/modules/mixed-pipeline/src/turn-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,11 @@ const envNumber = (name: string, fallback: number): number => {
return Number.isFinite(n) && n > 0 ? n : fallback;
};

/** How long a source may go quiet before its turn is closed. The sensor already waits 400ms before
* declaring a deactivation, so this is the SECOND grace: it spans a breath, a DTX gap, a dropped
* packet train. Too small shatters a sentence into turns; too large merges a real handoff. */
/** How long a source may go quiet before its turn is closed. The sensor already holds a source
* active for its own inactivity window before it synthesizes a deactivation (the bot composes
* 800 ms; the brick's built-in default is 400 ms), so this is the SECOND grace: it spans a
* breath, a DTX gap, a dropped packet train. Too small shatters a sentence into turns; too large
* merges a real handoff. */
export const CSRC_HYSTERESIS_MS = envNumber('VEXA_CSRC_HYSTERESIS_MS', 600);
/** Energetic audio this long with NOT ONE transition ⇒ the transport stopped talking to us while
* the meeting continued. Measured in audio time, on frames that carry energy, so an honestly
Expand Down
50 changes: 48 additions & 2 deletions core/meetings/services/bot/src/capture-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,47 @@ const TEAMS_ENABLE_CAPTIONS = process.env.VEXA_TEAMS_ENABLE_CAPTIONS === '1';
/** How many times to try the menu path before giving up (each attempt is ~3 s of UI waits). */
const TEAMS_ENABLE_CAPTIONS_ATTEMPTS = Math.max(1, Number(process.env.VEXA_TEAMS_ENABLE_CAPTIONS_ATTEMPTS || 3));

/** The sensor's poll cadence (`CSRC_POLL_MS`). Not imported: this file must not pull the page-side
* bundle into the Node build. Kept as the floor only, so a drift there is a slacker floor here,
* never a wrong window. */
const CSRC_MIN_INACTIVE_MS = 100;
/** No plausible window is 10 s: the measurement below shows 1600 ms already merges distinct
* turns, so a huge override is a typo wearing a number — the NaN failure mode again, silent. */
const CSRC_MAX_INACTIVE_MS = 10_000;
/**
* The measured window — how long the transport sensor holds a source active after its last
* observed contribution. One name, so the fallback, the message and the default cannot drift.
*
* The sensor's own default is 400 ms (`CSRC_INACTIVE_MS` in @vexa/mixed-capture-core) — a value
* sized to span a packet gap. A speech PAUSE is longer. Measured in a downstream deployment on
* real Teams sessions and replayed through the Teams CSRC channelizer (#1383 carries the
* measurement; it is not reproduced in this tree): entry-timestamp staleness during natural
* pauses is p50 550 ms, so at 400 ms the MEDIAN pause trips a synthesized deactivation and one
* speech phase fragments; 400 ms yields 1.67 lane activations per real turn, 800 ms yields 1.13 —
* the knee — for 1.8 pp more contamination; 1600 ms starts merging distinct turns. So the
* production window is 800 ms.
*
* The number lives HERE, at the composition root; the sensor owns the `inactiveMs` seam and its
* packet-gap default, nothing more. Overridable via VEXA_CSRC_INACTIVE_MS: the deployment renders
* it onto the runtime (compose, helm `runtime.csrcInactiveMs`, lite — see docs/configuration),
* the runtime kernel forwards it to spawned bots through profiles.py's tuning allowlist, and the
* lite/process backend inherits the host env directly.
*/
const CSRC_DEFAULT_INACTIVE_MS = 800;
export function resolveCsrcInactiveMs(raw: string | undefined, warn: (m: string) => void = () => { /* silent */ }): number {
// Unset and blank are both "no override": compose, helm and lite render an UNSET knob as an
// empty string (`${VAR:-}`), and every other bot tuning knob reads it that way. A NON-blank value
// is an intent — usable ⇒ honoured; unusable ⇒ said out loud and replaced by the measured default.
if (raw === undefined || raw.trim() === '') return CSRC_DEFAULT_INACTIVE_MS;
const n = Number(raw);
if (!Number.isFinite(n) || n < CSRC_MIN_INACTIVE_MS || n > CSRC_MAX_INACTIVE_MS) {
warn(`[bot] VEXA_CSRC_INACTIVE_MS=${JSON.stringify(raw)} is not a usable window (finite, ${CSRC_MIN_INACTIVE_MS}..${CSRC_MAX_INACTIVE_MS}ms) — using the measured ${CSRC_DEFAULT_INACTIVE_MS}ms`);
return CSRC_DEFAULT_INACTIVE_MS;
}
return n;
}
const CSRC_RESOLVED_INACTIVE_MS = resolveCsrcInactiveMs(process.env.VEXA_CSRC_INACTIVE_MS, (m) => console.warn(m));

/** Outcome of one enable attempt. `already-on` and `clicked` are successes; `failed` carries WHY,
* because "captions never appeared" has two very different causes — the menu path changed, or the
* tenant blocks captions — and only the reason distinguishes them on the first live run. */
Expand Down Expand Up @@ -796,7 +837,7 @@ export async function startCaptureBridge(
// ── Start the page-side capture (VexaBrowserUtils preferred; production inline fallback). ──
// The body of this callback runs IN THE BROWSER (Playwright serializes it); DOM globals are
// reached via globalThis (this file type-checks against the Node lib — no DOM types here).
await page.evaluate(async ({ isMixed, isPerTrack, isJitsi, isTeams, isZoom, botName, mainAudioGraceMs, mainAudioSilenceMs, mainAudioEnergyRms }) => {
await page.evaluate(async ({ isMixed, isPerTrack, isJitsi, isTeams, isZoom, botName, mainAudioGraceMs, mainAudioSilenceMs, mainAudioEnergyRms, csrcInactiveMs }) => {
const w = (globalThis as any) as Record<string, any>;
if (isMixed) {
// Zoom/Teams/Jitsi ride the WebRTC hook (installRemoteAudioHook, installed pre-nav), which mirrors
Expand Down Expand Up @@ -1159,6 +1200,9 @@ export async function startCaptureBridge(
w.logBot?.('[Csrc] observation ' + JSON.stringify(o));
w.__vexaObservation?.('csrc', o, Date.now());
},
// The MEASURED pause window, passed in rather than defaulted: the sensor's own 400 ms
// is shorter than the median natural speech pause (p50 550 ms) and fragments a turn.
inactiveMs: csrcInactiveMs,
log: (m: string) => w.logBot?.('[Csrc] ' + m),
});
} catch (e: any) {
Expand Down Expand Up @@ -1287,7 +1331,9 @@ export async function startCaptureBridge(
mainAudioGraceMs: Number(process.env.VEXA_TEAMS_MAIN_AUDIO_GRACE_MS || 15000),
// How long a PICKED mix may stay wholly silent before the lane abandons it for every track.
mainAudioSilenceMs: Number(process.env.VEXA_TEAMS_MAIN_AUDIO_SILENCE_MS || 20000),
mainAudioEnergyRms: Number(process.env.VEXA_TEAMS_MAIN_AUDIO_ENERGY_RMS || 0.006) }).catch((e) => {
mainAudioEnergyRms: Number(process.env.VEXA_TEAMS_MAIN_AUDIO_ENERGY_RMS || 0.006),
// The measured inactivity window — a parameter the sensor accepts, never a fork of it.
csrcInactiveMs: CSRC_RESOLVED_INACTIVE_MS }).catch((e) => {
console.error(`[bot] capture bridge: page-side start failed: ${String(e)}`); // L4: surfaces only on the VM
});

Expand Down
Loading
Loading