fix(call): improve virtual background performance

src/components/MediaSettings/VideoBackgroundEditor.vue

@@ -67,6 +67,24 @@
 			tabindex="-1"
 			aria-hidden="true"
 			@change="handleFileInput">
+		<div class="background-editor__debug-controls">
+			<label
+				v-for="control in debugControls"
+				:key="control.key"
+				class="background-editor__debug-control">
+				<span class="background-editor__debug-control-header">
+					<span>{{ control.label }}</span>
+					<span>{{ formatDebugConfigValue(control.key) }}</span>
+				</span>
+				<input
+					:min="control.min"
+					:max="control.max"
+					:step="control.step"
+					:value="debugConfigValues[control.key]"
+					type="range"
+					@input="handleDebugConfigInput(control.key, $event.target.value)">
+			</label>
+		</div>
 	</div>
 </template>
 
@@ -88,6 +106,7 @@ import { getDavClient } from '../../services/DavClient.ts'
 import { useActorStore } from '../../stores/actor.ts'
 import { useSettingsStore } from '../../stores/settings.ts'
 import { findUniquePath } from '../../utils/fileUpload.ts'
+import { VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES, virtualBackgroundDebugConfig, setVirtualBackgroundDebugConfigValue } from '../../utils/media/effects/virtual-background/runtimeConfig.js'
 
 const predefinedBackgroundLabels = {
 	'1_office': t('spreed', 'Select virtual office background'),
@@ -100,6 +119,24 @@ const predefinedBackgroundLabels = {
 	'8_space_station': t('spreed', 'Select virtual space station background'),
 }
 
+const virtualBackgroundDebugControlLabels = {
+	DEFAULT_BLUR_PASSES: t('spreed', 'Blur passes'),
+	SIGMA_SPACE: t('spreed', 'Sigma space'),
+	SIGMA_COLOR: t('spreed', 'Sigma color'),
+	SPARSITY_FACTOR: t('spreed', 'Sparsity factor'),
+	DEFAULT_FRAME_RATE: t('spreed', 'Default frame rate'),
+	MAX_SEGMENTATION_FRAME_RATE: t('spreed', 'Max segmentation frame rate'),
+}
+
+const virtualBackgroundDebugControlFractionDigits = {
+	DEFAULT_BLUR_PASSES: 0,
+	SIGMA_SPACE: 1,
+	SIGMA_COLOR: 2,
+	SPARSITY_FACTOR: 2,
+	DEFAULT_FRAME_RATE: 0,
+	MAX_SEGMENTATION_FRAME_RATE: 0,
+}
+
 export default {
 	name: 'VideoBackgroundEditor',
 
@@ -139,6 +176,7 @@ export default {
 	data() {
 		return {
 			selectedBackground: undefined,
+			debugConfigValues: { ...virtualBackgroundDebugConfig },
 		}
 	},
 
@@ -162,6 +200,15 @@ export default {
 		relativeBackgroundsFolderPath() {
 			return this.settingsStore.attachmentFolder + '/Backgrounds'
 		},
+
+		debugControls() {
+			return Object.entries(VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES).map(([key, range]) => ({
+				key,
+				label: virtualBackgroundDebugControlLabels[key],
+				fractionDigits: virtualBackgroundDebugControlFractionDigits[key],
+				...range,
+			}))
+		},
 	},
 
 	async mounted() {
@@ -268,6 +315,17 @@ export default {
 			this.handleSelectBackground(previewURL)
 		},
 
+		handleDebugConfigInput(key, value) {
+			this.debugConfigValues[key] = setVirtualBackgroundDebugConfigValue(key, value)
+		},
+
+		formatDebugConfigValue(key) {
+			const value = this.debugConfigValues[key]
+			const fractionDigits = virtualBackgroundDebugControlFractionDigits[key]
+
+			return fractionDigits > 0 ? value.toFixed(fractionDigits) : String(value)
+		},
+
 		loadBackground() {
 			// Set virtual background depending on browser storage's settings
 			if (BrowserStorage.getItem('virtualBackgroundEnabled') === 'true') {
@@ -305,6 +363,24 @@ export default {
 	max-height: calc(var(--background-button-height) * 3 + var(--default-grid-baseline) * 4);
 	overflow-y: auto;
 
+	&__debug-controls {
+		grid-column: 1 / -1;
+		display: grid;
+		gap: calc(var(--default-grid-baseline) * 2);
+	}
+
+	&__debug-control {
+		display: grid;
+		gap: var(--default-grid-baseline);
+	}
+
+	&__debug-control-header {
+		display: flex;
+		justify-content: space-between;
+		gap: calc(var(--default-grid-baseline) * 2);
+		font-size: 12px;
+	}
+
 	&__element {
 		border: none;
 		margin: 0 !important;

src/main.js

@@ -14,6 +14,7 @@ import store from './store/index.js'
 import pinia from './stores/pinia.ts'
 import { useSidebarStore } from './stores/sidebar.ts'
 import { NextcloudGlobalsVuePlugin } from './utils/NextcloudGlobalsVuePlugin.js'
+import { exposeVirtualBackgroundDebugConfig } from './utils/media/effects/virtual-background/runtimeConfig.js'
 
 import './init.js'
 // Leaflet icon patch
@@ -104,6 +105,7 @@ subscribe('viewer:sidebar:open', (node) => {
 if (!window.OCA.Talk) {
 	window.OCA.Talk = reactive({})
 }
+exposeVirtualBackgroundDebugConfig(window.OCA.Talk)
 OCA.Talk.instance = instance
 OCA.Talk.Settings = SettingsAPI
 

src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js

@@ -13,6 +13,7 @@ import {
 	timerWorkerScript,
 } from './TimerWorker.js'
 import WebGLCompositor from './WebGLCompositor.js'
+import { DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG, virtualBackgroundDebugConfig } from './runtimeConfig.js'
 
 // Cache MediaPipe resources to avoid loading them multiple times.
 let _WasmFileset = null
@@ -82,6 +83,10 @@ export default class VideoStreamBackgroundEffect {
 		this._inputVideoElement = document.createElement('video')
 		this._bgChanged = false
 		this._prevBgMode = null
+		this._inputTrackFrameRate = null
+		this._frameRate = virtualBackgroundDebugConfig.DEFAULT_FRAME_RATE
+		this._runInferenceInterval = 1000 / Math.min(this._frameRate, virtualBackgroundDebugConfig.MAX_SEGMENTATION_FRAME_RATE)
+		this._lastInferenceTime = 0
 	}
 
 	/**
@@ -184,6 +189,55 @@ export default class VideoStreamBackgroundEffect {
 		}
 	}
 
+	/**
+	 * Update the input frame rate and derive the rate of segmentation read from it.
+	 *
+	 * @private
+	 * @param {number|string} frameRate - Input frame rate from track settings.
+	 * @return {void}
+	 */
+	_resolveFrameRate(frameRate) {
+		const configuredFrameRate = virtualBackgroundDebugConfig.DEFAULT_FRAME_RATE
+		const parsedFrameRate = parseInt(frameRate, 10)
+
+		if (Number.isNaN(parsedFrameRate)) {
+			return configuredFrameRate
+		}
+
+		if (configuredFrameRate === DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG.DEFAULT_FRAME_RATE) {
+			return parsedFrameRate
+		}
+
+		return Math.min(parsedFrameRate, configuredFrameRate)
+	}
+
+	_syncFrameRateConfig() {
+		const nextFrameRate = this._resolveFrameRate(this._inputTrackFrameRate)
+		const nextRunInferenceInterval = 1000 / Math.min(nextFrameRate, virtualBackgroundDebugConfig.MAX_SEGMENTATION_FRAME_RATE)
+		const didFrameRateChange = this._frameRate !== nextFrameRate
+
+		this._frameRate = nextFrameRate
+		this._runInferenceInterval = nextRunInferenceInterval
+
+		if (!didFrameRateChange || !this._outputStream) {
+			return
+		}
+
+		const outputTrack = this._outputStream.getVideoTracks()[0]
+		if (!outputTrack) {
+			return
+		}
+
+		outputTrack.applyConstraints({ frameRate: this._frameRate }).catch((error) => {
+			console.error('Frame rate could not be adjusted in background effect', error)
+		})
+	}
+
+	_updateFrameRate(frameRate) {
+		this._inputTrackFrameRate = frameRate
+		this._syncFrameRateConfig()
+	}
+
 	/**
 	 * Process MediaPipe segmentation result and update internal mask.
 	 *
@@ -274,9 +328,18 @@ export default class VideoStreamBackgroundEffect {
 			this._frameId = this._lastFrameId
 		}
 
-		// Run inference if ready
-		if (this._loaded && this._frameId === this._lastFrameId) {
+		this._syncFrameRateConfig()
+
+		const now = performance.now()
+
+		// Run inference if ready and enough time passed since last _runInference() call
+		// Otherwise, reuse the mask from previous segmentation read
+		if (
+			this._loaded && this._frameId === this._lastFrameId
+			&& now - this._lastInferenceTime >= this._runInferenceInterval
+		) {
 			this._frameId++
+			this._lastInferenceTime = now
 			this._runInference().catch((e) => console.error(e))
 		} else if (this._useWebGL) {
 			this.runPostProcessing()
@@ -417,14 +480,13 @@ export default class VideoStreamBackgroundEffect {
 		const backgroundBlurValue = this._options.virtualBackground.blurValue * scaledBlurFactor
 		const edgesBlurValue = (backgroundType === VIRTUAL_BACKGROUND.BACKGROUND_TYPE.IMAGE ? 4 : 8) * scaledBlurFactor
 
-		if (!this._outputCanvasElement.width
-			|| !this._outputCanvasElement.height) {
-			return
+		// Update canvas size only when changed / unset
+		if (this._outputCanvasElement.width !== width
+			|| this._outputCanvasElement.height !== height) {
+			this._outputCanvasElement.width = width
+			this._outputCanvasElement.height = height
 		}
 
-		this._outputCanvasElement.width = width
-		this._outputCanvasElement.height = height
-
 		if (this._useWebGL) {
 			if (!this._glFx) {
 				return
@@ -606,7 +668,7 @@ export default class VideoStreamBackgroundEffect {
 		const { height, frameRate, width }
 			= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints()
 
-		this._frameRate = parseInt(frameRate, 10)
+		this._updateFrameRate(frameRate)
 
 		this._outputCanvasElement.width = parseInt(width, 10)
 		this._outputCanvasElement.height = parseInt(height, 10)
@@ -644,6 +706,7 @@ export default class VideoStreamBackgroundEffect {
 
 		this._frameId = -1
 		this._lastFrameId = -1
+		this._lastInferenceTime = 0
 
 		this._bgChanged = true
 
@@ -662,14 +725,11 @@ export default class VideoStreamBackgroundEffect {
 		const { frameRate }
 			= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints()
 
-		this._frameRate = parseInt(frameRate, 10)
-
-		this._outputStream.getVideoTracks()[0].applyConstraints({ frameRate: this._frameRate }).catch((error) => {
-			console.error('Frame rate could not be adjusted in background effect', error)
-		})
+		this._updateFrameRate(frameRate)
 
 		this._frameId = -1
 		this._lastFrameId = -1
+		this._lastInferenceTime = 0
 	}
 
 	/**

src/utils/media/effects/virtual-background/WebGLCompositor.js

@@ -4,6 +4,8 @@
  */
 // @flow
 
+import { virtualBackgroundDebugConfig } from './runtimeConfig.js'
+
 /**
  * WebGL-based compositor for background effects.
  * Incorporates joint bilateral filtering and multi-pass blur for improved quality.
@@ -264,8 +266,6 @@ export default class WebGLCompositor {
 		this.blitSamplerLoc = null
 
 		// --- Default parameters ---
-		this.sigmaSpace = 10.0
-		this.sigmaColor = 0.15
 		this.coverage = [0.45, 0.75]
 		this.lightWrapping = 0.3
 		this.progressBarColor = [0, 0.4, 0.62, 1] // Nextcloud default primary color (#00679E)
@@ -468,11 +468,13 @@ export default class WebGLCompositor {
 		// Calculate filter parameters
 		const texelWidth = 1 / width
 		const texelHeight = 1 / height
-		const kSparsityFactor = 0.66
-		const step = Math.max(1, Math.sqrt(this.sigmaSpace) * kSparsityFactor)
-		const radius = this.sigmaSpace
+		const sigmaSpace = virtualBackgroundDebugConfig.SIGMA_SPACE
+		const sigmaColor = virtualBackgroundDebugConfig.SIGMA_COLOR
+		const sparsityFactor = virtualBackgroundDebugConfig.SPARSITY_FACTOR
+		const step = Math.max(1, Math.sqrt(sigmaSpace) * sparsityFactor)
+		const radius = sigmaSpace
 		const offset = step > 1 ? step * 0.5 : 0
-		const sigmaTexel = Math.max(texelWidth, texelHeight) * this.sigmaSpace
+		const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace
 
 		// Set uniforms
 		gl.uniform1i(gl.getUniformLocation(this.progBilateral, 'u_inputFrame'), 0)
@@ -482,7 +484,7 @@ export default class WebGLCompositor {
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_radius'), radius)
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_offset'), offset)
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaTexel'), sigmaTexel)
-		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaColor'), this.sigmaColor)
+		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaColor'), sigmaColor)
 
 		// Bind textures
 		gl.activeTexture(gl.TEXTURE0)
@@ -532,8 +534,9 @@ export default class WebGLCompositor {
 		gl.activeTexture(gl.TEXTURE1)
 		gl.bindTexture(gl.TEXTURE_2D, this.texMaskFiltered)
 
-		// Apply 3 blur passes
-		for (let i = 0; i < 3; i++) {
+		const blurPasses = virtualBackgroundDebugConfig.DEFAULT_BLUR_PASSES
+
+		for (let i = 0; i < blurPasses; i++) {
 			// Horizontal pass
 			gl.uniform2f(gl.getUniformLocation(this.progBlur, 'u_texelSize'), 0, texelHeight)
 			gl.bindFramebuffer(gl.FRAMEBUFFER, this.fboBlur1)