diff --git a/apps/desktop/src/utils/queries.ts b/apps/desktop/src/utils/queries.ts index 0cf0b815d8..a2b27fcb04 100644 --- a/apps/desktop/src/utils/queries.ts +++ b/apps/desktop/src/utils/queries.ts @@ -7,7 +7,7 @@ import { useQuery, } from "@tanstack/solid-query"; import { getCurrentWindow } from "@tauri-apps/api/window"; -import { createEffect, createMemo, onCleanup } from "solid-js"; +import { batch, createEffect, createMemo, onCleanup } from "solid-js"; import { createStore, reconcile } from "solid-js/store"; import { useRecordingOptions } from "~/routes/(window-chrome)/OptionsContext"; import { @@ -170,20 +170,29 @@ export function createOptionsQuery() { if (e.key === PERSIST_KEY) _setState(JSON.parse(e.newValue ?? "{}")); }); + let initialized = false; + + recordingSettingsStore.get().then((data) => { + batch(() => { + if (data?.mode && data.mode !== _state.mode) { + _setState("mode", data.mode); + } + initialized = true; + }); + }); + createEffect(() => { - recordingSettingsStore.set({ + const settings = { target: _state.captureTarget, micName: _state.micName, cameraId: _state.cameraID, mode: _state.mode, systemAudio: _state.captureSystemAudio, organizationId: _state.organizationId, - }); - }); + }; - recordingSettingsStore.get().then((data) => { - if (data?.mode && data.mode !== _state.mode) { - _setState("mode", data.mode); + if (initialized) { + recordingSettingsStore.set(settings); } }); diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index aba2f45f8f..b6fb1fcf99 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -1,6 +1,6 @@ use std::{thread, time::Duration}; -use cap_media_info::{Pixel, VideoInfo}; +use cap_media_info::{Pixel, VideoInfo, ensure_even}; use ffmpeg::{ Dictionary, codec::{codec::Codec, context, encoder}, @@ -90,15 +90,21 @@ impl H264EncoderBuilder { output: &mut format::context::Output, ) -> Result { let input_config = self.input_config; - let (output_width, output_height) = self + let (raw_width, raw_height) = self .output_size .unwrap_or((input_config.width, input_config.height)); - if output_width == 0 || output_height == 0 { - return Err(H264EncoderError::InvalidOutputDimensions { - width: output_width, - height: output_height, - }); + let output_width = ensure_even(raw_width); + let output_height = ensure_even(raw_height); + + if raw_width != output_width || raw_height != output_height { + warn!( + raw_width, + raw_height, + output_width, + output_height, + "Auto-adjusted odd dimensions to even for H264 encoding" + ); } let candidates = get_codec_and_options(&input_config, self.preset); diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rs index 14284e580b..37b555592f 100644 --- a/crates/enc-ffmpeg/src/video/hevc.rs +++ b/crates/enc-ffmpeg/src/video/hevc.rs @@ -1,6 +1,6 @@ use std::{thread, time::Duration}; -use cap_media_info::{Pixel, VideoInfo}; +use cap_media_info::{Pixel, VideoInfo, ensure_even}; use ffmpeg::{ Dictionary, codec::{codec::Codec, context, encoder}, @@ -9,7 +9,7 @@ use ffmpeg::{ frame, threading::Config, }; -use tracing::{debug, error, trace}; +use tracing::{debug, error, trace, warn}; use crate::base::EncoderBase; @@ -89,15 +89,21 @@ impl HevcEncoderBuilder { output: &mut format::context::Output, ) -> Result { let input_config = self.input_config; - let (output_width, output_height) = self + let (raw_width, raw_height) = self .output_size .unwrap_or((input_config.width, input_config.height)); - if output_width == 0 || output_height == 0 { - return Err(HevcEncoderError::InvalidOutputDimensions { - width: output_width, - height: output_height, - }); + let output_width = ensure_even(raw_width); + let output_height = ensure_even(raw_height); + + if raw_width != output_width || raw_height != output_height { + warn!( + raw_width, + raw_height, + output_width, + output_height, + "Auto-adjusted odd dimensions to even for HEVC encoding" + ); } let candidates = get_codec_and_options(&input_config, self.preset); diff --git a/crates/media-info/src/lib.rs b/crates/media-info/src/lib.rs index 71b1682999..76d6fb950e 100644 --- a/crates/media-info/src/lib.rs +++ b/crates/media-info/src/lib.rs @@ -325,6 +325,11 @@ impl VideoInfo { } } +pub fn ensure_even(value: u32) -> u32 { + let adjusted = value - (value % 2); + if adjusted == 0 { 2 } else { adjusted } +} + pub fn ffmpeg_sample_format_for(sample_format: SampleFormat) -> Option { match sample_format { SampleFormat::U8 => Some(Sample::U8(Type::Planar)), diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index f260f92e72..527af4ac88 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -5,6 +5,7 @@ use crate::{ }, feeds::microphone::MicrophoneFeedLock, output_pipeline::{self, OutputPipeline}, + resolution_limits::ensure_even, sources::screen_capture::{ScreenCaptureConfig, ScreenCaptureTarget}, }; use anyhow::Context as _; @@ -227,7 +228,12 @@ async fn create_pipeline( ), ) }) - .unwrap_or((screen_info.width, screen_info.height)); + .unwrap_or_else(|| { + ( + ensure_even(screen_info.width), + ensure_even(screen_info.height), + ) + }); let (screen_capture, system_audio) = screen_source.to_sources().await?; @@ -413,11 +419,6 @@ fn current_time_f64() -> f64 { .as_secs_f64() } -fn ensure_even(value: u32) -> u32 { - let adjusted = value - (value % 2); - if adjusted == 0 { 2 } else { adjusted } -} - fn clamp_size(input: (u32, u32), max: (u32, u32)) -> (u32, u32) { // 16/9-ish if input.0 >= input.1 && (input.0 as f64 / input.1 as f64) <= 16.0 / 9.0 { diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index f5db16cd33..e24fd3167f 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -32,6 +32,7 @@ const LARGE_FORWARD_JUMP_SECS: f64 = 5.0; struct AudioDriftTracker { baseline_offset_secs: Option, drift_warning_logged: bool, + first_frame_timestamp_secs: Option, } const AUDIO_WALL_CLOCK_TOLERANCE_SECS: f64 = 0.1; @@ -42,28 +43,38 @@ impl AudioDriftTracker { Self { baseline_offset_secs: None, drift_warning_logged: false, + first_frame_timestamp_secs: None, } } fn calculate_timestamp( &mut self, - samples_before_frame: u64, - sample_rate: u32, + frame_timestamp_secs: f64, wall_clock_secs: f64, - total_input_duration_secs: f64, ) -> Option { - let sample_time_secs = samples_before_frame as f64 / sample_rate as f64; + if frame_timestamp_secs < 0.0 { + return None; + } + + let first_timestamp = *self + .first_frame_timestamp_secs + .get_or_insert(frame_timestamp_secs); + let frame_elapsed_secs = frame_timestamp_secs - first_timestamp; + + if frame_elapsed_secs < 0.0 { + return None; + } - if sample_time_secs > wall_clock_secs + AUDIO_WALL_CLOCK_TOLERANCE_SECS { + if frame_elapsed_secs > wall_clock_secs + AUDIO_WALL_CLOCK_TOLERANCE_SECS { return None; } - if wall_clock_secs >= 2.0 && total_input_duration_secs >= 2.0 { + if wall_clock_secs >= 2.0 && frame_elapsed_secs >= 2.0 { if self.baseline_offset_secs.is_none() { - let offset = total_input_duration_secs - wall_clock_secs; + let offset = frame_elapsed_secs - wall_clock_secs; debug!( wall_clock_secs, - total_input_duration_secs, + frame_elapsed_secs, baseline_offset_secs = offset, "Capturing audio baseline offset after warmup" ); @@ -71,9 +82,9 @@ impl AudioDriftTracker { } let baseline = self.baseline_offset_secs.unwrap_or(0.0); - let adjusted_input_duration = total_input_duration_secs - baseline; - let drift_ratio = if adjusted_input_duration > 0.0 { - wall_clock_secs / adjusted_input_duration + let adjusted_frame_elapsed = frame_elapsed_secs - baseline; + let drift_ratio = if adjusted_frame_elapsed > 0.0 { + wall_clock_secs / adjusted_frame_elapsed } else { 1.0 }; @@ -82,15 +93,18 @@ impl AudioDriftTracker { warn!( drift_ratio, wall_clock_secs, - adjusted_input_duration, + adjusted_frame_elapsed, baseline, "Significant audio clock drift detected" ); self.drift_warning_logged = true; } + + let corrected_secs = adjusted_frame_elapsed * drift_ratio; + return Some(Duration::from_secs_f64(corrected_secs.max(0.0))); } - Some(Duration::from_secs_f64(sample_time_secs)) + Some(Duration::from_secs_f64(frame_elapsed_secs.max(0.0))) } } @@ -1056,7 +1070,6 @@ impl PreparedAudioSources { let _ = first_tx.send(frame.timestamp); } - let samples_before_frame = total_samples; let frame_samples = frame.inner.samples() as u64; total_samples += frame_samples; @@ -1066,21 +1079,24 @@ impl PreparedAudioSources { let effective_wall_clock = raw_wall_clock.saturating_sub(total_pause_duration); let wall_clock_secs = effective_wall_clock.as_secs_f64(); - let total_input_duration_secs = - total_samples as f64 / sample_rate as f64; + + let frame_timestamp_secs = + frame.timestamp.signed_duration_since_secs(timestamps); if wall_clock_secs >= 5.0 && (wall_clock_secs as u64).is_multiple_of(10) { - let drift_ratio = if total_input_duration_secs > 0.0 { - wall_clock_secs / total_input_duration_secs + let total_input_duration_secs = + total_samples as f64 / sample_rate as f64; + let drift_ratio = if frame_timestamp_secs > 0.0 { + wall_clock_secs / frame_timestamp_secs } else { 1.0 }; debug!( wall_clock_secs, total_input_duration_secs, + frame_timestamp_secs, drift_ratio, - samples_before_frame, total_samples, baseline_offset = drift_tracker.baseline_offset_secs, total_pause_ms = total_pause_duration.as_millis(), @@ -1088,12 +1104,8 @@ impl PreparedAudioSources { ); } - let timestamp = drift_tracker.calculate_timestamp( - samples_before_frame, - sample_rate, - wall_clock_secs, - total_input_duration_secs, - ); + let timestamp = drift_tracker + .calculate_timestamp(frame_timestamp_secs, wall_clock_secs); if let Some(timestamp) = timestamp && let Err(e) = @@ -1483,24 +1495,18 @@ mod tests { mod audio_drift_tracker { use super::*; - const SAMPLE_RATE: u32 = 48000; - - fn samples_for_duration(duration_secs: f64) -> u64 { - (duration_secs * SAMPLE_RATE as f64) as u64 - } - #[test] - fn returns_sample_based_time_during_warmup() { + fn returns_frame_based_time_during_warmup() { let mut tracker = AudioDriftTracker::new(); - let samples = samples_for_duration(1.0); + let frame_timestamp = 1.0; + let wall_clock = 1.5; let result = tracker - .calculate_timestamp(samples, SAMPLE_RATE, 1.5, 1.5) - .expect("Should not be capped when sample time < wall clock"); - let expected = Duration::from_secs_f64(1.0); + .calculate_timestamp(frame_timestamp, wall_clock) + .expect("Should not be capped when frame time < wall clock"); + let expected = Duration::ZERO; assert!( (result.as_secs_f64() - expected.as_secs_f64()).abs() < 0.001, - "Expected ~{:.3}s, got {:.3}s", - expected.as_secs_f64(), + "First frame should have ~0s timestamp, got {:.3}s", result.as_secs_f64() ); assert!( @@ -1513,11 +1519,12 @@ mod tests { fn captures_baseline_after_warmup() { let mut tracker = AudioDriftTracker::new(); let buffer_delay = 0.05; - let wall_clock = 2.0; - let input_duration = 2.0 + buffer_delay; - let samples = samples_for_duration(input_duration); - tracker.calculate_timestamp(samples, SAMPLE_RATE, wall_clock, input_duration); + tracker.calculate_timestamp(0.0, 0.0); + + let frame_timestamp = 2.0 + buffer_delay; + let wall_clock = 2.0; + tracker.calculate_timestamp(frame_timestamp, wall_clock); assert!(tracker.baseline_offset_secs.is_some()); let baseline = tracker.baseline_offset_secs.unwrap(); @@ -1528,30 +1535,26 @@ mod tests { } #[test] - fn returns_sample_based_time_after_warmup() { + fn applies_drift_correction_after_warmup() { let mut tracker = AudioDriftTracker::new(); let buffer_delay = 0.05; + tracker.calculate_timestamp(0.0, 0.0); + + let frame_timestamp_1 = 2.0 + buffer_delay; let wall_clock_1 = 2.0; - let input_duration_1 = 2.0 + buffer_delay; - tracker.calculate_timestamp( - samples_for_duration(input_duration_1), - SAMPLE_RATE, - wall_clock_1, - input_duration_1, - ); + tracker.calculate_timestamp(frame_timestamp_1, wall_clock_1); + let frame_timestamp_2 = 10.0 + buffer_delay; let wall_clock_2 = 10.0; - let input_duration_2 = 10.0 + buffer_delay; - let samples_2 = samples_for_duration(input_duration_2); let result = tracker - .calculate_timestamp(samples_2, SAMPLE_RATE, wall_clock_2, input_duration_2) + .calculate_timestamp(frame_timestamp_2, wall_clock_2) .expect("Should not be capped"); - let expected = Duration::from_secs_f64(input_duration_2); + let expected = Duration::from_secs_f64(wall_clock_2); assert!( - (result.as_secs_f64() - expected.as_secs_f64()).abs() < 0.001, - "Expected sample-based time ~{:.3}s, got {:.3}s", + (result.as_secs_f64() - expected.as_secs_f64()).abs() < 0.1, + "Expected drift-corrected time ~{:.3}s, got {:.3}s", expected.as_secs_f64(), result.as_secs_f64() ); @@ -1561,14 +1564,11 @@ mod tests { fn continuous_timestamps_no_gaps() { let mut tracker = AudioDriftTracker::new(); - let mut samples = 0u64; let mut last_timestamp = Duration::ZERO; for i in 0..100 { + let frame_timestamp = i as f64 * 0.02; let wall_clock = i as f64 * 0.02; - let input_duration = samples as f64 / SAMPLE_RATE as f64; - if let Some(result) = - tracker.calculate_timestamp(samples, SAMPLE_RATE, wall_clock, input_duration) - { + if let Some(result) = tracker.calculate_timestamp(frame_timestamp, wall_clock) { if i > 0 { let gap = result.as_secs_f64() - last_timestamp.as_secs_f64(); assert!( @@ -1579,7 +1579,6 @@ mod tests { last_timestamp = result; } - samples += 960; } } @@ -1587,62 +1586,53 @@ mod tests { fn continuous_across_warmup_boundary() { let mut tracker = AudioDriftTracker::new(); - let samples_at_2s = samples_for_duration(2.0); + tracker.calculate_timestamp(0.0, 0.0); + + let frame_timestamp_1 = 2.0; + let wall_clock_1 = 2.1; let result1 = tracker - .calculate_timestamp( - samples_at_2s, - SAMPLE_RATE, - 2.1, - samples_at_2s as f64 / SAMPLE_RATE as f64, - ) + .calculate_timestamp(frame_timestamp_1, wall_clock_1) .expect("Should not be capped"); - let samples_after = samples_at_2s + 960; + let frame_timestamp_2 = 2.02; + let wall_clock_2 = 2.12; let result2 = tracker - .calculate_timestamp( - samples_after, - SAMPLE_RATE, - 2.2, - samples_after as f64 / SAMPLE_RATE as f64, - ) + .calculate_timestamp(frame_timestamp_2, wall_clock_2) .expect("Should not be capped"); let gap = result2.as_secs_f64() - result1.as_secs_f64(); - let expected_gap = 960.0 / SAMPLE_RATE as f64; + let expected_gap = 0.02; assert!( - (gap - expected_gap).abs() < 0.001, + (gap - expected_gap).abs() < 0.01, "Gap across warmup boundary should be continuous: expected {expected_gap:.3}s, got {gap:.3}s" ); } #[test] - fn simulates_real_world_scenario() { + fn simulates_real_world_scenario_with_drift() { let mut tracker = AudioDriftTracker::new(); - let initial_buffer = 0.05; + let initial_offset = 0.05; let drift_rate = 0.004; - let mut total_audio = initial_buffer; + let mut frame_time = initial_offset; let mut wall_time = 0.0; let step = 0.5; while wall_time < 60.0 { - wall_time += step; - total_audio += step * (1.0 + drift_rate); - - let samples = samples_for_duration(total_audio); - if let Some(result) = - tracker.calculate_timestamp(samples, SAMPLE_RATE, wall_time, total_audio) - { - let expected = samples as f64 / SAMPLE_RATE as f64; - let error = (result.as_secs_f64() - expected).abs(); - assert!( - error < 0.001, - "At wall_time={:.1}s: result {:.3}s should equal sample time {:.3}s", - wall_time, - result.as_secs_f64(), - expected - ); + if let Some(result) = tracker.calculate_timestamp(frame_time, wall_time) { + if wall_time >= 2.0 { + let error = (result.as_secs_f64() - wall_time).abs(); + assert!( + error < 0.5, + "At wall_time={:.1}s: result {:.3}s should be close to wall clock", + wall_time, + result.as_secs_f64() + ); + } } + + wall_time += step; + frame_time += step * (1.0 + drift_rate); } } @@ -1650,17 +1640,36 @@ mod tests { fn preserves_baseline_across_multiple_calls() { let mut tracker = AudioDriftTracker::new(); - tracker.calculate_timestamp(samples_for_duration(2.1), SAMPLE_RATE, 2.0, 2.1); + tracker.calculate_timestamp(0.0, 0.0); + tracker.calculate_timestamp(2.1, 2.0); let first_baseline = tracker.baseline_offset_secs; - tracker.calculate_timestamp(samples_for_duration(10.1), SAMPLE_RATE, 10.0, 10.1); + tracker.calculate_timestamp(10.1, 10.0); assert_eq!( first_baseline, tracker.baseline_offset_secs, "Baseline should not change after initial capture" ); } + + #[test] + fn rejects_negative_timestamps() { + let mut tracker = AudioDriftTracker::new(); + let result = tracker.calculate_timestamp(-1.0, 1.0); + assert!(result.is_none(), "Negative timestamps should be rejected"); + } + + #[test] + fn rejects_timestamps_too_far_ahead_of_wall_clock() { + let mut tracker = AudioDriftTracker::new(); + tracker.calculate_timestamp(0.0, 0.0); + let result = tracker.calculate_timestamp(5.0, 1.0); + assert!( + result.is_none(), + "Timestamps too far ahead of wall clock should be rejected" + ); + } } mod video_drift_tracker { diff --git a/crates/recording/src/resolution_limits.rs b/crates/recording/src/resolution_limits.rs index 309ff737f6..33af77c071 100644 --- a/crates/recording/src/resolution_limits.rs +++ b/crates/recording/src/resolution_limits.rs @@ -1,3 +1,5 @@ +pub use cap_media_info::ensure_even; + pub const H264_MAX_DIMENSION: u32 = 4096; pub fn calculate_gpu_compatible_size( @@ -5,10 +7,17 @@ pub fn calculate_gpu_compatible_size( height: u32, max_dimension: u32, ) -> Option<(u32, u32)> { - if width <= max_dimension && height <= max_dimension { + let needs_downscale = width > max_dimension || height > max_dimension; + let needs_even_adjustment = !width.is_multiple_of(2) || !height.is_multiple_of(2); + + if !needs_downscale && !needs_even_adjustment { return None; } + if !needs_downscale { + return Some((ensure_even(width), ensure_even(height))); + } + let aspect_ratio = width as f64 / height as f64; let (target_width, target_height) = if width >= height { @@ -50,19 +59,44 @@ pub fn calculate_gpu_compatible_size( Some((final_width, final_height)) } -fn ensure_even(value: u32) -> u32 { - let adjusted = value - (value % 2); - if adjusted == 0 { 2 } else { adjusted } -} - #[cfg(test)] mod tests { use super::*; + fn assert_valid_output(result: Option<(u32, u32)>, max_dim: u32) { + if let Some((w, h)) = result { + assert!(w <= max_dim, "Width {} exceeds max {}", w, max_dim); + assert!(h <= max_dim, "Height {} exceeds max {}", h, max_dim); + assert_eq!(w % 2, 0, "Width {} is odd", w); + assert_eq!(h % 2, 0, "Height {} is odd", h); + assert!(w >= 2, "Width {} is too small", w); + assert!(h >= 2, "Height {} is too small", h); + } + } + + fn assert_aspect_preserved( + original_w: u32, + original_h: u32, + result_w: u32, + result_h: u32, + tolerance: f64, + ) { + let original_aspect = original_w as f64 / original_h as f64; + let result_aspect = result_w as f64 / result_h as f64; + assert!( + (original_aspect - result_aspect).abs() < tolerance, + "Aspect ratio changed too much: {} -> {}", + original_aspect, + result_aspect + ); + } + #[test] - fn test_below_limits_returns_none() { - assert_eq!(calculate_gpu_compatible_size(3840, 2160, 4096), None); + fn test_standard_resolutions_even_return_none() { + assert_eq!(calculate_gpu_compatible_size(1280, 720, 4096), None); assert_eq!(calculate_gpu_compatible_size(1920, 1080, 4096), None); + assert_eq!(calculate_gpu_compatible_size(2560, 1440, 4096), None); + assert_eq!(calculate_gpu_compatible_size(3840, 2160, 4096), None); } #[test] @@ -70,6 +104,45 @@ mod tests { assert_eq!(calculate_gpu_compatible_size(4096, 4096, 4096), None); assert_eq!(calculate_gpu_compatible_size(4096, 2160, 4096), None); assert_eq!(calculate_gpu_compatible_size(3840, 4096, 4096), None); + assert_eq!(calculate_gpu_compatible_size(4096, 2304, 4096), None); + } + + #[test] + fn test_ultrawide_21_9_resolutions() { + assert_eq!(calculate_gpu_compatible_size(2560, 1080, 4096), None); + assert_eq!(calculate_gpu_compatible_size(3440, 1440, 4096), None); + + let result = calculate_gpu_compatible_size(5120, 2160, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_valid_output(result, 4096); + assert_aspect_preserved(5120, 2160, w, h, 0.02); + } + + #[test] + fn test_super_ultrawide_32_9_resolutions() { + assert_eq!(calculate_gpu_compatible_size(3840, 1080, 4096), None); + + let result = calculate_gpu_compatible_size(5120, 1440, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(7680, 2160, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + } + + #[test] + fn test_legacy_4_3_resolutions() { + assert_eq!(calculate_gpu_compatible_size(1024, 768, 4096), None); + assert_eq!(calculate_gpu_compatible_size(1600, 1200, 4096), None); + assert_eq!(calculate_gpu_compatible_size(2048, 1536, 4096), None); + } + + #[test] + fn test_legacy_5_4_resolutions() { + assert_eq!(calculate_gpu_compatible_size(1280, 1024, 4096), None); + assert_eq!(calculate_gpu_compatible_size(2560, 2048, 4096), None); } #[test] @@ -77,46 +150,126 @@ mod tests { let result = calculate_gpu_compatible_size(5120, 2880, 4096); assert!(result.is_some()); let (w, h) = result.unwrap(); - assert!(w <= 4096); - assert!(h <= 4096); - assert_eq!(w % 2, 0); - assert_eq!(h % 2, 0); - let original_aspect = 5120.0 / 2880.0; - let result_aspect = w as f64 / h as f64; - assert!((original_aspect - result_aspect).abs() < 0.02); + assert_valid_output(result, 4096); + assert_aspect_preserved(5120, 2880, w, h, 0.02); } #[test] - fn test_5k_ultrawide_downscales() { - let result = calculate_gpu_compatible_size(5120, 2160, 4096); + fn test_8k_resolution_downscales() { + let result = calculate_gpu_compatible_size(7680, 4320, 4096); assert!(result.is_some()); let (w, h) = result.unwrap(); - assert!(w <= 4096); - assert!(h <= 4096); - assert_eq!(w % 2, 0); - assert_eq!(h % 2, 0); + assert_valid_output(result, 4096); + assert_aspect_preserved(7680, 4320, w, h, 0.02); + } + + #[test] + fn test_portrait_standard_resolutions() { + assert_eq!(calculate_gpu_compatible_size(1080, 1920, 4096), None); + assert_eq!(calculate_gpu_compatible_size(1440, 2560, 4096), None); + assert_eq!(calculate_gpu_compatible_size(2160, 3840, 4096), None); } #[test] - fn test_portrait_mode_downscales() { + fn test_portrait_exceeding_limit() { let result = calculate_gpu_compatible_size(2880, 5120, 4096); assert!(result.is_some()); let (w, h) = result.unwrap(); - assert!(w <= 4096); - assert!(h <= 4096); - assert_eq!(w % 2, 0); - assert_eq!(h % 2, 0); + assert_valid_output(result, 4096); + assert_aspect_preserved(2880, 5120, w, h, 0.02); + + let result = calculate_gpu_compatible_size(4320, 7680, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); } #[test] - fn test_extreme_resolution() { - let result = calculate_gpu_compatible_size(7680, 4320, 4096); + fn test_window_capture_odd_height() { + let result = calculate_gpu_compatible_size(2560, 1055, 4096); assert!(result.is_some()); let (w, h) = result.unwrap(); - assert!(w <= 4096); - assert!(h <= 4096); - assert_eq!(w % 2, 0); - assert_eq!(h % 2, 0); + assert_eq!(w, 2560); + assert_eq!(h, 1054); + assert_valid_output(result, 4096); + } + + #[test] + fn test_window_capture_odd_width() { + let result = calculate_gpu_compatible_size(1921, 1080, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 1920); + assert_eq!(h, 1080); + } + + #[test] + fn test_window_capture_both_odd() { + let result = calculate_gpu_compatible_size(1921, 1081, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 1920); + assert_eq!(h, 1080); + } + + #[test] + fn test_window_capture_common_browser_sizes() { + let browser_sizes = [ + (1903, 969), + (1423, 800), + (1279, 719), + (800, 600), + (1024, 768), + (1366, 768), + (1536, 864), + ]; + + for (w, h) in browser_sizes { + let result = calculate_gpu_compatible_size(w, h, 4096); + if w % 2 != 0 || h % 2 != 0 { + assert!(result.is_some(), "Should adjust odd dimensions {}x{}", w, h); + } + assert_valid_output(result, 4096); + } + } + + #[test] + fn test_small_window_captures() { + let small_sizes = [(100, 100), (50, 50), (200, 150), (320, 240), (640, 480)]; + + for (w, h) in small_sizes { + let result = calculate_gpu_compatible_size(w, h, 4096); + assert_valid_output(result, 4096); + } + } + + #[test] + fn test_small_odd_dimensions() { + let result = calculate_gpu_compatible_size(101, 101, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 100); + assert_eq!(h, 100); + + let result = calculate_gpu_compatible_size(51, 33, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 50); + assert_eq!(h, 32); + } + + #[test] + fn test_very_small_dimensions() { + let result = calculate_gpu_compatible_size(3, 3, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 2); + assert_eq!(h, 2); + + let result = calculate_gpu_compatible_size(1, 1, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 2); + assert_eq!(h, 2); } #[test] @@ -125,5 +278,250 @@ mod tests { assert_eq!(ensure_even(101), 100); assert_eq!(ensure_even(1), 2); assert_eq!(ensure_even(0), 2); + assert_eq!(ensure_even(2), 2); + assert_eq!(ensure_even(3), 2); + assert_eq!(ensure_even(4095), 4094); + assert_eq!(ensure_even(4096), 4096); + } + + #[test] + fn test_just_over_limit() { + let result = calculate_gpu_compatible_size(4097, 4097, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_valid_output(result, 4096); + assert!(w <= 4096); + assert!(h <= 4096); + } + + #[test] + fn test_one_dimension_over_limit() { + let result = calculate_gpu_compatible_size(4097, 2160, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(3840, 4097, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + } + + #[test] + fn test_one_dimension_at_limit_other_odd() { + let result = calculate_gpu_compatible_size(4096, 2161, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 4096); + assert_eq!(h, 2160); + + let result = calculate_gpu_compatible_size(4095, 2160, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 4094); + assert_eq!(h, 2160); + } + + #[test] + fn test_maximum_odd_dimensions_under_limit() { + let result = calculate_gpu_compatible_size(4095, 4095, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 4094); + assert_eq!(h, 4094); + } + + #[test] + fn test_tall_narrow_window() { + let result = calculate_gpu_compatible_size(400, 1200, 4096); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(401, 1201, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 400); + assert_eq!(h, 1200); + } + + #[test] + fn test_wide_short_window() { + let result = calculate_gpu_compatible_size(2000, 300, 4096); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(2001, 301, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!(w, 2000); + assert_eq!(h, 300); + } + + #[test] + fn test_extreme_aspect_ratios() { + let result = calculate_gpu_compatible_size(100, 4000, 4096); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(4000, 100, 4096); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(101, 4001, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + } + + #[test] + fn test_extreme_aspect_ratios_over_limit() { + let result = calculate_gpu_compatible_size(100, 5000, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + + let result = calculate_gpu_compatible_size(5000, 100, 4096); + assert!(result.is_some()); + assert_valid_output(result, 4096); + } + + #[test] + fn test_real_world_window_sizes() { + let window_sizes = [ + (1920, 1055), + (2560, 1055), + (1280, 800), + (1440, 900), + (1680, 1050), + (1920, 1200), + (2560, 1600), + (1366, 768), + (1536, 864), + (1600, 900), + ]; + + for (w, h) in window_sizes { + let result = calculate_gpu_compatible_size(w, h, 4096); + assert_valid_output(result, 4096); + if w % 2 == 0 && h % 2 == 0 && w <= 4096 && h <= 4096 { + assert_eq!( + result, None, + "Even dimensions {}x{} should return None", + w, h + ); + } + } + } + + #[test] + fn test_mixed_odd_even_under_limit() { + let cases = [ + (1920, 1081, 1920, 1080), + (1919, 1080, 1918, 1080), + (2559, 1439, 2558, 1438), + (3839, 2159, 3838, 2158), + ]; + + for (in_w, in_h, exp_w, exp_h) in cases { + let result = calculate_gpu_compatible_size(in_w, in_h, 4096); + assert!(result.is_some()); + let (w, h) = result.unwrap(); + assert_eq!( + w, exp_w, + "Expected width {} for input {}x{}", + exp_w, in_w, in_h + ); + assert_eq!( + h, exp_h, + "Expected height {} for input {}x{}", + exp_h, in_w, in_h + ); + } + } + + #[test] + fn test_different_max_dimensions() { + let result = calculate_gpu_compatible_size(2560, 1440, 1920); + assert!(result.is_some()); + assert_valid_output(result, 1920); + + let result = calculate_gpu_compatible_size(1920, 1080, 1280); + assert!(result.is_some()); + assert_valid_output(result, 1280); + + let result = calculate_gpu_compatible_size(1280, 720, 8192); + assert_eq!(result, None); + } + + #[test] + fn test_all_standard_resolutions_comprehensive() { + let standard_resolutions = [ + (640, 480), + (800, 600), + (1024, 768), + (1152, 864), + (1280, 720), + (1280, 768), + (1280, 800), + (1280, 960), + (1280, 1024), + (1360, 768), + (1366, 768), + (1400, 1050), + (1440, 900), + (1600, 900), + (1600, 1024), + (1600, 1200), + (1680, 1050), + (1920, 1080), + (1920, 1200), + (2048, 1152), + (2048, 1536), + (2560, 1080), + (2560, 1440), + (2560, 1600), + (2560, 2048), + (3440, 1440), + (3840, 1080), + (3840, 1600), + (3840, 2160), + (4096, 2160), + (4096, 2304), + (5120, 1440), + (5120, 2160), + (5120, 2880), + (7680, 2160), + (7680, 4320), + ]; + + for (w, h) in standard_resolutions { + let result = calculate_gpu_compatible_size(w, h, 4096); + assert_valid_output(result, 4096); + + if w <= 4096 && h <= 4096 && w % 2 == 0 && h % 2 == 0 { + assert_eq!( + result, None, + "Standard resolution {}x{} should return None", + w, h + ); + } else if w > 4096 || h > 4096 { + assert!( + result.is_some(), + "Over-limit resolution {}x{} should downscale", + w, + h + ); + } + } + } + + #[test] + fn test_downscaled_output_preserves_aspect_ratio() { + let test_cases = [ + (5120, 2880), + (7680, 4320), + (5120, 2160), + (6016, 3384), + (5120, 1440), + ]; + + for (w, h) in test_cases { + let result = calculate_gpu_compatible_size(w, h, 4096); + assert!(result.is_some()); + let (out_w, out_h) = result.unwrap(); + assert_aspect_preserved(w, h, out_w, out_h, 0.03); + } } } diff --git a/crates/recording/src/sources/audio_mixer.rs b/crates/recording/src/sources/audio_mixer.rs index 1dc52e5b5b..44db8537de 100644 --- a/crates/recording/src/sources/audio_mixer.rs +++ b/crates/recording/src/sources/audio_mixer.rs @@ -32,7 +32,6 @@ struct MixerSource { buffer_timeout: Duration, buffer: VecDeque, buffer_last: Option<(Timestamp, Duration)>, - input_duration_secs: f64, last_input_timestamp: Option, } @@ -66,7 +65,6 @@ impl AudioMixerBuilder { buffer_timeout, buffer: VecDeque::new(), buffer_last: None, - input_duration_secs: 0.0, last_input_timestamp: None, }); } @@ -167,6 +165,7 @@ impl AudioMixerBuilder { timestamps: Timestamps::now(), max_buffer_timeout, wall_clock_start: None, + baseline_offset_secs: None, }) } @@ -228,6 +227,7 @@ pub struct AudioMixer { start_timestamp: Option, max_buffer_timeout: Duration, wall_clock_start: Option, + baseline_offset_secs: Option, } impl AudioMixer { @@ -238,47 +238,78 @@ impl AudioMixer { ); fn calculate_drift_corrected_timestamp( - &self, + &mut self, start_timestamp: Timestamp, - nominal_output_rate: f64, wall_clock_elapsed: Duration, ) -> Timestamp { - let nominal_elapsed_secs = self.samples_out as f64 / nominal_output_rate; - let nominal_elapsed = Duration::from_secs_f64(nominal_elapsed_secs); - - let mut max_input_duration_secs: f64 = 0.0; + let mut latest_input_timestamp_secs: Option = None; for source in &self.sources { - if source.input_duration_secs > max_input_duration_secs { - max_input_duration_secs = source.input_duration_secs; + if let Some(ts) = source.last_input_timestamp { + let ts_secs = ts.signed_duration_since_secs(self.timestamps); + match latest_input_timestamp_secs { + Some(current) if ts_secs > current => { + latest_input_timestamp_secs = Some(ts_secs); + } + None => { + latest_input_timestamp_secs = Some(ts_secs); + } + _ => {} + } } } + let start_secs = start_timestamp.signed_duration_since_secs(self.timestamps); let wall_clock_secs = wall_clock_elapsed.as_secs_f64(); - if max_input_duration_secs < 0.5 || wall_clock_secs < 0.5 { - return start_timestamp + nominal_elapsed; + let Some(latest_secs) = latest_input_timestamp_secs else { + return start_timestamp; + }; + + let input_elapsed_secs = latest_secs - start_secs; + + if input_elapsed_secs < 0.0 { + return start_timestamp; } - if max_input_duration_secs <= 0.0 { - return start_timestamp + nominal_elapsed; + if input_elapsed_secs < 2.0 || wall_clock_secs < 2.0 { + return start_timestamp + Duration::from_secs_f64(input_elapsed_secs.max(0.0)); } - let drift_ratio = wall_clock_secs / max_input_duration_secs; + if self.baseline_offset_secs.is_none() { + let offset = input_elapsed_secs - wall_clock_secs; + debug!( + wall_clock_secs, + input_elapsed_secs, + baseline_offset_secs = offset, + "AudioMixer: Capturing baseline offset after warmup" + ); + self.baseline_offset_secs = Some(offset); + } - if !(0.95..=1.05).contains(&drift_ratio) { + let baseline = self.baseline_offset_secs.unwrap_or(0.0); + let adjusted_input_elapsed = input_elapsed_secs - baseline; + + let drift_ratio = if adjusted_input_elapsed > 0.0 { + wall_clock_secs / adjusted_input_elapsed + } else { + 1.0 + }; + + if !(0.90..=1.10).contains(&drift_ratio) { warn!( drift_ratio, wall_clock_secs, - max_input_duration_secs, - "Extreme audio clock drift detected, clamping" + adjusted_input_elapsed, + baseline, + "AudioMixer: Significant clock drift detected" ); let clamped_ratio = drift_ratio.clamp(0.95, 1.05); - let corrected_secs = nominal_elapsed_secs * clamped_ratio; - return start_timestamp + Duration::from_secs_f64(corrected_secs); + let corrected_secs = adjusted_input_elapsed * clamped_ratio; + return start_timestamp + Duration::from_secs_f64(corrected_secs.max(0.0)); } - let corrected_secs = nominal_elapsed_secs * drift_ratio; - start_timestamp + Duration::from_secs_f64(corrected_secs) + let corrected_secs = adjusted_input_elapsed * drift_ratio; + start_timestamp + Duration::from_secs_f64(corrected_secs.max(0.0)) } fn buffer_sources(&mut self, now: Timestamp) { @@ -294,10 +325,6 @@ impl AudioMixer { timestamp, })) = source.rx.try_next() { - let source_rate = source.info.rate() as f64; - if source_rate > 0.0 { - source.input_duration_secs += frame.samples() as f64 / source_rate; - } source.last_input_timestamp = Some(timestamp); if let Some((buffer_last_timestamp, buffer_last_duration)) = source.buffer_last { @@ -431,15 +458,11 @@ impl AudioMixer { let mut filtered = ffmpeg::frame::Audio::empty(); while self.abuffersink.sink().frame(&mut filtered).is_ok() { let output_rate_i32 = Self::INFO.rate(); - let output_rate = output_rate_i32 as f64; filtered.set_rate(output_rate_i32 as u32); - let output_timestamp = self.calculate_drift_corrected_timestamp( - start_timestamp, - output_rate, - wall_clock_elapsed, - ); + let output_timestamp = + self.calculate_drift_corrected_timestamp(start_timestamp, wall_clock_elapsed); let frame_samples = filtered.samples(); let mut frame = AudioFrame::new(filtered, output_timestamp);