Add clip speed editing

apps/desktop/src-tauri/src/lib.rs

@@ -1747,7 +1747,7 @@ async fn seek_to(editor_instance: WindowEditorInstance, frame_number: u32) -> Re
 async fn get_mic_waveforms(editor_instance: WindowEditorInstance) -> Result<Vec<Vec<f32>>, String> {
     let mut out = Vec::new();
 
-    for segment in editor_instance.segments.iter() {
+    for segment in editor_instance.segment_medias.iter() {
         if let Some(audio) = &segment.audio {
             out.push(audio::get_waveform(audio));
         } else {
@@ -1766,7 +1766,7 @@ async fn get_system_audio_waveforms(
 ) -> Result<Vec<Vec<f32>>, String> {
     let mut out = Vec::new();
 
-    for segment in editor_instance.segments.iter() {
+    for segment in editor_instance.segment_medias.iter() {
         if let Some(audio) = &segment.system_audio {
             out.push(audio::get_waveform(audio));
         } else {

apps/desktop/src-tauri/src/recording.rs

@@ -1307,7 +1307,7 @@ fn project_config_from_recording(
         .iter()
         .enumerate()
         .map(|(i, segment)| TimelineSegment {
-            recording_segment: i as u32,
+            recording_clip: i as u32,
             start: 0.0,
             end: segment.duration(),
             timescale: 1.0,

apps/desktop/src/routes/editor/ConfigSidebar.tsx

@@ -2538,7 +2538,41 @@ function ClipSegmentConfig(props: {
 				</EditorButton>
 			</div>
 
-			<div class="space-y-1">
+			<div class="space-y-0.5">
+				<h3 class="font-medium text-gray-12">Segment Settings</h3>
+				<p class="text-gray-11">
+					These settings apply to only the selected segment
+				</p>
+			</div>
+
+			<Field name="Speed" icon={<IconLucideFastForward class="size-4" />}>
+				<p class="text-gray-11 -mt-3">
+					Modifying speed will mute this segment's audio.
+				</p>
+
+				<KRadioGroup
+					class="flex flex-row gap-1.5 -mt-1"
+					value={props.segment.timescale.toString()}
+					onChange={(v) => {
+						projectActions.setClipSegmentTimescale(
+							props.segmentIndex,
+							parseFloat(v),
+						);
+					}}
+				>
+					<For each={[0.25, 0.5, 1, 1.5, 2, 4, 8]}>
+						{(mult) => (
+							<KRadioGroup.Item value={mult.toString()}>
+								<KRadioGroup.ItemControl class="px-2 py-1 text-gray-11 hover:text-gray-12 bg-gray-1 border border-gray-3 rounded-md ui-checked:bg-gray-3 ui-checked:border-gray-4 ui-checked:text-gray-12">
+									{mult}x
+								</KRadioGroup.ItemControl>
+							</KRadioGroup.Item>
+						)}
+					</For>
+				</KRadioGroup>
+			</Field>
+
+			<div class="space-y-0.5 pt-2">
 				<h3 class="font-medium text-gray-12">Clip Settings</h3>
 				<p class="text-gray-11">
 					These settings apply to all segments for the current clip

apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx

@@ -224,14 +224,13 @@ export function ClipTrack(
 
 					const relativeSegment = createMemo(() => {
 						const ds = startHandleDrag();
-						const offset = ds ? ds.offset / segment.timescale : 0;
+						const offset = ds?.offset ?? 0;
 
 						return {
 							start: Math.max(prevDuration() + offset, 0),
 							end:
 								prevDuration() +
-								offset +
-								(segment.end - segment.start) / segment.timescale,
+								(offset + (segment.end - segment.start)) / segment.timescale,
 							timescale: segment.timescale,
 							recordingSegment: segment.recordingSegment,
 						};
@@ -296,9 +295,7 @@ export function ClipTrack(
 									<div
 										class="absolute w-0 z-10 h-full *:absolute"
 										style={{
-											transform: `translateX(${
-												i() === 0 ? segmentX() : segmentX()
-											}px)`,
+											transform: `translateX(${segmentX()}px)`,
 										}}
 									>
 										<div class="w-[2px] bottom-0 -top-2 rounded-full from-red-300 to-transparent bg-gradient-to-b -translate-x-1/2" />
@@ -475,12 +472,14 @@ export function ClipTrack(
 									}
 								}}
 							>
-								<WaveformCanvas
-									micWaveform={micWaveform()}
-									systemWaveform={systemAudioWaveform()}
-									segment={segment}
-									secsPerPixel={secsPerPixel()}
-								/>
+								{segment.timescale === 1 && (
+									<WaveformCanvas
+										micWaveform={micWaveform()}
+										systemWaveform={systemAudioWaveform()}
+										segment={segment}
+										secsPerPixel={secsPerPixel()}
+									/>
+								)}
 
 								<Markings segment={segment} prevDuration={prevDuration()} />
 
@@ -590,6 +589,11 @@ export function ClipTrack(
 													<div class="flex gap-1 items-center text-md dark:text-gray-12 text-gray-1">
 														<IconLucideClock class="size-3.5" />{" "}
 														{formatTime(segment.end - segment.start)}
+														<Show when={segment.timescale !== 1}>
+															<div class="w-0.5" />
+															<IconLucideFastForward class="size-3" />
+															{segment.timescale}x
+														</Show>
 													</div>
 												</div>
 											</Show>

apps/desktop/src/routes/editor/context.ts

@@ -200,6 +200,43 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider(
 					setEditorState("timeline", "selection", null);
 				});
 			},
+			setClipSegmentTimescale: (index: number, timescale: number) => {
+				setProject(
+					produce((project) => {
+						const timeline = project.timeline;
+						if (!timeline) return;
+
+						const segment = timeline.segments[index];
+						if (!segment) return;
+
+						const currentLength =
+							(segment.end - segment.start) / segment.timescale;
+						const nextLength = (segment.end - segment.start) / timescale;
+
+						const lengthDiff = nextLength - currentLength;
+
+						const absoluteStart = timeline.segments.reduce((acc, curr, i) => {
+							if (i >= index) return acc;
+							return acc + (curr.end - curr.start) / curr.timescale;
+						}, 0);
+
+						const diff = (v: number) => {
+							const diff = (lengthDiff * (v - absoluteStart)) / currentLength;
+
+							if (v > absoluteStart + currentLength) return lengthDiff;
+							else if (v > absoluteStart) return diff;
+							else return 0;
+						};
+
+						for (const zoomSegment of timeline.zoomSegments) {
+							zoomSegment.start += diff(zoomSegment.start);
+							zoomSegment.end += diff(zoomSegment.end);
+						}
+
+						segment.timescale = timescale;
+					}),
+				);
+			},
 		};
 
 		createEffect(

crates/editor/src/audio.rs

@@ -21,7 +21,8 @@ pub struct AudioRenderer {
 
 #[derive(Clone, Copy, Debug)]
 pub struct AudioRendererCursor {
-    segment_index: u32,
+    clip_index: u32,
+    timescale: f64,
     // excludes channels
     samples: usize,
 }
@@ -85,8 +86,9 @@ impl AudioRenderer {
         Self {
             data,
             cursor: AudioRendererCursor {
-                segment_index: 0,
+                clip_index: 0,
                 samples: 0,
+                timescale: 1.0,
             },
             elapsed_samples: 0,
         }
@@ -96,12 +98,14 @@ impl AudioRenderer {
         self.elapsed_samples = self.playhead_to_samples(playhead);
 
         self.cursor = match project.get_segment_time(playhead) {
-            Some((segment_time, segment_i)) => AudioRendererCursor {
-                segment_index: segment_i,
+            Some((segment_time, segment)) => AudioRendererCursor {
+                clip_index: segment.recording_clip,
+                timescale: segment.timescale,
                 samples: self.playhead_to_samples(segment_time),
             },
             None => AudioRendererCursor {
-                segment_index: 0,
+                clip_index: 0,
+                timescale: 1.0,
                 samples: self.elapsed_samples,
             },
         };
@@ -115,18 +119,20 @@ impl AudioRenderer {
         // (corresponding to a trim or split point). Currently this change is at least 0.2 seconds
         // - not sure we offer that much precision in the editor even!
         let new_cursor = match timeline.get_segment_time(playhead) {
-            Some((segment_time, segment_i)) => AudioRendererCursor {
-                segment_index: segment_i,
+            Some((segment_time, segment)) => AudioRendererCursor {
+                clip_index: segment.recording_clip,
+                timescale: segment.timescale,
                 samples: self.playhead_to_samples(segment_time),
             },
             None => AudioRendererCursor {
-                segment_index: 0,
+                clip_index: 0,
+                timescale: 1.0,
                 samples: 0,
             },
         };
 
         let cursor_diff = new_cursor.samples as isize - self.cursor.samples as isize;
-        if new_cursor.segment_index != self.cursor.segment_index
+        if new_cursor.clip_index != self.cursor.clip_index
             || cursor_diff.unsigned_abs() > (AudioData::SAMPLE_RATE as usize) / 5
         {
             self.cursor = new_cursor;
@@ -168,7 +174,11 @@ impl AudioRenderer {
         }
         let channels: usize = 2;
 
-        let tracks = &self.data[self.cursor.segment_index as usize].tracks;
+        if self.cursor.timescale != 1.0 {
+            return None;
+        };
+
+        let tracks = &self.data[self.cursor.clip_index as usize].tracks;
 
         if tracks.is_empty() {
             return None;
@@ -197,7 +207,7 @@ impl AudioRenderer {
                 let offsets = project
                     .clips
                     .iter()
-                    .find(|c| c.index == start.segment_index)
+                    .find(|c| c.index == start.clip_index)
                     .map(|c| c.offsets)
                     .unwrap_or_default();
 

crates/editor/src/editor_instance.rs

@@ -26,7 +26,7 @@ pub struct EditorInstance {
         watch::Receiver<ProjectConfiguration>,
     ),
     // ws_shutdown_token: CancellationToken,
-    pub segments: Arc<Vec<Segment>>,
+    pub segment_medias: Arc<Vec<SegmentMedia>>,
     meta: RecordingMeta,
 }
 
@@ -81,7 +81,7 @@ impl EditorInstance {
             on_state_change: Box::new(on_state_change),
             preview_tx,
             project_config: watch::channel(project),
-            segments: Arc::new(segments),
+            segment_medias: Arc::new(segments),
             meta: recording_meta,
         });
 
@@ -146,7 +146,7 @@ impl EditorInstance {
             let start_frame_number = state.playhead_position;
 
             let playback_handle = match (playback::Playback {
-                segments: self.segments.clone(),
+                segment_medias: self.segment_medias.clone(),
                 renderer: self.renderer.clone(),
                 render_constants: self.render_constants.clone(),
                 start_frame_number,
@@ -207,17 +207,20 @@ impl EditorInstance {
 
                 let project = self.project_config.1.borrow().clone();
 
-                let Some((segment_time, segment_i)) =
+                let Some((segment_time, segment)) =
                     project.get_segment_time(frame_number as f64 / fps as f64)
                 else {
                     continue;
                 };
 
-                let segment = &self.segments[segment_i as usize];
-                let clip_config = project.clips.iter().find(|v| v.index == segment_i);
+                let segment_medias = &self.segment_medias[segment.recording_clip as usize];
+                let clip_config = project
+                    .clips
+                    .iter()
+                    .find(|v| v.index == segment.recording_clip);
                 let clip_offsets = clip_config.map(|v| v.offsets).unwrap_or_default();
 
-                if let Some(segment_frames) = segment
+                if let Some(segment_frames) = segment_medias
                     .decoders
                     .get_frames(segment_time as f32, !project.camera.hide, clip_offsets)
                     .await
@@ -228,11 +231,11 @@ impl EditorInstance {
                         frame_number,
                         fps,
                         resolution_base,
-                        &segment.cursor,
+                        &segment_medias.cursor,
                         &segment_frames,
                     );
                     self.renderer
-                        .render_frame(segment_frames, uniforms, segment.cursor.clone())
+                        .render_frame(segment_frames, uniforms, segment_medias.cursor.clone())
                         .await;
                 }
             }
@@ -278,7 +281,7 @@ pub struct EditorState {
     pub preview_task: Option<tokio::task::JoinHandle<()>>,
 }
 
-pub struct Segment {
+pub struct SegmentMedia {
     pub audio: Option<Arc<AudioData>>,
     pub system_audio: Option<Arc<AudioData>>,
     pub cursor: Arc<CursorEvents>,
@@ -288,7 +291,7 @@ pub struct Segment {
 pub async fn create_segments(
     recording_meta: &RecordingMeta,
     meta: &StudioRecordingMeta,
-) -> Result<Vec<Segment>, String> {
+) -> Result<Vec<SegmentMedia>, String> {
     match &meta {
         cap_project::StudioRecordingMeta::SingleSegment { segment: s } => {
             let audio = s
@@ -313,7 +316,7 @@ pub async fn create_segments(
             .await
             .map_err(|e| format!("SingleSegment / {e}"))?;
 
-            Ok(vec![Segment {
+            Ok(vec![SegmentMedia {
                 audio,
                 system_audio: None,
                 cursor: Default::default(),
@@ -358,7 +361,7 @@ pub async fn create_segments(
                 .await
                 .map_err(|e| format!("MultipleSegments {i} / {e}"))?;
 
-                segments.push(Segment {
+                segments.push(SegmentMedia {
                     audio,
                     system_audio,
                     cursor,

crates/editor/src/lib.rs

@@ -5,5 +5,5 @@ mod playback;
 mod segments;
 
 pub use audio::AudioRenderer;
-pub use editor_instance::{EditorInstance, EditorState, Segment, create_segments};
+pub use editor_instance::{EditorInstance, EditorState, SegmentMedia, create_segments};
 pub use segments::get_audio_segments;