Add ffmpeg wasm

frontend/viewer/package.json

@@ -11,6 +11,7 @@
   "scripts": {
     "dev": "vite",
     "build": "vite build",
+    "build-ffmpeg-worker": "vite build --config vite.config.ffmpeg-worker.ts",
     "preview": "vite preview",
     "pretest:playwright": "playwright install",
     "test:playwright": "playwright test",
@@ -89,6 +90,9 @@
     "@microsoft/signalr": "^8.0.7",
     "autoprefixer": "^10.4.21",
     "fast-json-patch": "^3.1.1",
+    "@ffmpeg/ffmpeg": "0.12.15",
+    "@ffmpeg/util": "0.12.2",
+    "@ffmpeg/core": "0.12.10",
     "jsdom": "^26.1.0",
     "just-throttle": "^4.2.0",
     "postcss": "catalog:",

frontend/viewer/src/lib/components/audio/AudioDialog.svelte

@@ -5,10 +5,8 @@
   import {useDialogsService} from '$lib/services/dialogs-service.js';
   import {useBackHandler} from '$lib/utils/back-handler.svelte';
   import {watch} from 'runed';
-  import {delay} from '$lib/utils/time';
   import AudioProvider from './audio-provider.svelte';
   import AudioEditor from './audio-editor.svelte';
-  import Loading from '$lib/components/Loading.svelte';
   import {useLexboxApi} from '$lib/services/service-provider';
   import {UploadFileResult} from '$lib/dotnet-types/generated-types/MiniLcm/Media/UploadFileResult';
   import {AppNotification} from '$lib/notifications/notifications';
@@ -21,8 +19,8 @@
 
   let submitting = $state(false);
   let selectedFile = $state<File>();
-  let audio = $state<Blob>();
-  const tooBig = $derived((audio?.size ?? 0) > 10 * 1024 * 1024);
+  let finalAudio = $state<File>();
+  const tooBig = $derived((finalAudio?.size ?? 0) > 10 * 1024 * 1024);
 
   let requester: {
     resolve: (mediaUri: string | undefined) => void
@@ -41,6 +39,10 @@
     if (!open) reset();
   });
 
+  watch(() => selectedFile, () => {
+    if (!selectedFile) finalAudio = undefined;
+  })
+
   function close() {
     open = false;
     reset();
@@ -53,12 +55,12 @@
   }
 
   function clearAudio() {
-    audio = selectedFile = undefined;
+    selectedFile = undefined;
     submitting = false;
   }
 
   async function submitAudio() {
-    if (!audio) throw new Error('No audio to upload');
+    if (!selectedFile) throw new Error('No audio to upload');
     if (!requester) throw new Error('No requester');
 
     submitting = true;
@@ -72,8 +74,8 @@
   }
 
   async function uploadAudio() {
-    if (!audio || !selectedFile) throw new Error($t`No file selected`);
-    const response = await lexboxApi.saveFile(audio, {filename: selectedFile.name, mimeType: audio.type});
+    if (!finalAudio) throw new Error($t`No file to upload`);
+    const response = await lexboxApi.saveFile(finalAudio, {filename: finalAudio.name, mimeType: finalAudio.type});
     switch (response.result) {
       case UploadFileResult.SavedLocally:
         AppNotification.display($t`Audio saved locally`, 'success');
@@ -94,16 +96,13 @@
     return response.mediaUri;
   }
 
-  async function onFileSelected(file: File) {
+  function onFileSelected(file: File) {
     selectedFile = file;
-    audio = await processAudio(file);
   }
 
-  async function onRecordingComplete(blob: Blob) {
+  function onRecordingComplete(blob: Blob) {
     let fileExt = mimeTypeToFileExtension(blob.type);
     selectedFile = new File([blob], `recording-${Date.now()}.${fileExt}`, {type: blob.type});
-    if (!open) return;
-    audio = await processAudio(blob);
   }
 
   function mimeTypeToFileExtension(mimeType: string) {
@@ -133,17 +132,8 @@
   }
 
   function onDiscard() {
-    audio = undefined;
     selectedFile = undefined;
   }
-
-  let loading = $state(false);
-  async function processAudio(blob: Blob): Promise<Blob> {
-    loading = true;
-    await delay(1000); // Simulate processing delay
-    loading = false;
-    return blob;
-  }
 </script>
 
 
@@ -152,20 +142,16 @@
     <Dialog.DialogHeader>
       <Dialog.DialogTitle>{$t`Add audio`}</Dialog.DialogTitle>
     </Dialog.DialogHeader>
-    {#if !audio || !selectedFile}
-      {#if loading}
-        <Loading class="self-center justify-self-center size-16"/>
-      {:else}
-        <AudioProvider {onFileSelected} {onRecordingComplete}/>
-      {/if}
+    {#if !selectedFile}
+      <AudioProvider {onFileSelected} {onRecordingComplete}/>
     {:else}
-      <AudioEditor {audio} name={selectedFile.name} onDiscard={onDiscard}/>
+      <AudioEditor audio={selectedFile} bind:finalAudio onDiscard={onDiscard}/>
       {#if tooBig}
         <p class="text-destructive text-lg text-end">{$t`File too big`}</p>
       {/if}
       <Dialog.DialogFooter>
         <Button onclick={() => open = false} variant="secondary">{$t`Cancel`}</Button>
-        <Button onclick={() => submitAudio()} disabled={tooBig} loading={submitting}>
+        <Button onclick={() => submitAudio()} disabled={tooBig || !finalAudio} loading={submitting}>
           {$t`Save audio`}
         </Button>
       </Dialog.DialogFooter>

frontend/viewer/src/lib/components/audio/audio-editor.svelte

@@ -1,54 +1,105 @@
 <script lang="ts">
-  import { t } from 'svelte-i18n-lingui';
+  import {t} from 'svelte-i18n-lingui';
   import Button from '../ui/button/button.svelte';
   import Waveform from './wavesurfer/waveform.svelte';
   import type WaveSurfer from 'wavesurfer.js';
   import {formatDigitalDuration} from '../ui/format/format-duration';
   import DevContent from '$lib/layout/DevContent.svelte';
   import {Label} from '../ui/label';
+  import {FFmpegApi} from './ffmpeg';
+  import Loading from '$lib/components/Loading.svelte';
+  import {resource, watch} from 'runed';
+  import {onDestroy} from 'svelte';
 
   type Props = {
-    audio: Blob;
-    name: string
+    audio: File;
+    finalAudio: File | undefined;
     onDiscard: () => void;
   };
 
-  let { audio, name, onDiscard }: Props = $props();
+  let {
+    audio,
+    finalAudio = $bindable(undefined),
+    onDiscard
+  }: Props = $props();
 
   let audioApi = $state<WaveSurfer>();
   let playing = $state(false);
   let duration = $state<number | null>(null);
-  const mb = $derived((audio.size / 1024 / 1024).toFixed(2));
-  const formatedDuration = $derived(duration ? formatDigitalDuration({ seconds: duration }) : 'unknown');
+  const mb = $derived(!finalAudio ? '0' : (finalAudio.size / 1024 / 1024).toFixed(2));
+  const formatedDuration = $derived(duration ? formatDigitalDuration({seconds: duration}) : 'unknown');
+  let ffmpegApi: FFmpegApi | undefined;
 
+  let ffmpegFile = resource(() => audio, async (audio, _, {signal}) => {
+    ffmpegApi ??= await FFmpegApi.create();
+    return await ffmpegApi.toFFmpegFile(audio, signal);
+  });
+
+  let flacFile = resource(() => [ffmpegFile.current], async ([file], _, {signal}) => {
+    if (!file) return;
+    ffmpegApi ??= await FFmpegApi.create();
+    return await ffmpegApi.convertToFlac(file, signal);
+  });
+
+  let readFile = resource(() => [flacFile.current], async ([file], _, {signal}) => {
+    if (!file) return;
+    ffmpegApi ??= await FFmpegApi.create();
+    return await ffmpegApi.readFile(file, signal);
+  });
+
+  watch(() => [readFile.current, readFile.loading] as const, ([file, loading]) => {
+    if (loading || !file) {
+      finalAudio = undefined;
+    } else {
+      finalAudio = file;
+    }
+  });
+
+  const loading = $derived(ffmpegFile.loading || flacFile.loading);
+  const error = $derived((ffmpegFile.error || flacFile.error)?.toString());
+
+  const abortController = new AbortController();
+  onDestroy(() => {
+    abortController.abort();
+    ffmpegApi?.terminate();
+  });
 </script>
 
 <div class="flex flex-col gap-4 items-center justify-center">
+  {#if loading || !finalAudio}
+    <Loading class="self-center justify-self-center size-16"/>
+  {:else}
   <span class="inline-grid grid-cols-[auto_auto_1rem_auto_auto] gap-2 items-baseline">
     <Label class="justify-self-end">{$t`Length:`}</Label> <span>{$t`${formatedDuration}`}</span>
     <span></span>
     <Label class="justify-self-end">{$t`Size:`}</Label> <span>{$t`${mb} MB`}</span>
-    {#if name}
+    {#if finalAudio.name}
       <Label class="justify-self-end">{$t`File name:`}</Label>
-      <span class="col-span-4">{$t`${name}`}</span>
+      <span class="col-span-4">{$t`${finalAudio.name}`}</span>
     {/if}
     <DevContent>
       <Label class="justify-self-end">{$t`Type:`}</Label>
-      <span class="col-span-4">{$t`${audio.type}`}</span>
+      <span class="col-span-4">{$t`${finalAudio.type}`}</span>
     </DevContent>
   </span>
-  <!-- contain-inline-size prevents wavesurfer from freaking out inside a grid -->
-  <!-- pb-8 ensures the timeline is in the bounds of the container -->
-  <div class="w-full grow max-h-32 pb-3 contain-inline-size border-y">
-    <Waveform {audio} bind:playing bind:audioApi bind:duration showTimeline autoplay class="size-full" />
-  </div>
-  <div class="flex gap-2">
-    <Button variant="secondary" icon="i-mdi-close" onclick={onDiscard} disabled={!audio}>{$t`Discard`}</Button>
-    <Button
-      icon={playing ? 'i-mdi-pause' : 'i-mdi-play'}
-      onclick={() => (playing ? audioApi?.pause() : audioApi?.play())}
-      disabled={!audioApi}
-      size="icon"
-    />
-  </div>
+    <!-- contain-size prevents wavesurfer from freaking out inside a grid
+    contain-inline-size would improve the height reactivity of the waveform, but
+    results in the waveform sometimes change its height unexpectedly -->
+    <!-- pb-8 ensures the timeline is in the bounds of the container -->
+    <div class="w-full grow max-h-32 pb-3 contain-size border-y">
+      <Waveform audio={finalAudio} bind:playing bind:audioApi bind:duration showTimeline autoplay class="size-full"/>
+    </div>
+    <div class="flex gap-2">
+      <Button variant="secondary" icon="i-mdi-close" onclick={onDiscard} disabled={!audio}>{$t`Discard`}</Button>
+      <Button
+        icon={playing ? 'i-mdi-pause' : 'i-mdi-play'}
+        onclick={() => (playing ? audioApi?.pause() : audioApi?.play())}
+        disabled={!audioApi}
+        size="icon"
+      />
+    </div>
+  {/if}
+  {#if error}
+    <p class="text-destructive">{error}</p>
+  {/if}
 </div>