Refactor audio conversion API to improve memory management and add streaming response

Author: richardr1126

src/app/api/audio/convert/route.ts

@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { spawn } from 'child_process';
-import { writeFile, readFile, mkdir, unlink, rmdir } from 'fs/promises';
+import { writeFile, mkdir, unlink, rmdir } from 'fs/promises';
+import { createReadStream } from 'fs';
 import { existsSync } from 'fs';
 import { join } from 'path';
 import { randomUUID } from 'crypto';
@@ -65,9 +66,19 @@ async function runFFmpeg(args: string[]): Promise<void> {
   });
 }
 
+async function cleanup(files: string[], directories: string[]) {
+  await Promise.all([
+    ...files.map(f => unlink(f).catch(console.error)),
+    ...directories.map(d => rmdir(d).catch(console.error))
+  ]);
+}
+
 export async function POST(request: NextRequest) {
+  const tempFiles: string[] = [];
+  const tempDirs: string[] = [];
+
   try {
-    // Parse the request body
+    // Parse the request body as a stream
     const data: ConversionRequest = await request.json();
 
     // Create temp directory if it doesn't exist
@@ -82,40 +93,65 @@ export async function POST(request: NextRequest) {
     const metadataPath = join(tempDir, `${id}.txt`);
     const intermediateDir = join(tempDir, `${id}-intermediate`);
 
+    tempFiles.push(outputPath, metadataPath);
+    tempDirs.push(intermediateDir);
+
     // Create intermediate directory
     if (!existsSync(intermediateDir)) {
       await mkdir(intermediateDir);
     }
 
-    // Process each chapter - no need for initial conversion since input is WAV
+    // Process chapters sequentially to avoid memory issues
     const chapterFiles: { path: string; title: string; duration: number }[] = [];
     let currentTime = 0;
 
     for (let i = 0; i < data.chapters.length; i++) {
       const chapter = data.chapters[i];
+      const inputPath = join(intermediateDir, `${i}-input.mp3`);
       const outputPath = join(intermediateDir, `${i}.wav`);
 
-      // Write the chapter audio directly since it's already WAV
-      await writeFile(outputPath, Buffer.from(new Uint8Array(chapter.buffer)));
+      tempFiles.push(inputPath, outputPath);
+
+      // Write the chapter audio to a temp file using a Buffer chunk size of 64KB
+      const chunkSize = 64 * 1024; // 64KB chunks
+      const buffer = Buffer.from(new Uint8Array(chapter.buffer));
+      const chunks: Buffer[] = [];
+      
+      for (let offset = 0; offset < buffer.length; offset += chunkSize) {
+        chunks.push(buffer.slice(offset, offset + chunkSize));
+      }
+      
+      await writeFile(inputPath, Buffer.concat(chunks));
+      chunks.length = 0; // Clear chunks array
+
+      // Convert to WAV with consistent format
+      await runFFmpeg([
+        '-i', inputPath,
+        '-acodec', 'pcm_s16le',
+        '-ar', '44100',
+        '-ac', '2',
+        outputPath
+      ]);
 
-      // Get the duration of this chapter
       const duration = await getAudioDuration(outputPath);
 
       chapterFiles.push({
         path: outputPath,
         title: chapter.title,
         duration
       });
+
+      // Clean up input file early
+      await unlink(inputPath).catch(console.error);
+      const index = tempFiles.indexOf(inputPath);
+      if (index > -1) {
+        tempFiles.splice(index, 1);
+      }
     }
 
     // Create chapter metadata file
     const metadata: string[] = [];
-    metadata.push(
-      `title=Kokoro Audiobook`,
-      `artist=KokoroTTS`,
-    );
 
-    // Calculate chapter timings based on actual durations
     chapterFiles.forEach((chapter) => {
       const startMs = Math.floor(currentTime * 1000);
       currentTime += chapter.duration;
@@ -134,6 +170,8 @@ export async function POST(request: NextRequest) {
 
     // Create list file for concat
     const listPath = join(tempDir, `${id}-list.txt`);
+    tempFiles.push(listPath);
+    
     await writeFile(
       listPath,
       chapterFiles.map(f => `file '${f.path}'`).join('\n')
@@ -152,24 +190,46 @@ export async function POST(request: NextRequest) {
       outputPath
     ]);
 
-    // Read the converted file
-    const m4bData = await readFile(outputPath);
-
-    // Clean up temp files
-    await Promise.all([
-      ...chapterFiles.map(f => unlink(f.path)),
-      unlink(metadataPath),
-      unlink(listPath),
-      unlink(outputPath),
-      rmdir(intermediateDir)
-    ].map(p => p.catch(console.error)));
+    // Create a readable stream from the output file
+    const fileStream = createReadStream(outputPath);
+
+    // Create a web-compatible ReadableStream from the Node.js stream
+    const webStream = new ReadableStream({
+      start(controller) {
+        fileStream.on('data', (chunk) => {
+          controller.enqueue(chunk);
+        });
+        
+        fileStream.on('end', () => {
+          controller.close();
+          // Clean up only after the stream has been fully sent
+          cleanup(tempFiles, tempDirs).catch(console.error);
+        });
+        
+        fileStream.on('error', (error) => {
+          console.error('Stream error:', error);
+          controller.error(error);
+          cleanup(tempFiles, tempDirs).catch(console.error);
+        });
+      },
+      cancel() {
+        fileStream.destroy();
+        cleanup(tempFiles, tempDirs).catch(console.error);
+      }
+    });
 
-    return new NextResponse(m4bData, {
+    // Return the streaming response
+    return new NextResponse(webStream, {
       headers: {
         'Content-Type': 'audio/mp4',
+        'Transfer-Encoding': 'chunked'
       },
     });
+
   } catch (error) {
+    // Clean up in case of error
+    await cleanup(tempFiles, tempDirs).catch(console.error);
+    
     console.error('Error converting audio:', error);
     return NextResponse.json(
       { error: 'Failed to convert audio format' }, 

src/app/api/tts/route.ts

@@ -6,8 +6,8 @@ export async function POST(req: NextRequest) {
     // Get API credentials from headers or fall back to environment variables
     const openApiKey = req.headers.get('x-openai-key') || process.env.API_KEY || 'none';
     const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
-    const { text, voice, speed, format } = await req.json();
-    console.log('Received TTS request:', text, voice, speed, format);
+    const { text, voice, speed } = await req.json();
+    console.log('Received TTS request:', text, voice, speed);
 
     if (!openApiKey) {
       return NextResponse.json({ error: 'Missing OpenAI API key' }, { status: 401 });
@@ -29,19 +29,16 @@ export async function POST(req: NextRequest) {
       voice: voice as "alloy",
       input: text,
       speed: speed,
-      // Use wav format for audiobook generation to avoid initial conversion
-      response_format: format === 'audiobook' ? 'wav' : (format === 'aac' ? 'aac' : 'mp3'),
-    }, { signal: req.signal }); // Pass the abort signal to OpenAI client
+      response_format: 'mp3',  // Always use mp3 since we convert to WAV later if needed
+    }, { signal: req.signal });
 
     // Get the audio data as array buffer
-    // This will also be aborted if the client cancels
     const stream = response.body;
 
     // Return audio data with appropriate headers
-    const contentType = format === 'audiobook' ? 'audio/wav' : (format === 'aac' ? 'audio/aac' : 'audio/mpeg');
     return new NextResponse(stream, {
       headers: {
-        'Content-Type': contentType
+        'Content-Type': 'audio/mpeg'
       }
     });
   } catch (error) {

src/components/DocumentSettings.tsx

@@ -6,6 +6,8 @@ import { useConfig, ViewType } from '@/contexts/ConfigContext';
 import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons';
 import { useEPUB } from '@/contexts/EPUBContext';
 import { usePDF } from '@/contexts/PDFContext';
+import { useTimeEstimation } from '@/hooks/useTimeEstimation';
+import { LoadingSpinner } from './Spinner';
 
 const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
 
@@ -37,9 +39,9 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
     rightMargin,
     updateConfigKey
   } = useConfig();
-  const { createFullAudioBook } = useEPUB();
-  const { createFullAudioBook: createPDFAudioBook } = usePDF();
-  const [progress, setProgress] = useState(0);
+  const { createFullAudioBook, isAudioCombining } = useEPUB();
+  const { createFullAudioBook: createPDFAudioBook, isAudioCombining: isPDFAudioCombining } = usePDF();
+  const { progress, setProgress, estimatedTimeRemaining } = useTimeEstimation();
   const [isGenerating, setIsGenerating] = useState(false);
   const [audioFormat, setAudioFormat] = useState<'mp3' | 'm4b'>('mp3');
   const [localMargins, setLocalMargins] = useState({
@@ -116,7 +118,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
       setProgress(0);
       abortControllerRef.current = null;
     }
-  }, [createFullAudioBook, createPDFAudioBook, epub, audioFormat]);
+  }, [createFullAudioBook, createPDFAudioBook, epub, audioFormat, setProgress]);
 
   const handleCancel = () => {
     if (abortControllerRef.current) {
@@ -195,7 +197,10 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
                         />
                       </div>
                       <div className="flex justify-between items-center text-sm text-muted">
-                        <span>{Math.round(progress)}% complete</span>
+                        <span>
+                          {Math.round(progress)}% complete
+                          {estimatedTimeRemaining && ` • ${estimatedTimeRemaining} remaining`}
+                        </span>
                         <Button
                           type="button"
                           className="inline-flex justify-center rounded-lg px-2.5 py-1 text-sm 
@@ -204,7 +209,13 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
                                     transform transition-transform duration-200 ease-in-out hover:scale-[1.02]"
                           onClick={handleCancel}
                         >
-                          Cancel and download
+                          {(epub ? isAudioCombining : isPDFAudioCombining) ? (
+                            <div className="w-full h-full flex items-center justify-end">
+                              <LoadingSpinner />
+                            </div>
+                          ) : (
+                            'Cancel and download'
+                          )}
                         </Button>
                       </div>
                     </div>