Refactor PDF text extraction to support configurable margins for header, footer, left, and right

Author: richardr1126

src/components/DocumentSettings.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { Fragment, useState, useRef, useCallback, useEffect } from 'react';
+import { Fragment, useState, useRef, useEffect } from 'react';
 import { Dialog, DialogPanel, Transition, TransitionChild, Listbox, ListboxButton, ListboxOptions, ListboxOption, Button } from '@headlessui/react';
 import { useConfig, ViewType } from '@/contexts/ConfigContext';
 import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons';
@@ -21,32 +21,54 @@ const viewTypes = [
 ];
 
 export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsProps) {
-  const { viewType, skipBlank, epubTheme, textExtractionMargin, updateConfigKey } = useConfig();
+  const { 
+    viewType, 
+    skipBlank, 
+    epubTheme, 
+    headerMargin,
+    footerMargin,
+    leftMargin,
+    rightMargin,
+    updateConfigKey 
+  } = useConfig();
   const { createFullAudioBook } = useEPUB();
   const [progress, setProgress] = useState(0);
   const [isGenerating, setIsGenerating] = useState(false);
-  const [localMargin, setLocalMargin] = useState(textExtractionMargin);
+  const [localMargins, setLocalMargins] = useState({
+    header: headerMargin,
+    footer: footerMargin,
+    left: leftMargin,
+    right: rightMargin
+  });
   const abortControllerRef = useRef<AbortController | null>(null);
   const selectedView = viewTypes.find(v => v.id === viewType) || viewTypes[0];
 
-  //console.log(localMargin, textExtractionMargin);
-
-  // Sync local margin with global state
+  // Sync local margins with global state
   useEffect(() => {
-    setLocalMargin(textExtractionMargin);
-  }, [textExtractionMargin]);
+    setLocalMargins({
+      header: headerMargin,
+      footer: footerMargin,
+      left: leftMargin,
+      right: rightMargin
+    });
+  }, [headerMargin, footerMargin, leftMargin, rightMargin]);
 
   // Handler for slider change (updates local state only)
-  const handleMarginChange = useCallback((event: React.ChangeEvent<HTMLInputElement>) => {
-    setLocalMargin(Number(event.target.value));
-  }, []);
+  const handleMarginChange = (margin: keyof typeof localMargins) => (event: React.ChangeEvent<HTMLInputElement>) => {
+    setLocalMargins(prev => ({
+      ...prev,
+      [margin]: Number(event.target.value)
+    }));
+  };
 
   // Handler for slider release
-  const handleMarginChangeComplete = useCallback(() => {
-    if (localMargin !== textExtractionMargin) {
-      updateConfigKey('textExtractionMargin', localMargin);
+  const handleMarginChangeComplete = (margin: keyof typeof localMargins) => () => {
+    const value = localMargins[margin];
+    const configKey = `${margin}Margin`;
+    if (value !== (useConfig)[configKey as keyof typeof useConfig]) {
+      updateConfigKey(configKey as 'headerMargin' | 'footerMargin' | 'leftMargin' | 'rightMargin', value);
     }
-  }, [localMargin, textExtractionMargin, updateConfigKey]);
+  };
 
   const handleStartGeneration = async () => {
     setIsGenerating(true);
@@ -154,28 +176,92 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
                   </div>}
                   {!epub && <div className="space-y-6">
                     <div className="mt-4 space-y-2">
-                      <label className="block text-sm font-medium text-foreground">
-                        Text Extraction Margin
+                      <label className="block text-sm font-medium text-foreground mb-4">
+                        Adjust extraction margins (experimental)
                       </label>
-                      <div className="flex justify-between">
-                        <span className="text-xs">0%</span>
-                        <span className="text-xs font-bold">{Math.round(localMargin * 100)}%</span>
-                        <span className="text-xs">20%</span>
+                      <div className="grid grid-cols-1 sm:grid-cols-2 gap-2">
+                        {/* Header Margin */}
+                        <div className="space-y-1">
+                          <div className="flex justify-between">
+                            <span className="text-xs">Header</span>
+                            <span className="text-xs font-bold">{Math.round(localMargins.header * 100)}%</span>
+                          </div>
+                          <input
+                            type="range"
+                            min="0"
+                            max="0.2"
+                            step="0.01"
+                            value={localMargins.header}
+                            onChange={handleMarginChange('header')}
+                            onMouseUp={handleMarginChangeComplete('header')}
+                            onKeyUp={handleMarginChangeComplete('header')}
+                            onTouchEnd={handleMarginChangeComplete('header')}
+                            className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
+                          />
+                        </div>
+                        
+                        {/* Footer Margin */}
+                        <div className="space-y-1">
+                          <div className="flex justify-between">
+                            <span className="text-xs">Footer</span>
+                            <span className="text-xs font-bold">{Math.round(localMargins.footer * 100)}%</span>
+                          </div>
+                          <input
+                            type="range"
+                            min="0"
+                            max="0.2"
+                            step="0.01"
+                            value={localMargins.footer}
+                            onChange={handleMarginChange('footer')}
+                            onMouseUp={handleMarginChangeComplete('footer')}
+                            onKeyUp={handleMarginChangeComplete('footer')}
+                            onTouchEnd={handleMarginChangeComplete('footer')}
+                            className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
+                          />
+                        </div>
+
+                        {/* Left Margin */}
+                        <div className="space-y-1">
+                          <div className="flex justify-between">
+                            <span className="text-xs">Left</span>
+                            <span className="text-xs font-bold">{Math.round(localMargins.left * 100)}%</span>
+                          </div>
+                          <input
+                            type="range"
+                            min="0"
+                            max="0.2"
+                            step="0.01"
+                            value={localMargins.left}
+                            onChange={handleMarginChange('left')}
+                            onMouseUp={handleMarginChangeComplete('left')}
+                            onKeyUp={handleMarginChangeComplete('left')}
+                            onTouchEnd={handleMarginChangeComplete('left')}
+                            className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
+                          />
+                        </div>
+
+                        {/* Right Margin */}
+                        <div className="space-y-1">
+                          <div className="flex justify-between">
+                            <span className="text-xs">Right</span>
+                            <span className="text-xs font-bold">{Math.round(localMargins.right * 100)}%</span>
+                          </div>
+                          <input
+                            type="range"
+                            min="0"
+                            max="0.2"
+                            step="0.01"
+                            value={localMargins.right}
+                            onChange={handleMarginChange('right')}
+                            onMouseUp={handleMarginChangeComplete('right')}
+                            onKeyUp={handleMarginChangeComplete('right')}
+                            onTouchEnd={handleMarginChangeComplete('right')}
+                            className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
+                          />
+                        </div>
                       </div>
-                      <input
-                        type="range"
-                        min="0"
-                        max="0.2"
-                        step="0.01"
-                        value={localMargin}
-                        onChange={handleMarginChange}
-                        onMouseUp={handleMarginChangeComplete}
-                        onKeyUp={handleMarginChangeComplete}
-                        onTouchEnd={handleMarginChangeComplete}
-                        className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
-                      />
-                      <p className="text-xs text-muted">
-                        {"Don't"} include content from outer rim of the page during text extraction (experimental)
+                      <p className="text-xs text-muted mt-2">
+                        Adjust margins to exclude content from edges of the page during text extraction
                       </p>
                     </div>
                     <Listbox

src/contexts/ConfigContext.tsx

@@ -16,6 +16,10 @@ type ConfigValues = {
   skipBlank: boolean;
   epubTheme: boolean;
   textExtractionMargin: number;
+  headerMargin: number;
+  footerMargin: number;
+  leftMargin: number;
+  rightMargin: number;
 };
 
 /** Interface defining the configuration context shape and functionality */
@@ -28,6 +32,10 @@ interface ConfigContextType {
   skipBlank: boolean;
   epubTheme: boolean;
   textExtractionMargin: number;
+  headerMargin: number;
+  footerMargin: number;
+  leftMargin: number;
+  rightMargin: number;
   updateConfig: (newConfig: Partial<{ apiKey: string; baseUrl: string; viewType: ViewType }>) => Promise<void>;
   updateConfigKey: <K extends keyof ConfigValues>(key: K, value: ConfigValues[K]) => Promise<void>;
   isLoading: boolean;
@@ -52,6 +60,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
   const [skipBlank, setSkipBlank] = useState<boolean>(true);
   const [epubTheme, setEpubTheme] = useState<boolean>(false);
   const [textExtractionMargin, setTextExtractionMargin] = useState<number>(0.07);
+  const [headerMargin, setHeaderMargin] = useState<number>(0.07);
+  const [footerMargin, setFooterMargin] = useState<number>(0.07);
+  const [leftMargin, setLeftMargin] = useState<number>(0.07);
+  const [rightMargin, setRightMargin] = useState<number>(0.07);
 
   const [isLoading, setIsLoading] = useState(true);
   const [isDBReady, setIsDBReady] = useState(false);
@@ -72,6 +84,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         const cachedSkipBlank = await getItem('skipBlank');
         const cachedEpubTheme = await getItem('epubTheme');
         const cachedMargin = await getItem('textExtractionMargin');
+        const cachedHeaderMargin = await getItem('headerMargin');
+        const cachedFooterMargin = await getItem('footerMargin');
+        const cachedLeftMargin = await getItem('leftMargin');
+        const cachedRightMargin = await getItem('rightMargin');
 
         // Only set API key and base URL if they were explicitly saved by the user
         if (cachedApiKey) {
@@ -90,6 +106,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         setSkipBlank(cachedSkipBlank === 'false' ? false : true);
         setEpubTheme(cachedEpubTheme === 'true');
         setTextExtractionMargin(parseFloat(cachedMargin || '0.07'));
+        setHeaderMargin(parseFloat(cachedHeaderMargin || '0.07'));
+        setFooterMargin(parseFloat(cachedFooterMargin || '0.07'));
+        setLeftMargin(parseFloat(cachedLeftMargin || '0.07'));
+        setRightMargin(parseFloat(cachedRightMargin || '0.07'));
 
         // Only save non-sensitive settings by default
         if (!cachedViewType) {
@@ -104,6 +124,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         if (cachedMargin === null) {
           await setItem('textExtractionMargin', '0.07');
         }
+        if (cachedHeaderMargin === null) await setItem('headerMargin', '0.07');
+        if (cachedFooterMargin === null) await setItem('footerMargin', '0.07');
+        if (cachedLeftMargin === null) await setItem('leftMargin', '0.0');
+        if (cachedRightMargin === null) await setItem('rightMargin', '0.0');
 
       } catch (error) {
         console.error('Error initializing:', error);
@@ -181,6 +205,18 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         case 'textExtractionMargin':
           setTextExtractionMargin(value as number);
           break;
+        case 'headerMargin':
+          setHeaderMargin(value as number);
+          break;
+        case 'footerMargin':
+          setFooterMargin(value as number);
+          break;
+        case 'leftMargin':
+          setLeftMargin(value as number);
+          break;
+        case 'rightMargin':
+          setRightMargin(value as number);
+          break;
       }
     } catch (error) {
       console.error(`Error updating config key ${key}:`, error);
@@ -198,6 +234,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
       skipBlank,
       epubTheme,
       textExtractionMargin,
+      headerMargin,
+      footerMargin,
+      leftMargin,
+      rightMargin,
       updateConfig, 
       updateConfigKey,
       isLoading, 

src/contexts/EPUBContext.tsx

@@ -233,8 +233,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
 
           audioChunks.push(audioBuffer);
 
-          // Add a small pause between sections (500ms of silence)
-          const silenceBuffer = new ArrayBuffer(24000);
+          // Add a small pause between sections (1s of silence)
+          const silenceBuffer = new ArrayBuffer(48000);
           audioChunks.push(silenceBuffer);
 
         } catch (error) {

src/contexts/PDFContext.tsx

@@ -77,8 +77,19 @@ const PDFContext = createContext<PDFContextType | undefined>(undefined);
  * @param {ReactNode} props.children - Child components to be wrapped by the provider
  */
 export function PDFProvider({ children }: { children: ReactNode }) {
-  const { setText: setTTSText, stop, currDocPageNumber: currDocPage, currDocPages, setCurrDocPages } = useTTS();
-  const { textExtractionMargin } = useConfig();
+  const { 
+    setText: setTTSText, 
+    stop, 
+    currDocPageNumber: currDocPage, 
+    currDocPages, 
+    setCurrDocPages 
+  } = useTTS();
+  const { 
+    headerMargin,
+    footerMargin,
+    leftMargin,
+    rightMargin
+  } = useConfig();
 
   // Current document state
   const [currDocURL, setCurrDocURL] = useState<string>();
@@ -106,7 +117,12 @@ export function PDFProvider({ children }: { children: ReactNode }) {
   const loadCurrDocText = useCallback(async () => {
     try {
       if (!pdfDocument) return;
-      const text = await extractTextFromPDF(pdfDocument, currDocPage, textExtractionMargin);
+      const text = await extractTextFromPDF(pdfDocument, currDocPage, {
+        header: headerMargin,
+        footer: footerMargin,
+        left: leftMargin,
+        right: rightMargin
+      });
       // Only update TTS text if the content has actually changed
       // This prevents unnecessary resets of the sentence index
       if (text !== currDocText || text === '') {
@@ -116,7 +132,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
     } catch (error) {
       console.error('Error loading PDF text:', error);
     }
-  }, [pdfDocument, currDocPage, setTTSText, currDocText, textExtractionMargin]);
+  }, [pdfDocument, currDocPage, setTTSText, currDocText, headerMargin, footerMargin, leftMargin, rightMargin]);
 
   /**
    * Effect hook to update document text when the page changes