Improved PDF text parsing

Author: Balearica

js/containers/fontContainer.js

@@ -412,7 +412,7 @@ export class FontCont {
         family = 'NimbusSans';
       } else if (/Arial/i.test(family)) {
         family = 'NimbusSans';
-      } else if (/Century/i.test(family)) {
+      } else if (/CenturySch/i.test(family)) {
         family = 'Century';
       } else if (/Palatino/i.test(family)) {
         family = 'Palatino';

js/import/convertPageStext.js

@@ -4,6 +4,7 @@ import {
   calcBboxUnion,
   calcBoxOverlap,
   calcLang,
+  cleanFamilyName,
   mean50,
   round6,
   unescapeXml,
@@ -33,6 +34,9 @@ export async function convertPageStext({ ocrStr, n }) {
   /** @type {Set<string>} */
   const langSet = new Set();
 
+  /** @type {Set<string>} */
+  const fontSet = new Set();
+
   function convertParStext(xmlPar) {
     /** @type {Array<OcrLine>} */
     const parLineArr = [];
@@ -245,7 +249,7 @@ export async function convertPageStext({ ocrStr, n }) {
 
         if (wordCharOrFontArr[i].length === 0) continue;
 
-        let wordInit = false;
+        let wordCharN = 0;
 
         for (let j = 0; j < wordCharOrFontArr[i].length; j++) {
           const charOrFont = wordCharOrFontArr[i][j];
@@ -257,7 +261,7 @@ export async function convertPageStext({ ocrStr, n }) {
             // (2) Runs of small caps that start with lower-case letters, which do not conform to the expectation that runs of small caps start with a capital letter.
             const sizePrevRaw = sizeCurrentRaw;
             sizeCurrentRaw = charOrFont.size;
-            const secondLetter = wordInit && textWordArr.length === 1 && /[A-Z]/.test(textWordArr[0]);
+            const secondLetter = wordCharN > 0 && textWordArr.length === 1 && /[A-Z]/.test(textWordArr[0]);
 
             let baselineNextLetter;
             const possibleNextLetter1 = wordCharOrFontArr[i][j + 1];
@@ -356,8 +360,6 @@ export async function convertPageStext({ ocrStr, n }) {
             smallCapsWord = smallCapsCurrent;
 
             if (/italic/i.test(charOrFont.name) || /-\w*ital/i.test(charOrFont.name) || /-it$/i.test(charOrFont.name) || /oblique/i.test(charOrFont.name)) {
-              // The word is already initialized, so we need to change the last element of the style array.
-              // Label as `smallCapsAlt` rather than `smallCaps`, as we confirm the word is all caps before marking as `smallCaps`.
               italicCurrent = true;
             } else {
               italicCurrent = false;
@@ -374,11 +376,13 @@ export async function convertPageStext({ ocrStr, n }) {
             baselineCurrent = charOrFont.origin.y;
           }
 
-          if (!wordInit) {
+          // This condition should make a word italic if the characters themselves are italic,
+          // even if leading/trailing punctuation is not italic.
+          if (wordCharN === 0 || wordCharN < 3 && /[A-Z\d]/i.test(charOrFont.text)) {
             boldWord = boldCurrent;
             italicWord = italicCurrent;
 
-            wordInit = true;
+            wordCharN++;
           }
 
           let bbox;
@@ -598,6 +602,8 @@ export async function convertPageStext({ ocrStr, n }) {
 
         wordObj.style.font = fontFamilyArr[i];
 
+        fontSet.add(cleanFamilyName(wordObj.style.font));
+
         wordObj.style.sup = superArr[i];
 
         wordObj.style.underline = underlineArr[i];
@@ -683,5 +689,7 @@ export async function convertPageStext({ ocrStr, n }) {
     });
   }
 
-  return { pageObj, dataTables: dataTablePage, langSet };
+  return {
+    pageObj, dataTables: dataTablePage, langSet, fontSet,
+  };
 }

js/utils/miscUtils.js

@@ -527,3 +527,47 @@ export const getStyleLookup = (style) => {
   }
   return styleStr;
 };
+
+/**
+ * Identify specific font families and return a standardized name.
+ * This function is not intended to map names to fonts supported by this program,
+ * but rather simply to normalize the names of common fonts to a single name.
+ */
+export const cleanFamilyName = (family) => {
+  let familyClean = family;
+  if (/NimbusRom/i.test(family)) {
+    familyClean = 'NimbusRoman';
+  } else if (/TimesNewRom/i.test(family)) {
+    familyClean = 'TimesNewRoman';
+  } else if (/NimbusSan/i.test(family)) {
+    familyClean = 'NimbusSans';
+  } else if (/Helvetica/i.test(family)) {
+    familyClean = 'Helvetica';
+  } else if (/Arial/i.test(family)) {
+    familyClean = 'Arial';
+  } else if (/CenturySch/i.test(family)) {
+    familyClean = 'CenturySchoolbook';
+  } else if (/Palatino/i.test(family)) {
+    familyClean = 'Palatino';
+  } else if (/Garamond/i.test(family)) {
+    familyClean = 'Garamond';
+  } else if (/CenturyGothic/i.test(family)) {
+    familyClean = 'CenturyGothic';
+  } else if (/AvantGarde/i.test(family)) {
+    familyClean = 'AvantGarde';
+  } else if (/Carlito/i.test(family)) {
+    familyClean = 'Carlito';
+  } else if (/Calibri/i.test(family)) {
+    familyClean = 'Calibri';
+  } else if (/Courier/i.test(family)) {
+    familyClean = 'Courier';
+  } else if (/NimbusMono/i.test(family)) {
+    familyClean = 'NimbusMono';
+  } else if (/Dingbats/i.test(family)) {
+    familyClean = 'Dingbats';
+  } else if (/Wingdings/i.test(family)) {
+    familyClean = 'Wingdings';
+  }
+
+  return familyClean;
+};

tests/module/importPdfText.spec.js

@@ -410,6 +410,12 @@ describe('Check that font style is detected for PDF imports.', function () {
     assert.isFalse(scribe.data.ocr.active[5].lines[22].words[4].style.underline);
   }).timeout(10000);
 
+  it('Italic style is detected when leading punctuation is non-italic', async () => {
+    await scribe.importFiles([`${ASSETS_PATH_KARMA}/high-risk_protection_order_application_for_and_declaration_in_support_of_mandatory_use.pdf`]);
+    assert.strictEqual(scribe.data.ocr.active[0].lines[15].words[1].text, '(Print');
+    assert.isTrue(scribe.data.ocr.active[0].lines[15].words[1].style.italic);
+  }).timeout(10000);
+
   it('Bold + italic style is detected', async () => {
     await scribe.importFiles([`${ASSETS_PATH_KARMA}/complaint_1.pdf`]);
     assert.isTrue(scribe.data.ocr.active[0].lines[1].words[0].style.italic);