fix: rework hyphenation (#3188)

* Add snapshot test for the textkit layout engine

* Fix hyphenation algorithm

The hyphenation algorithm may change the string (e.g. by removing some characters, namely soft hyphens).
Therefore, calculating the glyphs must come after hyphenation, so that the glyphs match the final string.

Fixes #3018

This was probably broken in #2600.

* Change default hyphenation algorithm to more flexible paradigm

The Hyphenation algorithm should be able to leave soft hyphens in, to
indicate that a hyphen should be placed there if the word breaks there.

* Variable width penalty nodes

The line breaking algorithm needs to distinguish syllables which end with
a soft hyphen from syllables that do not, and only mark a syllable for
adding a hyphen in the former case.

* Ensure zero advanceWidth for soft hyphens in both font and pdfkit

For the line breaking algorithm, soft hyphens should be considered to
have a width of zero, since they are never printed directly (they can
only lead to an inserted hyphen if at the end of a line).

The font package was already doing this correctly, but the pdfkit
package considered the soft hyphen to be the same as a normal hyphen
with an advanceWidth of 333 in Helvetica. Without this change, in some
edge cases the pdfkit would break apart lines already broken apart by
the line breaking algorithm in textkit.

Added tests for both packages to make sure they remain compatible in the
future.

* Consider end-of-line hyphen width in bestFit algorithm

In the best fit line breaking algorithm, the width of the hyphen must
be taken into account, in case one is to be inserted at the end of the line.
This is the most readable change I was able to find to acheive the goal.
Maybe the bestFit algorithm could be optimized in the future, along with
writing extensive tests for corner cases.

* Soft hyphens in the text should not be rendered as hyphens

Therefore, we remove all soft hyphens from the attributed string after
linebreaking is completed, and recalculate the glyphs afterwards.
This way, pdfkit never sees the soft hyphens, and does not mistake them
for normal hyphens.

* Add another test for the textkit layout engine

Tests the functionality of custom word splitting functions

* Add changeset

* Pass builtin hyphenation callback to custom callback

This allows library users to avoid importing the callback themselves,
which probably most of the implementations will want to do.

Author: carlobeltrame

.changeset/ninety-dogs-grow.md

@@ -0,0 +1,18 @@
+---
+"@react-pdf/pdfkit": minor
+"@react-pdf/textkit": minor
+---
+
+Fix and rework the hyphenation algorithm, and allow custom word hyphenation algorithms to specify whether a hyphen should be inserted in case the word is wrapped.
+
+**Caution**: If you have been using a custom hyphenation callback - which hasn't been working properly since at least version 2.0.21 - then you will need to change your implementation to leave a soft hyphen character (`'\u00AD'`) at the end of syllables where you want react-pdf to insert a hyphen when wrapping lines. Syllables without a final soft hyphen character will still be able to break, but will not produce a hyphen character at the end of the line.
+
+This allows you to break correctly on normal hyphens or other special characters in your text. For example, to use the default english-language syllable breaking built into react-pdf, but also break after hyphens naturally occurring in your text (such as is often present in hyperlinks), you could use the following hyphenation callback:
+```js
+import { Font } from '@react-pdf/renderer';
+
+Font.registerHyphenationCallback((word, originalHyphenationCallback) => {
+  return originalHyphenationCallback(word).flatMap(w => w.split(/(?<=-)/))
+})
+```
+(`flatMap` requires at least ES2019)

packages/font/tests/standard-fonts.test.ts

@@ -266,4 +266,13 @@ describe('standard fonts', () => {
 
     expect(font.src).toBe('Helvetica-BoldOblique');
   });
+
+  it('should resolve advanceWidth of soft hyphen to be zero', () => {
+    const SOFT_HYPHEN = '\u00AD';
+    const fontStore = new FontStore();
+
+    const font = fontStore.getFont({ fontFamily: 'Helvetica' });
+
+    expect(font.data.encode(SOFT_HYPHEN)[1][0].advanceWidth).toBe(0);
+  });
 });

packages/layout/tests/text/layoutText.test.ts

@@ -89,15 +89,18 @@ describe('text layoutText', () => {
 
   test('should allow hyphenation callback to be overriden', async () => {
     const text = 'reallylongtext';
-    const hyphens = ['really', 'long', 'text'];
+    const hyphens = ['really­', 'long', 'text'];
     const hyphenationCallback = vi.fn().mockReturnValue(hyphens);
 
     const node = createTextNode(text, {}, { hyphenationCallback });
     const lines = layoutText(node, 50, 100, fontStore);
 
     expect(lines[0].string).toEqual('really-');
-    expect(lines[1].string).toEqual('long-');
+    expect(lines[1].string).toEqual('long');
     expect(lines[2].string).toEqual('text');
-    expect(hyphenationCallback).toHaveBeenCalledWith('reallylongtext');
+    expect(hyphenationCallback).toHaveBeenCalledWith(
+      'reallylongtext',
+      expect.any(Function),
+    );
   });
 });

packages/pdfkit/package.json

@@ -19,6 +19,7 @@
     "url": "http://badassjs.com/"
   },
   "scripts": {
+    "test": "vitest",
     "clear": "rimraf ./lib ./src/font/data/*.json",
     "parse:afm": "node ./src/font/data/compressData.js",
     "build": "npm run clear && npm run parse:afm && rollup -c ",

packages/pdfkit/src/font/afm.js

@@ -80,7 +80,7 @@ oe            .notdef        zcaron         ydieresis
 space         exclamdown     cent           sterling
 currency      yen            brokenbar      section
 dieresis      copyright      ordfeminine    guillemotleft
-logicalnot    hyphen         registered     macron
+logicalnot    softhyphen     registered     macron
 degree        plusminus      twosuperior    threesuperior
 acute         mu             paragraph      periodcentered
 cedilla       onesuperior    ordmasculine   guillemotright

packages/pdfkit/tests/font/standard.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest';
+
+import StandardFont from '../../src/font.js';
+
+describe('standard fonts', () => {
+  it('should resolve advanceWidth of soft hyphen to be zero', () => {
+    const SOFT_HYPHEN = '\u00AD';
+    const font = StandardFont.open({}, 'Helvetica', 'Helvetica', 'foobar');
+
+    expect(font.encode(SOFT_HYPHEN)[1][0].advanceWidth).toBe(0);
+  });
+});

packages/textkit/src/engines/linebreaker/bestFit.ts

@@ -2,6 +2,16 @@ import { Node } from './types';
 
 const INFINITY = 10000;
 
+const skipPastGlueAndPenalty = (nodes: Node[], start: number): Node => {
+  let j = start + 1;
+  for (; j < nodes.length; j++) {
+    if (nodes[j].type !== 'glue' && nodes[j].type !== 'penalty') {
+      break;
+    }
+  }
+  return nodes[j - 1];
+};
+
 const getNextBreakpoint = (
   subnodes: Node[],
   widths: number[],
@@ -37,6 +47,8 @@ const getNextBreakpoint = (
     return 0;
   };
 
+  let hyphenWidth = 0;
+
   for (let i = 0; i < subnodes.length; i += 1) {
     const node = subnodes[i];
 
@@ -50,7 +62,11 @@ const getNextBreakpoint = (
       sum.shrink += node.shrink;
     }
 
-    if (sum.width - sum.shrink > lineLength) {
+    const potentialEndOfLine = skipPastGlueAndPenalty(subnodes, i);
+    hyphenWidth =
+      potentialEndOfLine.type === 'penalty' ? potentialEndOfLine.width : 0;
+
+    if (sum.width - sum.shrink + hyphenWidth > lineLength) {
       if (position === null) {
         let j = i === 0 ? i + 1 : i;
 
@@ -78,7 +94,7 @@ const getNextBreakpoint = (
     }
   }
 
-  return sum.width - sum.shrink > lineLength ? position : null;
+  return sum.width - sum.shrink + hyphenWidth > lineLength ? position : null;
 };
 
 const applyBestFit = (nodes: Node[], widths: number[]): number[] => {

packages/textkit/src/engines/linebreaker/index.ts

@@ -5,8 +5,10 @@ import insertGlyph from '../../attributedString/insertGlyph';
 import advanceWidthBetween from '../../attributedString/advanceWidthBetween';
 import { AttributedString, Attributes, LayoutOptions } from '../../types';
 import { Node } from './types';
+import generateGlyphs from '../../layout/generateGlyphs';
 
-const HYPHEN = 0x002d;
+const SOFT_HYPHEN = '\u00AD';
+const HYPHEN_CODE_POINT = 0x002d;
 const TOLERANCE_STEPS = 5;
 const TOLERANCE_LIMIT = 50;
 
@@ -45,23 +47,49 @@ const breakLines = (
       end = prevNode.end;
 
       line = slice(start, end, attributedString);
-      line = insertGlyph(line.string.length, HYPHEN, line);
+      if (node.width > 0) {
+        // A non-zero-width penalty indicates an additional hyphen should be inserted
+        line = insertGlyph(line.string.length, HYPHEN_CODE_POINT, line);
+      }
     } else {
       end = node.end;
       line = slice(start, end, attributedString);
     }
 
     start = end;
 
-    return [...acc, line];
+    return [...acc, removeSoftHyphens(line)];
   }, []);
 
-  // Last line
-  lines.push(slice(start, attributedString.string.length, attributedString));
+  const lastLine = slice(
+    start,
+    attributedString.string.length,
+    attributedString,
+  );
+  lines.push(removeSoftHyphens(lastLine));
 
   return lines;
 };
 
+/**
+ * Remove all soft hyphen characters from the line.
+ * Soft hyphens are not relevant anymore after line breaking, and will only
+ * disrupt the rendering later down the line if left in the text.
+ *
+ * @param line
+ */
+const removeSoftHyphens = (line: AttributedString): AttributedString => {
+  const modifiedLine = {
+    ...line,
+    string: line.string.split(SOFT_HYPHEN).join(''),
+  };
+
+  return {
+    ...modifiedLine,
+    ...generateGlyphs()(modifiedLine),
+  };
+};
+
 /**
  * Return Knuth & Plass nodes based on line and previously calculated syllables
  *
@@ -78,6 +106,7 @@ const getNodes = (
   let start = 0;
 
   const hyphenWidth = 5;
+  const softHyphen = '\u00ad';
 
   const { syllables } = attributedString;
 
@@ -107,7 +136,8 @@ const getNodes = (
 
       if (syllables[index + 1] && hyphenated) {
         // Add penalty node. Penalty nodes are used to represent hyphenation points.
-        acc.push(knuthPlass.penalty(hyphenWidth, hyphenPenalty, 1));
+        const penaltyWidth = s.endsWith(softHyphen) ? hyphenWidth : 0;
+        acc.push(knuthPlass.penalty(penaltyWidth, hyphenPenalty, 1));
       }
     }
 

packages/textkit/src/engines/wordHyphenation/index.ts

@@ -10,7 +10,7 @@ const hyphenator = hyphen(pattern);
  * @returns Word parts
  */
 const splitHyphen = (word: string) => {
-  return word.split(SOFT_HYPHEN);
+  return word.split(new RegExp(`(?<=${SOFT_HYPHEN})`));
 };
 
 const cache: Record<string, string[]> = {};

packages/textkit/src/layout/index.ts

@@ -32,8 +32,8 @@ const layoutEngine = (engines: Engines) => {
       resolveYOffset(),
       resolveAttachments(),
       verticalAlignment(),
-      wrapWords(engines, options),
       generateGlyphs(),
+      wrapWords(engines, options),
       bidiMirroring(),
       preprocessRuns(engines),
     );