fix: improve splitText efficiency

Author: giladgd

src/splitText.ts

@@ -7,35 +7,90 @@ export function splitText<const S extends string[]>(text: string, separators: S)
     if (separators.length === 0)
         return [text];
 
-    const [firstSeparator, ...otherSeparators] = separators;
+    const separatorsFindTree = separatorsToFindTree(separators);
+    const activeChecks: SeparatorCheck[] = [];
+    const textIndexToLowestSeparatorIndex = new Map<TextIndex, SeparatorIndex>();
+    const finalPartRanges: Array<{
+        textStartIndex: TextIndex,
+        separatorIndex: SeparatorIndex,
+        conflictedBy: TextIndex[]
+    }> = [];
 
-    let res: (string | Separator<S[number]>)[] = [];
+    for (let i = 0; i < text.length; i++) {
+        const char = text[i];
 
-    let lastSplitIndex = 0;
-    let splitIndex = text.indexOf(firstSeparator);
+        activeChecks.unshift({
+            currentNode: separatorsFindTree,
+            startIndex: i,
+            conflictedBy: []
+        });
+        let rangeToAdd: (typeof finalPartRanges[number]) | null = null;
 
-    while (splitIndex >= 0) {
-        res.push(text.slice(lastSplitIndex, splitIndex));
-        res.push(Separator._create(firstSeparator));
+        for (let j = 0; j < activeChecks.length; j++) {
+            const activeCheck = activeChecks[j];
+            const nextNode = activeCheck.currentNode.next.get(char);
 
-        lastSplitIndex = splitIndex + firstSeparator.length;
-        splitIndex = text.indexOf(firstSeparator, lastSplitIndex);
+            if (nextNode == null) {
+                activeChecks.splice(j, 1);
+                j--;
+                continue;
+            }
+
+            if (nextNode.separator != null && nextNode.separatorIndex != null) {
+                if (!textIndexToLowestSeparatorIndex.has(activeCheck.startIndex) ||
+                    nextNode.separatorIndex < textIndexToLowestSeparatorIndex.get(activeCheck.startIndex)!
+                )
+                    textIndexToLowestSeparatorIndex.set(activeCheck.startIndex, nextNode.separatorIndex);
+
+                if (rangeToAdd == null || nextNode.separatorIndex < rangeToAdd.separatorIndex)
+                    rangeToAdd = {
+                        textStartIndex: activeCheck.startIndex,
+                        separatorIndex: nextNode.separatorIndex,
+                        conflictedBy: activeCheck.conflictedBy.slice()
+                    };
+            }
+
+            activeCheck.currentNode = nextNode;
+        }
+
+        if (rangeToAdd != null) {
+            if (activeChecks.length > 0) {
+                for (const activeCheck of activeChecks) {
+                    rangeToAdd.conflictedBy.push(activeCheck.startIndex);
+                    activeCheck.conflictedBy.push(rangeToAdd.textStartIndex);
+                }
+            }
+
+            finalPartRanges.push(rangeToAdd);
+        }
     }
 
-    if (lastSplitIndex < text.length)
-        res.push(text.slice(lastSplitIndex));
-
-    if (otherSeparators.length > 0) {
-        res = res
-            .filter((item) => item !== "")
-            .flatMap((item) => {
-                if (typeof item === "string")
-                    return splitText(item, otherSeparators);
-                else
-                    return item;
-            });
+    const res: (string | Separator<S[number]>)[] = [];
+    let lastEndIndex = 0;
+
+    for (const range of finalPartRanges) {
+        const isConflicted = range.conflictedBy.some((textIndex) => {
+            const conflictedByIndexSeparatorIndex = textIndexToLowestSeparatorIndex.get(textIndex);
+
+            if (conflictedByIndexSeparatorIndex == null)
+                return false;
+
+            return conflictedByIndexSeparatorIndex < range.separatorIndex;
+        });
+
+        if (isConflicted)
+            continue;
+
+        if (lastEndIndex < range.textStartIndex)
+            res.push(text.slice(lastEndIndex, range.textStartIndex));
+
+        res.push(Separator._create(separators[range.separatorIndex]));
+        lastEndIndex = range.textStartIndex + separators[range.separatorIndex].length;
     }
 
+    if (lastEndIndex < text.length)
+        res.push(text.slice(lastEndIndex));
+
     if (res.length === 0 && text.length === 0)
         res.push("");
 
@@ -54,3 +109,43 @@ export class Separator<S extends string> {
         return new Separator(separator);
     }
 }
+
+function separatorsToFindTree(separators: string[]): SeparatorFindNode {
+    const root: SeparatorFindNode = {next: new Map()};
+
+    for (let i = 0; i < separators.length; i++) {
+        const separator = separators[i];
+        let node = root;
+
+        for (let j = 0; j < separator.length; j++) {
+            const char = separator[j];
+
+            if (!node.next.has(char))
+                node.next.set(char, {next: new Map()});
+
+            node = node.next.get(char)!;
+        }
+
+        if (node.separator == null) {
+            node.separator = separator;
+            node.separatorIndex = i;
+        }
+    }
+
+    return root;
+}
+
+type SeparatorIndex = number;
+type TextIndex = number;
+type Char = string;
+type SeparatorFindNode = {
+    separator?: string,
+    separatorIndex?: SeparatorIndex,
+    next: Map<Char, SeparatorFindNode>
+};
+
+type SeparatorCheck = {
+    currentNode: SeparatorFindNode,
+    readonly startIndex: TextIndex,
+    readonly conflictedBy: TextIndex[]
+};

test/splitText.test.ts

@@ -7,9 +7,65 @@ describe("splitText", () => {
 
         expect(res).toEqual([
             "Hello ",
-            new Separator("<and>"),
+            Separator._create("<and>"),
             " world ",
-            new Separator("[then]"),
+            Separator._create("[then]"),
+            " !"
+        ]);
+    });
+
+    test("splitting text prioritizes first match", () => {
+        const res1 = splitText("Hello <and> world [then] !", ["<and>", "<and> worl", "[then]"]);
+        expect(res1).toEqual([
+            "Hello ",
+            Separator._create("<and>"),
+            " world ",
+            Separator._create("[then]"),
+            " !"
+        ]);
+
+        const res2 = splitText("Hello <and> world [then] !", ["<and> worl", "<and>", "[then]"]);
+        expect(res2).toEqual([
+            "Hello ",
+            Separator._create("<and> worl"),
+            "d ",
+            Separator._create("[then]"),
+            " !"
+        ]);
+
+        const res3 = splitText("Hello <and> world [then] !", ["<and>", "llo <and>", "[then]"]);
+        expect(res3).toEqual([
+            "Hello ",
+            Separator._create("<and>"),
+            " world ",
+            Separator._create("[then]"),
+            " !"
+        ]);
+
+        const res4 = splitText("Hello <and> world [then] !", ["llo <and>", "<and>", "[then]"]);
+        expect(res4).toEqual([
+            "He",
+            Separator._create("llo <and>"),
+            " world ",
+            Separator._create("[then]"),
+            " !"
+        ]);
+
+        const res5 = splitText("Hello <and> world [then] !", ["llo <and> w", "<and>", "[then]"]);
+        expect(res5).toEqual([
+            "He",
+            Separator._create("llo <and> w"),
+            "orld ",
+            Separator._create("[then]"),
+            " !"
+        ]);
+
+        const res6 = splitText("Hello <and> world [then] !", ["<and>", "llo <and> w", "[then]"]);
+        expect(res6).toEqual([
+            "Hello ",
+            Separator._create("<and>"),
+            " world ",
+            Separator._create("[then]"),
             " !"
         ]);
     });