Refactor implementation of building hyphenation trees to avoid StackOverflowErrors

The recursive implementation is now rewritten in a sequential way
This solves failures (sometimes flaky ones that depend on JVM config)
of the tests involving hyphenating strings in some of the languages

DEVSIX-2036
DEVSIX-3183

Autoported commit.
Original commit hash: [b1281a42c]

Author: Snipx

itext.tests/itext.layout.tests/itext/layout/HyphenateResultTest.cs

@@ -47,14 +47,12 @@ source product.
 namespace iText.Layout {
     public class HyphenateResultTest : ExtendedITextTest {
         [NUnit.Framework.Test]
-        [NUnit.Framework.Ignore("DEVSIX-2036")]
         public virtual void UkraineHyphenTest() {
             //здравствуйте
             TestHyphenateResult("uk", "\u0437\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439", new int[] { 5 });
         }
 
         [NUnit.Framework.Test]
-        [NUnit.Framework.Ignore("DEVSIX-2036")]
         public virtual void UkraineNoneHyphenTest() {
             //день
             TestHyphenateResult("uk", "\u0434\u0435\u043D\u044C", null);

itext.tests/itext.layout.tests/itext/layout/HyphenateTest.cs

@@ -113,7 +113,6 @@ public class HyphenateTest : ExtendedITextTest {
         //సుస్వాగతం
         //здравствуй
         [NUnit.Framework.Test]
-        [NUnit.Framework.Ignore("DEVSIX-2036")]
         public virtual void RunTest() {
             foreach (HyphenateTest.TestParams param in @params) {
                 TryHyphenate(param.lang, param.testWorld, param.shouldPass);

itext/itext.layout/itext/layout/hyphenation/TernaryTree.cs

@@ -14,6 +14,9 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
+/*
+* PLEASE NOTE that implementation of "insert" function was refactored to consume less stack memory
+*/
 using System;
 using System.Collections;
 
@@ -153,7 +156,7 @@ public virtual void Insert(String key, char val) {
             char[] strkey = new char[len--];
             key.JGetChars(0, len, strkey, 0);
             strkey[len] = (char)0;
-            root = Insert(root, strkey, 0, val);
+            root = Insert(new TernaryTree.TreeInsertionParams(root, strkey, 0, val));
         }
 
         /// <summary>Insert key.</summary>
@@ -165,11 +168,17 @@ public virtual void Insert(char[] key, int start, char val) {
             if (freenode + len > eq.Length) {
                 RedimNodeArrays(eq.Length + BLOCK_SIZE);
             }
-            root = Insert(root, key, start, val);
+            root = Insert(new TernaryTree.TreeInsertionParams(root, key, start, val));
         }
 
-        /// <summary>The actual insertion function, recursive version.</summary>
-        private char Insert(char p, char[] key, int start, char val) {
+        // PLEASE NOTE that this function is a result of refactoring "insert" method which
+        // is a modification of the original work
+        // Returns null if insertion is not needed and the id of the new node if insertion was performed
+        private char? InsertNewBranchIfNeeded(TernaryTree.TreeInsertionParams @params) {
+            char p = @params.p;
+            char[] key = @params.key;
+            int start = @params.start;
+            char val = @params.val;
             int len = Strlen(key, start);
             if (p == 0) {
                 // this means there is no branch, this node will start a new branch.
@@ -193,61 +202,118 @@ private char Insert(char p, char[] key, int start, char val) {
                 }
                 return p;
             }
-            if (sc[p] == 0xFFFF) {
-                // branch is compressed: need to decompress
-                // this will generate garbage in the external key array
-                // but we can do some garbage collection later
-                char pp = freenode++;
-                lo[pp] = lo[p];
-                // previous pointer to key
-                eq[pp] = eq[p];
-                // previous pointer to data
-                lo[p] = (char)0;
-                if (len > 0) {
-                    sc[p] = kv.Get(lo[pp]);
-                    eq[p] = pp;
-                    lo[pp]++;
-                    if (kv.Get(lo[pp]) == 0) {
-                        // key completly decompressed leaving garbage in key array
-                        lo[pp] = (char)0;
-                        sc[pp] = (char)0;
-                        hi[pp] = (char)0;
+            else {
+                return null;
+            }
+        }
+
+        // PLEASE NOTE that this function is a result of refactoring "insert" method which
+        // is a modification of the original work
+        private char InsertIntoExistingBranch(TernaryTree.TreeInsertionParams @params) {
+            char initialP = @params.p;
+            TernaryTree.TreeInsertionParams paramsToInsertNext = @params;
+            while (paramsToInsertNext != null) {
+                char p = paramsToInsertNext.p;
+                // We are inserting into an existing branch hence the id must be non-zero
+                System.Diagnostics.Debug.Assert(p != 0);
+                char[] key = paramsToInsertNext.key;
+                int start = paramsToInsertNext.start;
+                char val = paramsToInsertNext.val;
+                int len = Strlen(key, start);
+                paramsToInsertNext = null;
+                if (sc[p] == 0xFFFF) {
+                    // branch is compressed: need to decompress
+                    // this will generate garbage in the external key array
+                    // but we can do some garbage collection later
+                    char pp = freenode++;
+                    lo[pp] = lo[p];
+                    // previous pointer to key
+                    eq[pp] = eq[p];
+                    // previous pointer to data
+                    lo[p] = (char)0;
+                    if (len > 0) {
+                        sc[p] = kv.Get(lo[pp]);
+                        eq[p] = pp;
+                        lo[pp]++;
+                        if (kv.Get(lo[pp]) == 0) {
+                            // key completly decompressed leaving garbage in key array
+                            lo[pp] = (char)0;
+                            sc[pp] = (char)0;
+                            hi[pp] = (char)0;
+                        }
+                        else {
+                            // we only got first char of key, rest is still there
+                            sc[pp] = (char)0xFFFF;
+                        }
                     }
                     else {
-                        // we only got first char of key, rest is still there
+                        // In this case we can save a node by swapping the new node
+                        // with the compressed node
                         sc[pp] = (char)0xFFFF;
+                        hi[p] = pp;
+                        sc[p] = (char)0;
+                        eq[p] = val;
+                        length++;
+                        break;
                     }
                 }
-                else {
-                    // In this case we can save a node by swapping the new node
-                    // with the compressed node
-                    sc[pp] = (char)0xFFFF;
-                    hi[p] = pp;
-                    sc[p] = (char)0;
-                    eq[p] = val;
-                    length++;
-                    return p;
-                }
-            }
-            char s = key[start];
-            if (s < sc[p]) {
-                lo[p] = Insert(lo[p], key, start, val);
-            }
-            else {
-                if (s == sc[p]) {
-                    if (s != 0) {
-                        eq[p] = Insert(eq[p], key, start + 1, val);
+                char s = key[start];
+                if (s < sc[p]) {
+                    TernaryTree.TreeInsertionParams branchParams = new TernaryTree.TreeInsertionParams(lo[p], key, start, val);
+                    char? insertNew = InsertNewBranchIfNeeded(branchParams);
+                    if (insertNew == null) {
+                        paramsToInsertNext = branchParams;
                     }
                     else {
-                        // key already in tree, overwrite data
-                        eq[p] = val;
+                        lo[p] = (char)insertNew;
                     }
                 }
                 else {
-                    hi[p] = Insert(hi[p], key, start, val);
+                    if (s == sc[p]) {
+                        if (s != 0) {
+                            TernaryTree.TreeInsertionParams branchParams = new TernaryTree.TreeInsertionParams(eq[p], key, start + 1, 
+                                val);
+                            char? insertNew = InsertNewBranchIfNeeded(branchParams);
+                            if (insertNew == null) {
+                                paramsToInsertNext = branchParams;
+                            }
+                            else {
+                                eq[p] = (char)insertNew;
+                            }
+                        }
+                        else {
+                            // key already in tree, overwrite data
+                            eq[p] = val;
+                        }
+                    }
+                    else {
+                        TernaryTree.TreeInsertionParams branchParams = new TernaryTree.TreeInsertionParams(hi[p], key, start, val);
+                        char? insertNew = InsertNewBranchIfNeeded(branchParams);
+                        if (insertNew == null) {
+                            paramsToInsertNext = branchParams;
+                        }
+                        else {
+                            hi[p] = (char)insertNew;
+                        }
+                    }
                 }
             }
-            return p;
+            return initialP;
+        }
+
+        /// <summary>The actual insertion function, recursive version.</summary>
+        /// <remarks>
+        /// The actual insertion function, recursive version.
+        /// PLEASE NOTE that the implementation has been adapted to consume less stack memory
+        /// </remarks>
+        private char Insert(TernaryTree.TreeInsertionParams @params) {
+            char? newBranch = InsertNewBranchIfNeeded(@params);
+            if (newBranch == null) {
+                return InsertIntoExistingBranch(@params);
+            }
+            else {
+                return (char)newBranch;
+            }
         }
 
         /// <summary>Compares 2 null terminated char arrays</summary>
@@ -496,5 +562,24 @@ private void Compact(CharVector kx, iText.Layout.Hyphenation.TernaryTree map, ch
         public virtual IEnumerator Keys() {
             return new TernaryTreeIterator(this);
         }
+
+        private class TreeInsertionParams {
+            internal char p;
+
+            internal char[] key;
+
+            internal int start;
+
+            internal char val;
+
+            public TreeInsertionParams(char p, char[] key, int start, char val) {
+                // PLEASE NOTE that this is a helper class that was added as a result of the file modification
+                // and is not a part of the original file
+                this.p = p;
+                this.key = key;
+                this.start = start;
+                this.val = val;
+            }
+        }
     }
 }

port-hash

@@ -1 +1 @@
-f5e58abfe5b1b017a9326dc516d1c001d4434fa3
+b1281a42c0cbeba0db22049a125ab6d52aa22504