Fix StringManipulatorTool to support Unicode characters

Co-authored-by: MrHinsh <[email protected]>

Author: Copilot

src/MigrationTools.Tests/ProcessorEnrichers/StringManipulatorEnricherTests.cs

@@ -169,6 +169,58 @@ public void StringManipulatorTool_MultipleManipulators(string? value, string? ex
             Assert.AreEqual(expected, newValue);
         }
 
+        [DataTestMethod(), TestCategory("L1")]
+        [DataRow("Hello", "Hello")]
+        [DataRow("Héllo", "Héllo")]    // New behavior: accented chars preserved
+        [DataRow("Привет", "Привет")]    // New behavior: Cyrillic chars preserved
+        [DataRow("你好", "你好")]      // New behavior: Chinese chars preserved
+        [DataRow("Café résumé", "Café résumé")]  // New behavior: accented chars preserved
+        [DataRow("Test\u0001\u0002", "Test")]  // Control chars should be removed
+        [DataRow("Line1\nLine2", "Line1\nLine2")]  // Newlines should be preserved
+        [DataRow("Tab\tSeparated", "Tab\tSeparated")]  // Tabs should be preserved
+        public void StringManipulatorTool_DefaultManipulator_UnicodeSupport(string value, string expected)
+        {
+            var options = new StringManipulatorToolOptions();
+            options.Enabled = true;
+            options.MaxStringLength = 1000;
+            // No manipulators set - should use default
+            var x = GetStringManipulatorTool(options);
+
+            string? newValue = x.ProcessString(value);
+            Assert.AreEqual(expected, newValue);
+        }
+
+        [DataTestMethod(), TestCategory("L1")]
+        [DataRow("Hello", "Hello")]
+        [DataRow("Héllo", "Héllo")]    // Expected behavior: accented chars preserved
+        [DataRow("Привет", "Привет")]    // Expected behavior: Cyrillic chars preserved
+        [DataRow("你好", "你好")]      // Expected behavior: Chinese chars preserved
+        [DataRow("Café résumé", "Café résumé")]  // Expected behavior: accented chars preserved
+        [DataRow("Test\u0001\u0002", "Test")]  // Control chars should still be removed
+        [DataRow("Line1\nLine2", "Line1\nLine2")]  // Newlines should be preserved
+        [DataRow("Tab\tSeparated", "Tab\tSeparated")]  // Tabs should be preserved
+        public void StringManipulatorTool_DefaultManipulator_ExpectedBehavior(string value, string expected)
+        {
+            var options = new StringManipulatorToolOptions();
+            options.Enabled = true;
+            options.MaxStringLength = 1000;
+            // Use improved Unicode-supporting pattern
+            options.Manipulators = new List<RegexStringManipulator>
+            {
+                new RegexStringManipulator
+                {
+                    Enabled = true,
+                    Description = "Default: Removes control characters but preserves Unicode letters and symbols",
+                    Pattern = @"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]+",
+                    Replacement = ""
+                }
+            };
+            var x = GetStringManipulatorTool(options);
+
+            string? newValue = x.ProcessString(value);
+            Assert.AreEqual(expected, newValue);
+        }
+
         private static StringManipulatorTool GetStringManipulatorTool(StringManipulatorToolOptions options)
         {
             var services = new ServiceCollection();

src/MigrationTools/Tools/StringManipulatorTool.cs

@@ -73,8 +73,8 @@ private void AddDefaultManipulator()
             Options.Manipulators.Add(new RegexStringManipulator()
             {
                 Enabled = true,
-                Description = "Default: Removes invalid chars!",
-                Pattern = "[^( -~)\n\r\t]+",
+                Description = "Default: Removes control characters but preserves Unicode letters and symbols",
+                Pattern = @"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]+",
                 Replacement = ""
             });
         }