Update tokenizer alg to use is [.] (#6561)

Author: peterwald

src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Common/SimpleWordTokenizer.cs

@@ -76,18 +76,13 @@ public static IEnumerable<string> WordTokenize(ReadOnlyMemory<char> text)
             }
 
             // Join hyphenated words
-            if (span[0] == '-' &&
-                span.Length > 1 &&
-                span[1] == '\n')
+            if (span is ['-', '\n', ..])
             {
                 text = text.Slice(2);
                 continue;
             }
 
-            if (span[0] == '-' &&
-                span.Length > 2 &&
-                span[1] == '\r' &&
-                span[2] == '\n')
+            if (span is ['-', '\r', '\n', ..])
             {
                 text = text.Slice(3);
                 continue;

test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/SimpleTokenizerTests.cs

@@ -20,6 +20,8 @@ public class SimpleTokenizerTests
     [InlineData("word1-word2", new[] { "WORD1", "-", "WORD2" })]
     [InlineData("word1 - word2", new[] { "WORD1", "-", "WORD2" })]
     [InlineData("word1-\n word2", new[] { "WORD1", "WORD2" })]
+    [InlineData("word1-", new[] { "WORD1", "-" })]
+    [InlineData("word1&", new[] { "WORD1", "&" })]
     [InlineData("word1-\r\n word2", new[] { "WORD1", "WORD2" })]
     [InlineData("word1-\r\nword2", new[] { "WORD1WORD2" })]
     [InlineData("word1-\nword2", new[] { "WORD1WORD2" })]