Allow marker processors to work with the implicit character attribute

Author: desplesda

CHANGELOG.md

@@ -8,10 +8,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Marker processors can now process implicit `[character/]` markers.
+  - Yarn Spinner automatically adds `[character]` markers to lines that begin with a character name. For example, `Mae: Hello!` gets implicitly rewritten to `[character name="Mae"]Mae: [/character]Hello!`. This allows games to know what character is speaking, as well as being able to trim off the character name.
+  - With this update, marker processors are able to register to handle this implicit marker, allowing games to customise the way that the name is handled.
+
 ### Changed
 
 - Fixed a bug where unbalanced markup without text siblings or children would cause an error
 - Fixed a bug where numbers inside markup properties were being parsed in a culture variant manner.
+- The implicit `[character/]` marker is now generated before all other markup processing takes place. As a result, replacement markers like `[select/]` that produce text that would have been identified as a character name won't have their output recognised as a character name.
+  - If you need this to keep working, manually add your own `[character name=etc] ... [/character]` markup around your content.
 
 ### Removed
 

YarnSpinner.Tests/MarkupTests.cs

@@ -1532,6 +1532,20 @@ public void TestSelfclosingReplacementMarkersDoNotConsumeWhitespace(string line,
             markup.diagnostics.Should().BeEmpty();
             markup.markup.Text.Should().Be(expected);
         }
+
+        [Fact]
+        public void TestMarkerProcessorsCanProcessCharacterNames()
+        {
+            var lineParser = new LineParser();
+            lineParser.RegisterMarkerProcessor("character", new MarkerUppercaseReplacer());
+
+            var markup = lineParser.ParseString("Mae: I'm talkin' here", "en-AU");
+            markup.Text.Should().Be("MAE: I'm talkin' here", "the character marker should be processed");
+            markup.Attributes.Should().Contain(m => m.Name == "character", "the marker should be left in place")
+                .Which.Properties.Should().Contain(
+                    kv => kv.Key == "name" && kv.Value.StringValue == "Mae", "the marker's properties should be unmodified"
+                );
+        }
     }
 
     public class BBCodeChevronReplacer : IAttributeMarkerProcessor
@@ -1556,4 +1570,19 @@ public ReplacementMarkerResult ProcessReplacementMarker(MarkupAttribute marker,
             return new ReplacementMarkerResult(diagnostics, 5 + tag.Length * 2);
         }
     }
+
+    /// <summary>
+    /// An <see cref="IAttributeMarkerProcessor"/> that makes markers uppercase. 
+    /// </summary>
+    public class MarkerUppercaseReplacer : IAttributeMarkerProcessor
+    {
+        public ReplacementMarkerResult ProcessReplacementMarker(MarkupAttribute marker, StringBuilder childBuilder, List<MarkupAttribute> childAttributes, string localeCode)
+        {
+            var contents = childBuilder.ToString();
+            childBuilder.Clear();
+            childBuilder.Append(contents.ToUpperInvariant());
+            childAttributes.Add(marker);
+            return new ReplacementMarkerResult([], 0);
+        }
+    }
 }

YarnSpinner/YarnSpinner.Markup/LineParser.cs

@@ -777,9 +777,9 @@ private MarkupProperty internalIDproperty()
         private MarkupTreeNode? sibling = null;
         // this keeps track of all accumulated invisible characters added by any replacement markup.
         // this is necessary to prevent a bug with the following:
-            // "this is a line [bold]with some replacement[/bold] markup and a non-replacement[a/]  markup"
+        // "this is a line [bold]with some replacement[/bold] markup and a non-replacement[a/]  markup"
         // assuming that this is intended to replace the [bold] with <b> in unity we would end up as though we had written:
-            // "this is a line <b>with some replacement</b> markup and a non-replacement[a/]  markup"
+        // "this is a line <b>with some replacement</b> markup and a non-replacement[a/]  markup"
         // but this has now pushed the [a] markup down by 7 invisible characters
         // so each replacement markup processor needs to let us know how many (if any) invisible characters there are so we can backshift any sibling attributes after a replacement markup
         private int invisibleCharacters = 0;
@@ -1486,8 +1486,11 @@ internal MarkupParseResult ParseString(string input, string localeCode, bool squ
             return ParseStringWithDiagnostics(input, localeCode, squish, sort, addImplicitCharacterAttribute).markup;
         }
 
-        private static readonly char[] trimChars = { ':', ' ' };
-        private static readonly System.Text.RegularExpressions.Regex implicitCharacterRegex = new(@"^[^:]*:\s*");
+        private static readonly System.Text.RegularExpressions.Regex implicitCharacterRegex = new(@"^(?<name>[^"":]+)(?<suffix>:\s*)");
+
+        // Matches a "[character" at the start of the string, which means that
+        // the string contains an explicit character marker.
+        private static readonly System.Text.RegularExpressions.Regex characterMarkerRegex = new(@"^\s*\[character");
 
         internal (MarkupParseResult markup, List<MarkupDiagnostic> diagnostics) ParseStringWithDiagnostics(string input, string localeCode, bool squish = true, bool sort = true, bool addImplicitCharacterAttribute = true)
         {
@@ -1497,6 +1500,19 @@ internal MarkupParseResult ParseString(string input, string localeCode, bool squ
             }
 
             input = input.Normalize();
+
+            if (addImplicitCharacterAttribute && characterMarkerRegex.IsMatch(input) == false)
+            {
+                // The line does not already contain a [character] marker at the
+                // start, and we've been asked to add an implicit character
+                // attribute. Attempt to find a character at the start, and
+                // replace it with markup that indicates the character name.
+                input = implicitCharacterRegex.Replace(
+                    input,
+                    (match) => $"[character name=\"{match.Groups["name"]}\"]{match.Groups["name"]}{match.Groups["suffix"]}[/character]"
+                );
+            }
+
             var tokens = LexMarkup(input);
             var parseResult = BuildMarkupTreeFromTokens(tokens, input);
 
@@ -1525,33 +1541,6 @@ internal MarkupParseResult ParseString(string input, string localeCode, bool squ
 
             var finalText = builder.ToString();
 
-            if (addImplicitCharacterAttribute)
-            {
-                var hasCharacterAttributeAlready = false;
-                foreach (var attribute in attributes)
-                {
-                    if (attribute.Name == "character")
-                    {
-                        hasCharacterAttributeAlready = true;
-                        break;
-                    }
-                }
-                if (!hasCharacterAttributeAlready)
-                {
-                    var match = implicitCharacterRegex.Match(finalText);
-                    if (match.Success)
-                    {
-                        var characterName = match.Value.TrimEnd(trimChars);
-                        var propertyList = new List<MarkupProperty>
-                        {
-                            new MarkupProperty("name", characterName),
-                        };
-                        var characterMarker = new MarkupAttribute(0, 0, match.Length, "character", propertyList);
-                        attributes.Add(characterMarker);
-                    }
-                }
-            }
-
             if (sort)
             {
                 // finally we want them sorted by their position in the source code