From 5642b211d0e2a3fabaf415202af66b2c1205bad4 Mon Sep 17 00:00:00 2001 From: Yves Bastide Date: Thu, 8 Jan 2026 12:19:54 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20MaybeEscapePattern:=20overfl?= =?UTF-8?q?ow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yves Bastide --- RobotsTxt/RobotsTxtParser.cs | 9 +++++---- TestRobotsTxt/GoogleTests.cs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/RobotsTxt/RobotsTxtParser.cs b/RobotsTxt/RobotsTxtParser.cs index 5fb8418..708ba36 100644 --- a/RobotsTxt/RobotsTxtParser.cs +++ b/RobotsTxt/RobotsTxtParser.cs @@ -141,9 +141,10 @@ public static ReadOnlySpan MaybeEscapePattern(ReadOnlySpan src) var c = src[i]; if (c == '%' && i + 2 < src.Length && src[i + 1].IsXDigit() && src[i + 2].IsXDigit()) { - dst[j++] = src[i++]; - dst[j++] = src[i++].ToUpper(); - dst[j++] = src[i++].ToUpper(); + dst[j++] = (byte)'%'; + dst[j++] = src[i + 1].ToUpper(); + dst[j++] = src[i + 2].ToUpper(); + i += 2; } else if (c >= 0x80) { @@ -157,7 +158,7 @@ public static ReadOnlySpan MaybeEscapePattern(ReadOnlySpan src) } } - return dst; + return dst.AsSpan(0, j); } private static bool NeedEscapeValueForKey(ParsedRobotsKey key) diff --git a/TestRobotsTxt/GoogleTests.cs b/TestRobotsTxt/GoogleTests.cs index 7422375..2d37b77 100644 --- a/TestRobotsTxt/GoogleTests.cs +++ b/TestRobotsTxt/GoogleTests.cs @@ -1011,6 +1011,7 @@ public void TestGetPathParamsQuery(string url, string expected) [InlineData("/a/b/c", "/a/b/c")] [InlineData("á", "%C3%A1")] [InlineData("%aa", "%AA")] + [InlineData("%ab%c", "%AB%c")] public void TestMaybeEscapePattern(string url, string expected) { var actual = From 8d31a620dd7e48db530bdcba7ec0b8e89481aa94 Mon Sep 17 00:00:00 2001 From: Yves Bastide Date: Thu, 8 Jan 2026 12:20:37 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20use=20ArrayPool.?= =?UTF-8?q?Shared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yves Bastide --- RobotsTxt/RobotsTxtParser.cs | 11 ++++++++--- TestRobotsTxt/GoogleTests.cs | 4 +++- TestRobotsTxt/TestRobotsTxtParser.cs | 6 +++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/RobotsTxt/RobotsTxtParser.cs b/RobotsTxt/RobotsTxtParser.cs index 708ba36..65ec04a 100644 --- a/RobotsTxt/RobotsTxtParser.cs +++ b/RobotsTxt/RobotsTxtParser.cs @@ -1,3 +1,5 @@ +using System.Buffers; + namespace RobotsTxt; public class RobotsTxtParser(byte[] robotsBody, IRobotsParseHandler handler) @@ -74,8 +76,9 @@ private void ParseAndEmitLine(int currentLine, ReadOnlySpan line) key.Parse(stringKey); if (NeedEscapeValueForKey(key)) { - var escapedValue = MaybeEscapePattern(value); + var escapedValue = MaybeEscapePattern(value, out var dst); EmitKeyValueToHandler(currentLine, key, escapedValue); + if (dst != null) ArrayPool.Shared.Return(dst); } else { @@ -107,7 +110,7 @@ private void EmitKeyValueToHandler(int currentLine, ParsedRobotsKey key, ReadOnl } } - public static ReadOnlySpan MaybeEscapePattern(ReadOnlySpan src) + public static ReadOnlySpan MaybeEscapePattern(ReadOnlySpan src, out byte[]? dst) { var numToEscape = 0; var needCapitalize = false; @@ -131,10 +134,12 @@ public static ReadOnlySpan MaybeEscapePattern(ReadOnlySpan src) if (numToEscape == 0 && !needCapitalize) { + dst = null; return src; } - var dst = new byte[numToEscape * 2 + src.Length]; + dst = ArrayPool.Shared.Rent(numToEscape * 2 + src.Length); + var j = 0; for (var i = 0; i < src.Length; i++) { diff --git a/TestRobotsTxt/GoogleTests.cs b/TestRobotsTxt/GoogleTests.cs index 2d37b77..0f0a08b 100644 --- a/TestRobotsTxt/GoogleTests.cs +++ b/TestRobotsTxt/GoogleTests.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Diagnostics; using System.Text; using RobotsTxt; @@ -1015,8 +1016,9 @@ public void TestGetPathParamsQuery(string url, string expected) public void TestMaybeEscapePattern(string url, string expected) { var actual = - Encoding.ASCII.GetString(RobotsTxtParser.MaybeEscapePattern(Encoding.UTF8.GetBytes(url)).ToArray()); + Encoding.ASCII.GetString(RobotsTxtParser.MaybeEscapePattern(Encoding.UTF8.GetBytes(url), out var dst).ToArray()); Assert.Equal(expected, actual); + if (dst != null) ArrayPool.Shared.Return(dst); } } } diff --git a/TestRobotsTxt/TestRobotsTxtParser.cs b/TestRobotsTxt/TestRobotsTxtParser.cs index bae5ff6..4074243 100644 --- a/TestRobotsTxt/TestRobotsTxtParser.cs +++ b/TestRobotsTxt/TestRobotsTxtParser.cs @@ -1,5 +1,8 @@ +using System.Buffers; using System.Text; + using RobotsTxt; + using Xunit; namespace TestRobotsTxt @@ -56,8 +59,9 @@ public void TestGetKeyAndValueFrom(string line, bool rc, string expectedKey, str [InlineData("é", "%C3%A9")] public void TestMaybeEscapePattern(string src, string expected) { - var actual = RobotsTxtParser.MaybeEscapePattern(Encoding.UTF8.GetBytes(src)); + var actual = RobotsTxtParser.MaybeEscapePattern(Encoding.UTF8.GetBytes(src), out var dst); Assert.Equal(expected, Encoding.UTF8.GetString(actual.ToArray())); + if (dst != null) ArrayPool.Shared.Return(dst); } } } From 86f9141fdf66463a0f306b1fa0505a8cc7c633ee Mon Sep 17 00:00:00 2001 From: Yves Bastide Date: Thu, 8 Jan 2026 12:27:10 +0100 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9C=85=20add=20tests=20suggested=20by=20?= =?UTF-8?q?Copilot=20=F0=9F=A4=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yves Bastide --- TestRobotsTxt/GoogleTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TestRobotsTxt/GoogleTests.cs b/TestRobotsTxt/GoogleTests.cs index 0f0a08b..1feade9 100644 --- a/TestRobotsTxt/GoogleTests.cs +++ b/TestRobotsTxt/GoogleTests.cs @@ -1013,6 +1013,8 @@ public void TestGetPathParamsQuery(string url, string expected) [InlineData("á", "%C3%A1")] [InlineData("%aa", "%AA")] [InlineData("%ab%c", "%AB%c")] + [InlineData("test%", "test%")] + [InlineData("%a", "%a")] public void TestMaybeEscapePattern(string url, string expected) { var actual = From 2acf20ec012b3cfe5204ba97a12ceab7e2fc7a01 Mon Sep 17 00:00:00 2001 From: Yves Bastide Date: Thu, 8 Jan 2026 12:29:33 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=91=B7=20update=20actions,=20.NET=20v?= =?UTF-8?q?ersion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yves Bastide --- .github/workflows/dotnet.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 235f1ef..ba01b9f 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -15,11 +15,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Setup .NET - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v5 with: - dotnet-version: 9.0.x + dotnet-version: 10 - name: Restore dependencies run: dotnet restore - name: Build