Fix low surrogate on post data (#2887)

* Fix low surrogate on post data

* bump version

* fix and tests

* ignore some tests in firefox

* fix test

* fallback to string

Author: kblok

lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.local.json

@@ -1,4 +1,11 @@
 [
+  {
+    "testIdPattern": "[network.spec] PuppeteerSharp network*",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["cdp", "firefox"],
+    "expectations": ["FAIL"],
+    "comment": "The should work tests are being ignore upstream, the PuppeteerSharp ones are based on the same one."
+  },
   {
     "comment": "We have a different test that needs to run on windows",
     "testIdPattern": "[chrome-data.spec] Chrome should resolve system executable path (windows)",

lib/PuppeteerSharp.Tests/NetworkTests/RequestPostDataTests.cs

@@ -19,6 +19,30 @@ public async Task ShouldWork()
             Assert.That(request.PostData, Is.EqualTo("{\"foo\":\"bar\"}"));
         }
 
+        [Test, Retry(2), PuppeteerTest("network.spec", "PuppeteerSharp network Request.postData", "should work plain text")]
+        public async Task ShouldWorkPlainText()
+        {
+            await Page.GoToAsync(TestConstants.EmptyPage);
+            Server.SetRoute("/post", _ => Task.CompletedTask);
+            var requestTask = Page.WaitForRequestAsync((request) => !TestUtils.IsFavicon(request));
+            await Page.EvaluateExpressionHandleAsync("fetch('./post', { method: 'POST', body: 'Hello, world!'})");
+            var request = await requestTask.WithTimeout();
+            Assert.That(request, Is.Not.Null);
+            Assert.That(request.PostData, Is.EqualTo("Hello, world!"));
+        }
+
+        [Test, Retry(2), PuppeteerTest("network.spec", "PuppeteerSharp network Request.postData", "should work with low surrogate")]
+        public async Task ShouldWorkWithLowSurrogate()
+        {
+            await Page.GoToAsync(TestConstants.EmptyPage);
+            Server.SetRoute("/post", _ => Task.CompletedTask);
+            var requestTask = Page.WaitForRequestAsync((request) => !TestUtils.IsFavicon(request));
+            await Page.EvaluateExpressionHandleAsync("fetch('./post', { method: 'POST', body: 'Hello, world!\uDD71'})");
+            var request = await requestTask.WithTimeout();
+            Assert.That(request, Is.Not.Null);
+            Assert.That(request.PostData, Is.EqualTo("Hello, world!\uFFFD"));
+        }
+
         [Test, Retry(2), PuppeteerTest("network.spec", "network Request.postData", "should be |undefined| when there is no post data")]
         public async Task ShouldBeUndefinedWhenThereIsNoPostData()
         {

lib/PuppeteerSharp/Cdp/Messaging/Request.cs

@@ -20,15 +20,19 @@
 //  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 //  * SOFTWARE.
 
+using PuppeteerSharp.Helpers.Json;
+
 namespace PuppeteerSharp.Cdp.Messaging;
 
 using System.Collections.Generic;
 using System.Net.Http;
+using System.Text.Json.Serialization;
 
 internal class Request
 {
     public HttpMethod Method { get; set; }
 
+    [JsonConverter(typeof(LowSurrogateConverter))]
     public string PostData { get; set; }
 
     public Dictionary<string, string> Headers { get; set; } = [];

lib/PuppeteerSharp/Helpers/Json/LowSurrogateConverter.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Buffers;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace PuppeteerSharp.Helpers.Json;
+
+internal class LowSurrogateConverter : JsonConverter<string>
+{
+    public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType != JsonTokenType.String)
+        {
+            return null;
+        }
+
+        var span = reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan.ToArray();
+        var value = Encoding.UTF8.GetString(span);
+
+        try
+        {
+            if (reader.ValueIsEscaped)
+            {
+                value = JsonUnescape(value);
+            }
+
+            return value;
+        }
+        catch
+        {
+            return value;
+        }
+    }
+
+    public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options)
+    {
+        writer.WriteStringValue(value);
+    }
+
+    private static string JsonUnescape(string jsonString)
+    {
+        using var doc = JsonDocument.Parse($"\"{jsonString}\"");
+        return doc.RootElement.GetString();
+    }
+}

lib/PuppeteerSharp/PageCoverage/JSCoverage.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -10,8 +11,8 @@ namespace PuppeteerSharp.PageCoverage
 {
     internal class JSCoverage
     {
-        private readonly Dictionary<string, string> _scriptURLs = new();
-        private readonly Dictionary<string, string> _scriptSources = new();
+        private readonly ConcurrentDictionary<string, string> _scriptURLs = new();
+        private readonly ConcurrentDictionary<string, string> _scriptSources = new();
         private readonly ILogger _logger;
 
         private CDPSession _client;
@@ -140,8 +141,8 @@ private async Task OnScriptParsedAsync(DebuggerScriptParsedResponse scriptParseR
                 {
                     ScriptId = scriptParseResponse.ScriptId,
                 }).ConfigureAwait(false);
-                _scriptURLs.Add(scriptParseResponse.ScriptId, scriptParseResponse.Url);
-                _scriptSources.Add(scriptParseResponse.ScriptId, response.ScriptSource);
+                _scriptURLs.TryAdd(scriptParseResponse.ScriptId, scriptParseResponse.Url);
+                _scriptSources.TryAdd(scriptParseResponse.ScriptId, response.ScriptSource);
             }
             catch (Exception ex)
             {

lib/PuppeteerSharp/PuppeteerSharp.csproj

@@ -12,10 +12,10 @@
     <Description>Headless Browser .NET API</Description>
     <PackageId>PuppeteerSharp</PackageId>
     <PackageReleaseNotes></PackageReleaseNotes>
-    <PackageVersion>20.1.1</PackageVersion>
-    <ReleaseVersion>20.1.1</ReleaseVersion>
-    <AssemblyVersion>20.1.1</AssemblyVersion>
-    <FileVersion>20.1.1</FileVersion>
+    <PackageVersion>20.1.2</PackageVersion>
+    <ReleaseVersion>20.1.2</ReleaseVersion>
+    <AssemblyVersion>20.1.2</AssemblyVersion>
+    <FileVersion>20.1.2</FileVersion>
     <SynchReleaseVersion>false</SynchReleaseVersion>
     <StyleCopTreatErrorsAsWarnings>false</StyleCopTreatErrorsAsWarnings>
     <DebugType>embedded</DebugType>