Multisession network manager (#2437)

kblok · web-flow · commit f149fa90a1cd · 2024-02-06T21:49:26.000-03:00
* small progress

* some progress

* Some progress

* Some nice progress

* Feature complete

* code factor

* some fixes

* some fixes

* some other fixes
diff --git a/lib/PuppeteerSharp.TestServer/wwwroot/oopif.html b/lib/PuppeteerSharp.TestServer/wwwroot/oopif.html
@@ -0,0 +1,5 @@
+<a id="navigate-within-document" href="#nav">Navigate within document</a>
+<a name="nav"></a>
+<script>
+    fetch('oopif.html?requestFromOOPIF')
+</script>
diff --git a/lib/PuppeteerSharp.Tests/NetworkManagerTests/NetworkManagerTests.cs b/lib/PuppeteerSharp.Tests/NetworkManagerTests/NetworkManagerTests.cs
diff --git a/lib/PuppeteerSharp.Tests/OOPIFTests/OOPIFTests.cs b/lib/PuppeteerSharp.Tests/OOPIFTests/OOPIFTests.cs
@@ -250,10 +250,12 @@ public async Task ShouldLoadOopifIframesWithSubresourcesAndRequestInterception()
             var frameTask = Page.WaitForFrameAsync((frame) => frame.Url.EndsWith("/oopif.html"));
             await Page.SetRequestInterceptionAsync(true);
             Page.Request += (sender, e) => _ = e.Request.ContinueAsync();
-
+            var requestTask = Page.WaitForRequestAsync(request => request.Url.Contains("requestFromOOPIF"));
             await Page.GoToAsync(TestConstants.ServerUrl + "/dynamic-oopif.html");
             await frameTask.WithTimeout();
+            await requestTask.WithTimeout();
             Assert.That(Oopifs, Has.Exactly(1).Items);
+            Assert.AreEqual(requestTask.Result.Frame, frameTask.Result);
         }
 
         [PuppeteerTest("oopif.spec.ts", "OOPIF", "should support frames within OOP iframes")]
diff --git a/lib/PuppeteerSharp.Tests/PrerenderTests/WithNetworkRequestsTests.cs b/lib/PuppeteerSharp.Tests/PrerenderTests/WithNetworkRequestsTests.cs
@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using PuppeteerSharp.Nunit;
+using PuppeteerSharp.Tests.Attributes;
+
+namespace PuppeteerSharp.Tests.PrerenderTests;
+
+public class WithNetworkRequestsTests : PuppeteerPageBaseTest
+{
+    [PuppeteerTest("prerender.spec.ts", "with network requests", "can receive requests from the prerendered page")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task CanNavigateToAPrerenderedPageViaInput()
+    {
+        var urls = new List<string>();
+        Page.Request += (_, e) => urls.Add(e.Request.Url);
+        await Page.GoToAsync(TestConstants.ServerUrl + "/prerender/index.html");
+
+        var button = await Page.WaitForSelectorAsync("button");
+        await button.ClickAsync();
+
+        var mainFrame = Page.MainFrame;
+        var link = await mainFrame.WaitForSelectorAsync("a");
+        await Task.WhenAll(
+            mainFrame.WaitForNavigationAsync(),
+            link.ClickAsync()
+        );
+
+        Assert.AreSame(mainFrame, Page.MainFrame);
+        Assert.AreEqual("target", await mainFrame.EvaluateExpressionAsync<string>("document.body.innerText"));
+        Assert.AreSame(mainFrame, Page.MainFrame);
+
+        Assert.True(urls.Exists(url => url.Contains("prerender/target.html")));
+        Assert.True(urls.Exists(url => url.Contains("prerender/index.html")));
+        Assert.True(urls.Exists(url => url.Contains("prerender/target.html?fromPrerendered")));
+    }
+}
diff --git a/lib/PuppeteerSharp.Tests/PuppeteerSharp.Tests.csproj b/lib/PuppeteerSharp.Tests/PuppeteerSharp.Tests.csproj
@@ -10,6 +10,7 @@
 		<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
 		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
 		<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
+		<PackageReference Include="NSubstitute" Version="5.1.0" />
 		<PackageReference Include="NUnit" Version="3.13.3" />
 		<PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
 		<PackageReference Include="System.Net.Http" Version="4.3.4" />
diff --git a/lib/PuppeteerSharp/CDPSession.cs b/lib/PuppeteerSharp/CDPSession.cs
@@ -2,6 +2,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json.Linq;
@@ -15,6 +16,7 @@ public class CDPSession : ICDPSession
     {
         private readonly ConcurrentDictionary<int, MessageTask> _callbacks = new();
         private readonly string _parentSessionId;
+        private int _lastId;
 
         internal CDPSession(Connection connection, TargetType targetType, string sessionId, string parentSessionId)
         {
@@ -81,7 +83,7 @@ public async Task<JObject> SendAsync(string method, object args = null, bool wai
                     CloseReason);
             }
 
-            var id = Connection.GetMessageID();
+            var id = GetMessageID();
             var message = Connection.GetMessage(id, method, args, Id);
 
             MessageTask callback = null;
@@ -148,6 +150,8 @@ internal void OnMessage(ConnectionResponse obj)
             }
         }
 
+        internal int GetMessageID() => Interlocked.Increment(ref _lastId);
+
         internal void Close(string closeReason)
         {
             if (IsClosed)
diff --git a/lib/PuppeteerSharp/ChromeTargetManager.cs b/lib/PuppeteerSharp/ChromeTargetManager.cs
@@ -288,7 +288,7 @@ private async Task OnAttachedToTargetAsync(object sender, TargetAttachedToTarget
                 _attachedTargetsBySessionId.TryAdd(session.Id, target);
             }
 
-            (parentConnection as CDPSession)?.OnSessionReady(session);
+            (parentSession ?? parentConnection as CDPSession)?.OnSessionReady(session);
 
             await EnsureTargetsIdsForInitAsync().ConfigureAwait(false);
             _targetsIdsForInit.Remove(target.TargetId);
diff --git a/lib/PuppeteerSharp/DisposableActionsStack.cs b/lib/PuppeteerSharp/DisposableActionsStack.cs
@@ -0,0 +1,58 @@
+// * MIT License
+//  *
+//  * Copyright (c) Microsoft Corporation.
+//  *
+//  * Permission is hereby granted, free of charge, to any person obtaining a copy
+//  * of this software and associated documentation files (the "Software"), to deal
+//  * in the Software without restriction, including without limitation the rights
+//  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  * copies of the Software, and to permit persons to whom the Software is
+//  * furnished to do so, subject to the following conditions:
+//  *
+//  * The above copyright notice and this permission notice shall be included in all
+//  * copies or substantial portions of the Software.
+//  *
+//  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+//  * SOFTWARE.
+
+using System;
+using System.Collections.Generic;
+
+namespace PuppeteerSharp;
+
+/// <summary>
+/// This is a DisposableStack in puppeteer. We don't need to dispose objects manually.
+/// But this is used to run actions when this object is disposed.
+/// </summary>
+internal class DisposableActionsStack : IDisposable
+{
+    private readonly List<Action> _actions = [];
+
+    public bool IsDisposed { get; private set; }
+
+    public void Dispose()
+    {
+        if (IsDisposed)
+        {
+            return;
+        }
+
+        IsDisposed = true;
+        _actions.Reverse();
+
+        foreach (var action in _actions)
+        {
+            action();
+        }
+    }
+
+    public void Defer(Action action)
+    {
+        _actions.Add(action);
+    }
+}
diff --git a/lib/PuppeteerSharp/FrameManager.cs b/lib/PuppeteerSharp/FrameManager.cs
@@ -11,7 +11,7 @@
 
 namespace PuppeteerSharp
 {
-    internal class FrameManager : IDisposable, IAsyncDisposable
+    internal class FrameManager : IDisposable, IAsyncDisposable, IFrameProvider
     {
         private const int TimeForWaitingForSwap = 200;
         private const string UtilityWorldName = "__puppeteer_utility_world__";
@@ -28,7 +28,7 @@ internal FrameManager(CDPSession client, Page page, bool ignoreHTTPSErrors, Time
             Client = client;
             Page = page;
             _logger = Client.Connection.LoggerFactory.CreateLogger<FrameManager>();
-            NetworkManager = new NetworkManager(client, ignoreHTTPSErrors, this);
+            NetworkManager = new NetworkManager(ignoreHTTPSErrors, this, client.Connection.LoggerFactory);
             TimeoutSettings = timeoutSettings;
 
             Client.MessageReceived += Client_MessageReceived;
@@ -59,10 +59,7 @@ internal FrameManager(CDPSession client, Page page, bool ignoreHTTPSErrors, Time
 
         internal Frame MainFrame => FrameTree.MainFrame;
 
-        public void Dispose()
-        {
-            _eventsQueue?.Dispose();
-        }
+        public void Dispose() => _eventsQueue?.Dispose();
 
         public async ValueTask DisposeAsync()
         {
@@ -72,6 +69,8 @@ public async ValueTask DisposeAsync()
             }
         }
 
+        public Task<Frame> GetFrameAsync(string frameId) => FrameTree.TryGetFrameAsync(frameId);
+
         internal ExecutionContext ExecutionContextById(int contextId, CDPSession session = null)
         {
             session ??= Client;
@@ -111,11 +110,11 @@ internal DeviceRequestPromptManager GetDeviceRequestPromptManager(CDPSession cli
 
         internal Frame[] GetFrames() => FrameTree.Frames;
 
-        internal async Task InitializeAsync(CDPSession client = null)
+        internal async Task InitializeAsync(CDPSession client)
         {
             try
             {
-                client ??= Client;
+                var networkInitTask = NetworkManager.AddClientAsync(client);
                 var getFrameTreeTask = client.SendAsync<PageGetFrameTreeResponse>("Page.getFrameTree");
                 var autoAttachTask = client != Client
                     ? client.SendAsync("Target.setAutoAttach", new TargetSetAutoAttachRequest
@@ -136,7 +135,7 @@ await Task.WhenAll(
                 await Task.WhenAll(
                     client.SendAsync("Page.setLifecycleEventsEnabled", new PageSetLifecycleEventsEnabledRequest { Enabled = true }),
                     client.SendAsync("Runtime.enable"),
-                    client == Client ? NetworkManager.InitializeAsync() : Task.CompletedTask).ConfigureAwait(false);
+                    networkInitTask).ConfigureAwait(false);
 
                 await CreateIsolatedWorldAsync(client, UtilityWorldName).ConfigureAwait(false);
             }
@@ -154,11 +153,12 @@ await Task.WhenAll(
             }
         }
 
-        /**
-       * When the main frame is replaced by another main frame,
-       * we maintain the main frame object identity while updating
-       * its frame tree and ID.
-       */
+        /// <summary>
+        /// When the main frame is replaced by another main frame
+        /// we maintain the main frame object identity while updating
+        /// its frame tree and ID.
+        /// </summary>
+        /// <param name="client">New session.</param>
         internal async Task SwapFrameTreeAsync(CDPSession client)
         {
             OnExecutionContextsCleared(Client);
@@ -181,11 +181,14 @@ internal async Task SwapFrameTreeAsync(CDPSession client)
             Client.Disconnected += (sender, e) => _ = OnClientDisconnectAsync();
 
             await InitializeAsync(client).ConfigureAwait(false);
-            await NetworkManager.UpdateClientAsync(client).ConfigureAwait(false);
+            await NetworkManager.AddClientAsync(client).ConfigureAwait(false);
 
             frame?.OnFrameSwappedByActivation();
         }
 
+        internal Task RegisterSpeculativeSessionAsync(CDPSession client)
+            => NetworkManager.AddClientAsync(client);
+
         private Frame GetFrame(string frameId) => FrameTree.GetById(frameId);
 
         private void Client_MessageReceived(object sender, MessageEventArgs e)
diff --git a/lib/PuppeteerSharp/IFrameProvider.cs b/lib/PuppeteerSharp/IFrameProvider.cs
@@ -0,0 +1,38 @@
+// * MIT License
+//  *
+//  * Copyright (c) Darío Kondratiuk
+//  *
+//  * Permission is hereby granted, free of charge, to any person obtaining a copy
+//  * of this software and associated documentation files (the "Software"), to deal
+//  * in the Software without restriction, including without limitation the rights
+//  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  * copies of the Software, and to permit persons to whom the Software is
+//  * furnished to do so, subject to the following conditions:
+//  *
+//  * The above copyright notice and this permission notice shall be included in all
+//  * copies or substantial portions of the Software.
+//  *
+//  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+//  * SOFTWARE.
+
+using System.Threading.Tasks;
+
+namespace PuppeteerSharp;
+
+/// <summary>
+/// Provides a way to get a frame.
+/// </summary>
+internal interface IFrameProvider
+{
+    /// <summary>
+    /// Gets a frame by its ID.
+    /// </summary>
+    /// <param name="frameId">The frame ID.</param>
+    /// <returns>A <see cref="Task"/> that completes when the frame is retrieved.</returns>
+    Task<Frame> GetFrameAsync(string frameId);
+}
diff --git a/lib/PuppeteerSharp/InitiatorType.cs b/lib/PuppeteerSharp/InitiatorType.cs
@@ -1,10 +1,13 @@
 using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 
 namespace PuppeteerSharp;
 
 /// <summary>
 /// Type of the <see cref="Initiator"/>.
 /// </summary>
+[JsonConverter(typeof(StringEnumConverter))]
 public enum InitiatorType
 {
     /// <summary>
diff --git a/lib/PuppeteerSharp/MessageEventArgs.cs b/lib/PuppeteerSharp/MessageEventArgs.cs
@@ -1,11 +1,12 @@
+using System;
 using Newtonsoft.Json.Linq;
 
 namespace PuppeteerSharp
 {
     /// <summary>
     /// <seealso cref="CDPSession.MessageReceived"/> arguments.
     /// </summary>
-    public class MessageEventArgs
+    public class MessageEventArgs : EventArgs
     {
         /// <summary>
         /// Gets the message identifier.
diff --git a/lib/PuppeteerSharp/Messaging/FetchEnableRequest.cs b/lib/PuppeteerSharp/Messaging/FetchEnableRequest.cs
@@ -6,9 +6,9 @@ internal class FetchEnableRequest
 
         public Pattern[] Patterns { get; set; }
 
-        internal class Pattern
+        internal class Pattern(string urlPattern)
         {
-            public string UrlPattern { get; set; }
+            public string UrlPattern { get; set; } = urlPattern;
         }
     }
 }
diff --git a/lib/PuppeteerSharp/Messaging/FetchRequestPausedResponse.cs b/lib/PuppeteerSharp/Messaging/FetchRequestPausedResponse.cs
@@ -1,11 +1,7 @@
 namespace PuppeteerSharp.Messaging
 {
-    internal class FetchRequestPausedResponse
+    internal class FetchRequestPausedResponse : RequestWillBeSentPayload
     {
-        public string RequestId { get; set; }
-
-        public Payload Request { get; set; }
-
-        public string NetworkId { get; set; }
+        public ResourceType? ResourceType { get; set; }
     }
 }
diff --git a/lib/PuppeteerSharp/Messaging/NetworkSetCacheDisabledRequest.cs b/lib/PuppeteerSharp/Messaging/NetworkSetCacheDisabledRequest.cs
@@ -1,7 +1,7 @@
 namespace PuppeteerSharp.Messaging
 {
-    internal class NetworkSetCacheDisabledRequest
+    internal class NetworkSetCacheDisabledRequest(bool cacheDisabled)
     {
-        public bool CacheDisabled { get; set; }
+        public bool CacheDisabled { get; set; } = cacheDisabled;
     }
 }
diff --git a/lib/PuppeteerSharp/Messaging/NetworkSetExtraHTTPHeadersRequest.cs b/lib/PuppeteerSharp/Messaging/NetworkSetExtraHTTPHeadersRequest.cs
@@ -2,8 +2,8 @@
 
 namespace PuppeteerSharp.Messaging
 {
-    internal class NetworkSetExtraHTTPHeadersRequest
+    internal class NetworkSetExtraHTTPHeadersRequest(Dictionary<string, string> headers)
     {
-        public Dictionary<string, string> Headers { get; set; }
+        public Dictionary<string, string> Headers { get; set; } = headers;
     }
 }
diff --git a/lib/PuppeteerSharp/Messaging/RequestWillBeSentPayload.cs b/lib/PuppeteerSharp/Messaging/RequestWillBeSentPayload.cs
@@ -18,6 +18,8 @@ internal class RequestWillBeSentPayload
 
         public bool RedirectHasExtraInfo { get; set; }
 
+        public string NetworkId { get; set; }
+
         public Initiator Initiator { get; set; }
     }
 }
diff --git a/lib/PuppeteerSharp/NetworkManager.cs b/lib/PuppeteerSharp/NetworkManager.cs
diff --git a/lib/PuppeteerSharp/Page.cs b/lib/PuppeteerSharp/Page.cs
diff --git a/lib/PuppeteerSharp/Properties/AssemblyInfo.cs b/lib/PuppeteerSharp/Properties/AssemblyInfo.cs
diff --git a/lib/PuppeteerSharp/Request.cs b/lib/PuppeteerSharp/Request.cs
diff --git a/lib/PuppeteerSharp/WebWorker.cs b/lib/PuppeteerSharp/WebWorker.cs

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@`
`2`	`2`	`using System.Collections.Concurrent;`
`3`	`3`	`using System.Collections.Generic;`
`4`	`4`	`using System.Linq;`
	`5`	`+using System.Threading;`
`5`	`6`	`using System.Threading.Tasks;`
`6`	`7`	`using Microsoft.Extensions.Logging;`
`7`	`8`	`using Newtonsoft.Json.Linq;`
`@@ -15,6 +16,7 @@ public class CDPSession : ICDPSession`
`15`	`16`	`{`
`16`	`17`	`private readonly ConcurrentDictionary<int, MessageTask> _callbacks = new();`
`17`	`18`	`private readonly string _parentSessionId;`
	`19`	`+ private int _lastId;`
`18`	`20`
`19`	`21`	`internal CDPSession(Connection connection, TargetType targetType, string sessionId, string parentSessionId)`
`20`	`22`	`{`
`@@ -81,7 +83,7 @@ public async Task<JObject> SendAsync(string method, object args = null, bool wai`
`81`	`83`	`CloseReason);`
`82`	`84`	`}`
`83`	`85`
`84`		`- var id = Connection.GetMessageID();`
	`86`	`+ var id = GetMessageID();`
`85`	`87`	`var message = Connection.GetMessage(id, method, args, Id);`
`86`	`88`
`87`	`89`	`MessageTask callback = null;`
`@@ -148,6 +150,8 @@ internal void OnMessage(ConnectionResponse obj)`
`148`	`150`	`}`
`149`	`151`	`}`
`150`	`152`
	`153`	`+ internal int GetMessageID() => Interlocked.Increment(ref _lastId);`
	`154`	`+`
`151`	`155`	`internal void Close(string closeReason)`
`152`	`156`	`{`
`153`	`157`	`if (IsClosed)`
Original file line number	Diff line number	Diff line change
`@@ -288,7 +288,7 @@ private async Task OnAttachedToTargetAsync(object sender, TargetAttachedToTarget`
`288`	`288`	`_attachedTargetsBySessionId.TryAdd(session.Id, target);`
`289`	`289`	`}`
`290`	`290`
`291`		`- (parentConnection as CDPSession)?.OnSessionReady(session);`
	`291`	`+ (parentSession ?? parentConnection as CDPSession)?.OnSessionReady(session);`
`292`	`292`
`293`	`293`	`await EnsureTargetsIdsForInitAsync().ConfigureAwait(false);`
`294`	`294`	`_targetsIdsForInit.Remove(target.TargetId);`