Frame.GoTo and Frame.Navigate

* started work on frame navigation methods. currently the test just times out

* implement #674. Page tests are passing. Frame test still hanging.

* Update lib/PuppeteerSharp/NetworkManager.cs

fix extra line

* - remove unused variable after merge conflict
- missing frame detached handler

* - added test for frame detaching

* - rename navigate to navigateasync
- rename test to reflect class being tested
- added more tests
- added more docs

* - add test for waitfornaviationasync (second test broken)
- some more clean up

* add final set of test

* - PR feedback

* - add missing assert

Author: bdurrani

lib/PuppeteerSharp.Tests/FrameTests/ContextTests.cs renamed to lib/PuppeteerSharp.Tests/FrameTests/ExecutionContextTests.cs

@@ -5,9 +5,9 @@
 namespace PuppeteerSharp.Tests.FrameTests
 {
     [Collection("PuppeteerLoaderFixture collection")]
-    public class ContextTests : PuppeteerPageBaseTest
+    public class ExecutionContextTests : PuppeteerPageBaseTest
     {
-        public ContextTests(ITestOutputHelper output) : base(output)
+        public ExecutionContextTests(ITestOutputHelper output) : base(output)
         {
         }
 

lib/PuppeteerSharp.Tests/FrameTests/GoToTests.cs

@@ -0,0 +1,83 @@
+using System.Collections.Generic;
+using System.Net;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace PuppeteerSharp.Tests.FrameTests
+{
+    [Collection("PuppeteerLoaderFixture collection")]
+    public class GoToTests : PuppeteerPageBaseTest
+    {
+        public GoToTests(ITestOutputHelper output) : base(output)
+        {
+        }
+
+        [Fact]
+        public async Task ShouldNavigateSubFrames()
+        {
+            await Page.GoToAsync(TestConstants.ServerUrl + "/frames/one-frame.html");
+            Assert.Contains("/frames/one-frame.html", Page.Frames[0].Url);
+            Assert.Contains("/frames/frame.html", Page.Frames[1].Url);
+            var response = await Page.Frames[1].GoToAsync(TestConstants.EmptyPage);
+            Assert.Equal(HttpStatusCode.OK, response.Status);
+            Assert.Same(response.Frame, Page.Frames[1]);
+        }
+
+        [Fact]
+        public async Task ShouldRejectWhenFrameDetaches()
+        {
+            await Page.GoToAsync(TestConstants.ServerUrl + "/frames/one-frame.html");
+            Server.SetRoute("/empty.html", context => Task.Delay(10000));
+            var waitForRequestTask = Server.WaitForRequest("/empty.html");
+            var navigationTask = Page.Frames[1].GoToAsync(TestConstants.EmptyPage);
+            await waitForRequestTask;
+            await Page.QuerySelectorAsync("iframe").EvaluateFunctionAsync("frame => frame.remove()");
+            var exception = await Assert.ThrowsAsync<NavigationException>(async () => await navigationTask);
+            Assert.Equal("Navigating frame was detached", exception.Message);
+        }
+
+        [Fact]
+        public async Task ShouldReturnMatchingResponses()
+        {
+            // Disable cache: otherwise, chromium will cache similar requests.
+            await Page.SetCacheEnabledAsync(false);
+            await Page.GoToAsync(TestConstants.EmptyPage);
+            // Attach three frames.
+            var frameTasks = new List<Task<Frame>>
+            {
+                FrameUtils.AttachFrameAsync(Page, "frame1", TestConstants.EmptyPage),
+                FrameUtils.AttachFrameAsync(Page, "frame2", TestConstants.EmptyPage),
+                FrameUtils.AttachFrameAsync(Page, "frame3", TestConstants.EmptyPage)
+            };
+            await Task.WhenAll(frameTasks);
+
+            // Navigate all frames to the same URL.
+            var serverResponses = new List<TaskCompletionSource<string>>();
+            Server.SetRoute("/one-style.html", async (context) =>
+            {
+                var tcs = new TaskCompletionSource<string>();
+                serverResponses.Add(tcs);
+                await context.Response.WriteAsync(await tcs.Task);
+            });
+
+            var navigations = new List<Task<Response>>();
+            for (var i = 0; i < 3; ++i)
+            {
+                var waitRequestTask = Server.WaitForRequest("/one-style.html");
+                navigations.Add(frameTasks[i].Result.GoToAsync(TestConstants.ServerUrl + "/one-style.html"));
+                await waitRequestTask;
+            }
+            // Respond from server out-of-order.
+            var serverResponseTexts = new string[] { "AAA", "BBB", "CCC" };
+            for (var i = 0; i < 3; ++i)
+            {
+                serverResponses[i].TrySetResult(serverResponseTexts[i]);
+                var response = await navigations[i];
+                Assert.Same(frameTasks[i].Result, response.Frame);
+                Assert.Equal(serverResponseTexts[i], await response.TextAsync());
+            }
+        }
+    }
+}

lib/PuppeteerSharp.Tests/FrameTests/WaitForNavigationTests.cs

@@ -0,0 +1,48 @@
+using System.Net;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace PuppeteerSharp.Tests.FrameTests
+{
+    [Collection("PuppeteerLoaderFixture collection")]
+    public class WaitForNavigationTests : PuppeteerPageBaseTest
+    {
+        public WaitForNavigationTests(ITestOutputHelper output) : base(output)
+        {
+        }
+
+        [Fact]
+        public async Task ShouldWork()
+        {
+            await Page.GoToAsync(TestConstants.ServerUrl + "/frames/one-frame.html");
+            var frame = Page.Frames[1];
+            var waitForNavigationResult = frame.WaitForNavigationAsync();
+
+            await Task.WhenAll(
+                waitForNavigationResult,
+                frame.EvaluateFunctionAsync("url => window.location.href = url", TestConstants.ServerUrl + "/grid.html")
+            );
+            var response = await waitForNavigationResult;
+            Assert.Equal(HttpStatusCode.OK, response.Status);
+            Assert.Contains("grid.html", response.Url); 
+            Assert.Same(frame, response.Frame);
+            Assert.Contains("/frames/one-frame.html", Page.Url);
+        }
+
+        [Fact]
+        public async Task ShouldRejectWhenFrameDetaches()
+        {
+            await Page.GoToAsync(TestConstants.ServerUrl + "/frames/one-frame.html");
+            var frame = Page.Frames[1];
+            Server.SetRoute("/empty.html", context => Task.Delay(10000));
+            var waitForNavigationResult = frame.WaitForNavigationAsync();
+            await Task.WhenAll(
+             Server.WaitForRequest("/empty.html"),
+            frame.EvaluateFunctionAsync($"() => window.location = '{TestConstants.EmptyPage}'"));
+
+            await Page.QuerySelectorAsync("iframe").EvaluateFunctionAsync("frame => frame.remove()");
+            var response = await waitForNavigationResult;
+        } 
+    }
+}

lib/PuppeteerSharp.Tests/FrameUtils.cs

@@ -1,20 +1,21 @@
-using System;
-using System.Text.RegularExpressions;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
 namespace PuppeteerSharp.Tests
 {
     public static class FrameUtils
     {
-        public static async Task AttachFrameAsync(Page page, string frameId, string url)
+        public static async Task<Frame> AttachFrameAsync(Page page, string frameId, string url)
         {
-            await page.EvaluateFunctionAsync(@"(frameId, url) => {
+            var handle = await page.EvaluateFunctionHandleAsync(@" async (frameId, url) => {
               const frame = document.createElement('iframe');
               frame.src = url;
               frame.id = frameId;
               document.body.appendChild(frame);
-              return new Promise(x => frame.onload = x);
-            }", frameId, url);
+              await new Promise(x => frame.onload = x);
+              return frame
+            }", frameId, url) as ElementHandle;
+            return await handle.ContentFrameAsync();
         }
 
         public static async Task DetachFrameAsync(Page page, string frameId)

lib/PuppeteerSharp/Frame.cs

@@ -45,7 +45,7 @@ public class Frame
         internal List<WaitTask> WaitTasks { get; }
         internal string Id { get; set; }
         internal string LoaderId { get; set; }
-        internal List<string> LifecycleEvents { get; }
+        internal List<string> LifecycleEvents { get; }
         internal string NavigationURL { get; private set; }
 
         internal Frame(FrameManager frameManager, CDPSession client, Frame parentFrame, string frameId)
@@ -63,7 +63,7 @@ internal Frame(FrameManager frameManager, CDPSession client, Frame parentFrame,
             SetDefaultContext(null);
 
             WaitTasks = new List<WaitTask>();
-            LifecycleEvents = new List<string>();
+            LifecycleEvents = new List<string>();
         }
 
         #region Properties
@@ -98,6 +98,50 @@ internal Frame(FrameManager frameManager, CDPSession client, Frame parentFrame,
 
         #region Public Methods
 
+        /// <summary>
+        /// Navigates to an url
+        /// </summary>        
+        /// <param name="url">URL to navigate page to. The url should include scheme, e.g. https://.</param>
+        /// <param name="options">Navigation parameters.</param>
+        /// <returns>Task which resolves to the main resource response. In case of multiple redirects, the navigation will resolve with the response of the last redirect.</returns>
+        /// <seealso cref="GoToAsync(string, int?, WaitUntilNavigation[])"/>
+        public Task<Response> GoToAsync(string url, NavigationOptions options) => FrameManager.NavigateFrameAsync(this, url, options);
+
+        /// <summary>
+        /// Navigates to an url
+        /// </summary>
+        /// <param name="url">URL to navigate page to. The url should include scheme, e.g. https://.</param>
+        /// <param name="timeout">maximum navigation time in milliseconds. Defaults to 30 seconds. Pass 0
+        /// to disable timeout. The default value can be changed by using the <see cref="Page.DefaultNavigationTimeout"/>
+        /// property.</param>
+        /// <param name="waitUntil">When to consider navigation succeeded, defaults to <see cref="WaitUntilNavigation.Load"/>. Given an array of <see cref="WaitUntilNavigation"/>, navigation is considered to be successful after all events have been fired</param>
+        /// <returns>Task which resolves to the main resource response. In case of multiple redirects, the navigation will resolve with the response of the last redirect</returns>
+        public Task<Response> GoToAsync(string url, int? timeout = null, WaitUntilNavigation[] waitUntil = null)
+            => GoToAsync(url, new NavigationOptions { Timeout = timeout, WaitUntil = waitUntil });
+
+        /// <summary>
+        /// This resolves when the frame navigates to a new URL or reloads.
+        /// It is useful for when you run code which will indirectly cause the frame to navigate.
+        /// </summary>
+        /// <param name="options">navigation options</param>
+        /// <returns>Task which resolves to the main resource response. 
+        /// In case of multiple redirects, the navigation will resolve with the response of the last redirect.
+        /// In case of navigation to a different anchor or navigation due to History API usage, the navigation will resolve with `null`.
+        /// </returns>
+        /// <remarks>
+        /// Usage of the <c>History API</c> <see href="https://developer.mozilla.org/en-US/docs/Web/API/History_API"/> to change the URL is considered a navigation
+        /// </remarks>
+        /// <example>
+        /// <code>
+        /// <![CDATA[
+        /// var navigationTask =frame.page.WaitForNavigationAsync();
+        /// await frame.ClickAsync("a.my-link");
+        /// await navigationTask;
+        /// ]]>
+        /// </code>
+        /// </example>
+        public Task<Response> WaitForNavigationAsync(NavigationOptions options = null) => FrameManager.WaitForFrameNavigationAsync(this, options);
+
         /// <summary>
         /// Executes a script in browser context
         /// </summary>

lib/PuppeteerSharp/FrameManager.cs

@@ -1,8 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.Contracts;
+using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json.Linq;
+using PuppeteerSharp.Helpers;
 using PuppeteerSharp.Messaging;
 
 namespace PuppeteerSharp
@@ -11,21 +12,25 @@ internal class FrameManager
     {
         private readonly CDPSession _client;
         private Dictionary<int, ExecutionContext> _contextIdToContext;
+        private bool _ensureNewDocumentNavigation;
         private readonly ILogger _logger;
+        private readonly NetworkManager _networkManager;
 
-        internal FrameManager(CDPSession client, FrameTree frameTree, Page page)
+        internal FrameManager(CDPSession client, FrameTree frameTree, Page page, NetworkManager networkManager)
         {
             _client = client;
             Page = page;
             Frames = new Dictionary<string, Frame>();
             _contextIdToContext = new Dictionary<int, ExecutionContext>();
             _logger = _client.Connection.LoggerFactory.CreateLogger<FrameManager>();
+            _networkManager = networkManager;
 
             _client.MessageReceived += _client_MessageReceived;
             HandleFrameTree(frameTree);
         }
 
         #region Properties
+
         internal event EventHandler<FrameEventArgs> FrameAttached;
         internal event EventHandler<FrameEventArgs> FrameDetached;
         internal event EventHandler<FrameEventArgs> FrameNavigated;
@@ -35,6 +40,7 @@ internal FrameManager(CDPSession client, FrameTree frameTree, Page page)
         internal Dictionary<string, Frame> Frames { get; set; }
         internal Frame MainFrame { get; set; }
         internal Page Page { get; }
+        internal int DefaultNavigationTimeout { get; set; } = 30000;
 
         #endregion
 
@@ -51,6 +57,89 @@ internal ExecutionContext ExecutionContextById(int contextId)
             return context;
         }
 
+        public async Task<Response> NavigateFrameAsync(Frame frame, string url, NavigationOptions options)
+        {
+            var referrer = string.IsNullOrEmpty(options.Referer)
+               ? _networkManager.ExtraHTTPHeaders?.GetValueOrDefault(MessageKeys.Referer)
+               : options.Referer;
+            var requests = new Dictionary<string, Request>();
+            var timeout = options?.Timeout ?? DefaultNavigationTimeout;
+            var watcher = new NavigatorWatcher(_client, this, frame, _networkManager, timeout, options);
+
+            var navigateTask = NavigateAsync(_client, url, referrer, frame.Id);
+            await Task.WhenAny(
+                watcher.TimeoutOrTerminationTask,
+                navigateTask).ConfigureAwait(false);
+
+            AggregateException exception = null;
+            if (navigateTask.IsFaulted)
+            {
+                exception = navigateTask.Exception;
+            }
+            else
+            {
+                await Task.WhenAny(
+                    watcher.TimeoutOrTerminationTask,
+                    _ensureNewDocumentNavigation ? watcher.NewDocumentNavigationTask : watcher.SameDocumentNavigationTask
+                ).ConfigureAwait(false);
+
+                if (watcher.TimeoutOrTerminationTask.IsCompleted && watcher.TimeoutOrTerminationTask.Result.IsFaulted)
+                {
+                    exception = watcher.TimeoutOrTerminationTask.Result.Exception;
+                }
+            }
+
+            if (exception != null)
+            {
+                throw new NavigationException(exception.InnerException.Message, exception.InnerException);
+            }
+
+            return watcher.NavigationResponse;
+        }
+
+        private async Task NavigateAsync(CDPSession client, string url, string referrer, string frameId)
+        {
+            var response = await client.SendAsync<PageNavigateResponse>("Page.navigate", new
+            {
+                url,
+                referrer = referrer ?? string.Empty,
+                frameId
+            }).ConfigureAwait(false);
+
+            _ensureNewDocumentNavigation = !string.IsNullOrEmpty(response.LoaderId);
+
+            if (!string.IsNullOrEmpty(response.ErrorText))
+            {
+                throw new NavigationException(response.ErrorText, url);
+            }
+        }
+
+        public async Task<Response> WaitForFrameNavigationAsync(Frame frame, NavigationOptions options = null)
+        {
+            var timeout = options?.Timeout ?? DefaultNavigationTimeout;
+            var watcher = new NavigatorWatcher(_client, this, frame, _networkManager, timeout, options);
+
+            var raceTask = await Task.WhenAny(
+                watcher.NewDocumentNavigationTask,
+                watcher.SameDocumentNavigationTask,
+                watcher.TimeoutOrTerminationTask
+            ).ConfigureAwait(false);
+
+            var exception = raceTask.Exception;
+            if (exception == null &&
+                watcher.TimeoutOrTerminationTask.IsCompleted &&
+                watcher.TimeoutOrTerminationTask.Result.IsFaulted)
+            {
+                exception = watcher.TimeoutOrTerminationTask.Result.Exception;
+            }
+            if (exception != null)
+            {
+                throw new NavigationException(exception.Message, exception);
+            }
+
+            return watcher.NavigationResponse;
+        }
+
         #endregion
 
         #region Private Methods