HTTP request notification

- Add 2 internal notifications
  1 - http_request_start
  2 - http_request_complete

- When Network.enable CDP message is received, browser context registers for
  these 2 events (when Network.disable is called, it unregisters)

- On http_request_start, CDP will emit a Network.requestWillBeSent message.
  This _does not_ include all the fields, but what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

- On http_request_complete, CDP will emit a Network.responseReceived message.
  This _does not_ include all the fields, bu what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

We currently don't emit any other new events, including any network-specific
lifecycleEvent (i.e. Chrome will emit an networkIdle and networkAlmostIdle).

To support this, the following other things were done:
- CDP now has a `notification_arena` which is re-used between browser contexts.
  Normally, CDP code runs based on a "cmd" which has its own message_arena, but
  these notifications happen out-of-band, so we needed a new arena which is
  valid for handling 1 notification.

- HTTP Client is notification-aware. The SessionState no longer includes the
  *http.Client directly. It instead includes an http.RequestFactory which is
  the combination fo the client + a specific configuration (i.e. *Notification).
  This ensures that all requests made from that factory have the same settings.

- However, despite the above, _some_ requests do not appear to emit CDP events,
  such as loading a <script src="X">. So the page still deals directly with the
  *http.Client.

- Playwright and Puppeteer (but Playwright in particular) are very sensitive to
  event ordering. These new events have introduced additional sensitivity.
  The result sent to Page.navigate had to be moved to inside the navigate event
  handler, which meant passing some cdp-specific data (the input.id) into the
  NavigateOpts. This is the only way I found to keep both happy - the sequence
  of events is closer (but still pretty far) from what Chrome does.

Author: karlseguin

src/browser/env.zig

@@ -7,7 +7,7 @@ const storage = @import("storage/storage.zig");
 const generate = @import("../runtime/generate.zig");
 const Renderer = @import("renderer.zig").Renderer;
 const Loop = @import("../runtime/loop.zig").Loop;
-const HttpClient = @import("../http/client.zig").Client;
+const RequestFactory = @import("../http/client.zig").RequestFactory;
 
 const WebApis = struct {
     // Wrapped like this for debug ergonomics.
@@ -54,8 +54,8 @@ pub const SessionState = struct {
     window: *Window,
     renderer: *Renderer,
     arena: std.mem.Allocator,
-    http_client: *HttpClient,
     cookie_jar: *storage.CookieJar,
+    request_factory: RequestFactory,
 
     // dangerous, but set by the JS framework
     // shorter-lived than the arena above, which

src/browser/page.zig

@@ -98,7 +98,7 @@ pub const Page = struct {
                 .renderer = &self.renderer,
                 .loop = browser.app.loop,
                 .cookie_jar = &session.cookie_jar,
-                .http_client = browser.http_client,
+                .request_factory = browser.http_client.requestFactory(browser.notification),
             },
             .scope = try session.executor.startScope(&self.window, &self.state, self, true),
             .module_map = .empty,
@@ -174,6 +174,7 @@ pub const Page = struct {
     pub fn navigate(self: *Page, request_url: URL, opts: NavigateOpts) !void {
         const arena = self.arena;
         const session = self.session;
+        const notification = session.browser.notification;
 
         log.debug("starting GET {s}", .{request_url});
 
@@ -195,10 +196,11 @@ pub const Page = struct {
         // load the data
         var request = try self.newHTTPRequest(.GET, &self.url, .{ .navigation = true });
         defer request.deinit();
+        request.notification = notification;
 
-        session.browser.notification.dispatch(.page_navigate, &.{
+        notification.dispatch(.page_navigate, &.{
+            .opts = opts,
             .url = &self.url,
-            .reason = opts.reason,
             .timestamp = timestamp(),
         });
 
@@ -238,7 +240,7 @@ pub const Page = struct {
             self.raw_data = arr.items;
         }
 
-        session.browser.notification.dispatch(.page_navigated, &.{
+        notification.dispatch(.page_navigated, &.{
             .url = &self.url,
             .timestamp = timestamp(),
         });
@@ -464,7 +466,9 @@ pub const Page = struct {
     }
 
     fn newHTTPRequest(self: *const Page, method: http.Request.Method, url: *const URL, opts: storage.cookie.LookupOpts) !http.Request {
-        var request = try self.state.http_client.request(method, &url.uri);
+        // Don't use the state's request_factory here, since requests made by the
+        // page (i.e. to load <scripts>) should not generate notifications.
+        var request = try self.session.browser.http_client.request(method, &url.uri);
         errdefer request.deinit();
 
         var arr: std.ArrayListUnmanaged(u8) = .{};
@@ -661,7 +665,8 @@ pub const NavigateReason = enum {
     address_bar,
 };
 
-const NavigateOpts = struct {
+pub const NavigateOpts = struct {
+    cdp_id: ?i64 = null,
     reason: NavigateReason = .address_bar,
 };
 

src/browser/xhr/xhr.zig

@@ -80,7 +80,6 @@ const XMLHttpRequestBodyInit = union(enum) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
-    client: *http.Client,
     request: ?http.Request = null,
 
     priv_state: PrivState = .new,
@@ -252,7 +251,6 @@ pub const XMLHttpRequest = struct {
             .state = .unsent,
             .url = null,
             .origin_url = session_state.url,
-            .client = session_state.http_client,
             .cookie_jar = session_state.cookie_jar,
         };
     }
@@ -420,7 +418,7 @@ pub const XMLHttpRequest = struct {
         self.send_flag = true;
         self.priv_state = .open;
 
-        self.request = try self.client.request(self.method, &self.url.?.uri);
+        self.request = try session_state.request_factory.create(self.method, &self.url.?.uri);
         var request = &self.request.?;
         errdefer request.deinit();
 

src/cdp/cdp.zig

@@ -69,6 +69,9 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // 1 message at a time.
         message_arena: std.heap.ArenaAllocator,
 
+        // Used for processing notifications within a browser context.
+        notification_arena: std.heap.ArenaAllocator,
+
         const Self = @This();
 
         pub fn init(app: *App, client: TypeProvider.Client) !Self {
@@ -82,6 +85,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .allocator = allocator,
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
+                .notification_arena = std.heap.ArenaAllocator.init(allocator),
             };
         }
 
@@ -91,6 +95,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             }
             self.browser.deinit();
             self.message_arena.deinit();
+            self.notification_arena.deinit();
         }
 
         pub fn handleMessage(self: *Self, msg: []const u8) bool {
@@ -259,7 +264,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             });
         }
 
-        fn sendJSON(self: *Self, message: anytype) !void {
+        pub fn sendJSON(self: *Self, message: anytype) !void {
             return self.client.sendJSON(message, .{
                 .emit_null_optional_fields = false,
             });
@@ -283,6 +288,12 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         // Points to the session arena
         arena: Allocator,
 
+        // From the parent's notification_arena.allocator(). Most of the CDP
+        // code paths deal with a cmd which has its own arena (from the
+        // message_arena). But notifications happen outside of the typical CDP
+        // request->response, and thus don't have a cmd and don't have an arena.
+        notification_arena: Allocator,
+
         // Maps to our Page. (There are other types of targets, but we only
         // deal with "pages" for now). Since we only allow 1 open page at a
         // time, we only have 1 target_id.
@@ -336,6 +347,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                 .node_search_list = undefined,
                 .isolated_world = null,
                 .inspector = inspector,
+                .notification_arena = cdp.notification_arena.allocator(),
             };
             self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
             errdefer self.deinit();
@@ -397,6 +409,16 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             return if (raw_url.len == 0) null else raw_url;
         }
 
+        pub fn networkEnable(self: *Self) !void {
+            try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
+            try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete);
+        }
+
+        pub fn networkDisable(self: *Self) void {
+            self.cdp.browser.notification.unregister(.http_request_start, self);
+            self.cdp.browser.notification.unregister(.http_request_complete, self);
+        }
+
         pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             return @import("domains/page.zig").pageRemove(self);
@@ -409,14 +431,31 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         pub fn onPageNavigate(ctx: *anyopaque, data: *const Notification.PageNavigate) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
-            return @import("domains/page.zig").pageNavigate(self, data);
+            defer self.resetNotificationArena();
+            return @import("domains/page.zig").pageNavigate(self.notification_arena, self, data);
         }
 
         pub fn onPageNavigated(ctx: *anyopaque, data: *const Notification.PageNavigated) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             return @import("domains/page.zig").pageNavigated(self, data);
         }
 
+        pub fn onHttpRequestStart(ctx: *anyopaque, data: *const Notification.RequestStart) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data);
+        }
+
+        pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data);
+        }
+
+        fn resetNotificationArena(self: *Self) void {
+            defer _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });
+        }
+
         pub fn callInspector(self: *const Self, msg: []const u8) void {
             self.inspector.send(msg);
             // force running micro tasks after send input to the inspector.

src/cdp/domains/network.zig

@@ -17,15 +17,130 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
+const Notification = @import("../../notification.zig").Notification;
+
+const Allocator = std.mem.Allocator;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
+        disable,
         setCacheDisabled,
     }, cmd.input.action) orelse return error.UnknownMethod;
 
     switch (action) {
-        .enable => return cmd.sendResult(null, .{}),
+        .enable => return enable(cmd),
+        .disable => return disable(cmd),
         .setCacheDisabled => return cmd.sendResult(null, .{}),
     }
 }
+
+fn enable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    try bc.networkEnable();
+    return cmd.sendResult(null, .{});
+}
+
+fn disable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    bc.networkDisable();
+    return cmd.sendResult(null, .{});
+}
+
+pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+    const page = bc.session.currentPage() orelse unreachable;
+
+    const document_url = try urlToString(arena, &page.url.uri, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_fragment = try urlToString(arena, request.url, .{
+        .fragment = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.requestWillBeSent", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .documentUrl = document_url,
+        .request = .{
+            .url = request_url,
+            .urlFragment = request_fragment,
+            .method = @tagName(request.method),
+            .hasPostData = request.has_body,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+
+    const url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.responseReceived", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .response = .{
+            .url = url,
+            .status = request.status,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
+    var buf: std.ArrayListUnmanaged(u8) = .empty;
+    try url.writeToStream(opts, buf.writer(arena));
+    return buf.items;
+}