Merge pull request #946 from lightpanda-io/request_interception

Request Interception

Author: karlseguin

src/browser/ScriptManager.zig

@@ -24,8 +24,8 @@ const parser = @import("netsurf.zig");
 const Env = @import("env.zig").Env;
 const Page = @import("page.zig").Page;
 const DataURI = @import("DataURI.zig");
+const Http = @import("../http/Http.zig");
 const Browser = @import("browser.zig").Browser;
-const HttpClient = @import("../http/Client.zig");
 const URL = @import("../url.zig").URL;
 
 const Allocator = std.mem.Allocator;
@@ -57,7 +57,7 @@ deferreds: OrderList,
 
 shutdown: bool = false,
 
-client: *HttpClient,
+client: *Http.Client,
 allocator: Allocator,
 buffer_pool: BufferPool,
 script_pool: std.heap.MemoryPool(PendingScript),
@@ -229,7 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
 
     errdefer pending_script.deinit();
 
-    var headers = try HttpClient.Headers.init();
+    var headers = try Http.Headers.init();
     try page.requestCookie(.{}).headersForRequest(page.arena, remote_url.?, &headers);
 
     try self.client.request(.{
@@ -238,6 +238,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
         .method = .GET,
         .headers = headers,
         .cookie_jar = page.cookie_jar,
+        .resource_type = .script,
         .start_callback = if (log.enabled(.http, .debug)) startCallback else null,
         .header_done_callback = headerCallback,
         .data_callback = dataCallback,
@@ -296,7 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
         .buffer_pool = &self.buffer_pool,
     };
 
-    var headers = try HttpClient.Headers.init();
+    var headers = try Http.Headers.init();
     try self.page.requestCookie(.{}).headersForRequest(self.page.arena, url, &headers);
 
     var client = self.client;
@@ -306,6 +307,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
         .headers = headers,
         .cookie_jar = self.page.cookie_jar,
         .ctx = &blocking,
+        .resource_type = .script,
         .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null,
         .header_done_callback = Blocking.headerCallback,
         .data_callback = Blocking.dataCallback,
@@ -423,15 +425,15 @@ fn getList(self: *ScriptManager, script: *const Script) *OrderList {
     return &self.scripts;
 }
 
-fn startCallback(transfer: *HttpClient.Transfer) !void {
+fn startCallback(transfer: *Http.Transfer) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.startCallback(transfer) catch |err| {
         log.err(.http, "SM.startCallback", .{ .err = err, .transfer = transfer });
         return err;
     };
 }
 
-fn headerCallback(transfer: *HttpClient.Transfer) !void {
+fn headerCallback(transfer: *Http.Transfer) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.headerCallback(transfer) catch |err| {
         log.err(.http, "SM.headerCallback", .{
@@ -443,7 +445,7 @@ fn headerCallback(transfer: *HttpClient.Transfer) !void {
     };
 }
 
-fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.dataCallback(transfer, data) catch |err| {
         log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
@@ -488,12 +490,12 @@ const PendingScript = struct {
         }
     }
 
-    fn startCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
+    fn startCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
         _ = self;
         log.debug(.http, "script fetch start", .{ .req = transfer });
     }
 
-    fn headerCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
+    fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
         const header = &transfer.response_header.?;
         log.debug(.http, "script header", .{
             .req = transfer,
@@ -513,7 +515,7 @@ const PendingScript = struct {
         self.script.source = .{ .remote = self.manager.buffer_pool.get() };
     }
 
-    fn dataCallback(self: *PendingScript, transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn dataCallback(self: *PendingScript, transfer: *Http.Transfer, data: []const u8) !void {
         _ = transfer;
         // too verbose
         // log.debug(.http, "script data chunk", .{
@@ -766,11 +768,11 @@ const Blocking = struct {
         done: BlockingResult,
     };
 
-    fn startCallback(transfer: *HttpClient.Transfer) !void {
+    fn startCallback(transfer: *Http.Transfer) !void {
         log.debug(.http, "script fetch start", .{ .req = transfer, .blocking = true });
     }
 
-    fn headerCallback(transfer: *HttpClient.Transfer) !void {
+    fn headerCallback(transfer: *Http.Transfer) !void {
         const header = &transfer.response_header.?;
         log.debug(.http, "script header", .{
             .req = transfer,
@@ -787,7 +789,7 @@ const Blocking = struct {
         self.buffer = self.buffer_pool.get();
     }
 
-    fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         // too verbose
         // log.debug(.http, "script data chunk", .{
         //     .req = transfer,

src/browser/page.zig

@@ -30,7 +30,7 @@ const Renderer = @import("renderer.zig").Renderer;
 const Window = @import("html/window.zig").Window;
 const Walker = @import("dom/walker.zig").WalkerDepthFirst;
 const Scheduler = @import("Scheduler.zig");
-const HttpClient = @import("../http/Client.zig");
+const Http = @import("../http/Http.zig");
 const ScriptManager = @import("ScriptManager.zig");
 const HTMLDocument = @import("html/document.zig").HTMLDocument;
 
@@ -87,13 +87,23 @@ pub const Page = struct {
     polyfill_loader: polyfill.Loader = .{},
 
     scheduler: Scheduler,
-    http_client: *HttpClient,
+    http_client: *Http.Client,
     script_manager: ScriptManager,
 
     mode: Mode,
 
     load_state: LoadState = .parsing,
 
+    // Page.wait balances waiting for resources / tasks and producing an output.
+    // Up until a timeout, Page.wait will always wait for inflight or pending
+    // HTTP requests, via the Http.Client.active counter. However, intercepted
+    // requests (via CDP, but it could be anything), aren't considered "active"
+    // connection. So it's possible that we have intercepted requests (which are
+    // pending on some driver to continue/abort) while Http.Client.active == 0.
+    // This boolean exists to supplment Http.Client.active and inform Page.wait
+    // of pending connections.
+    request_intercepted: bool = false,
+
     const Mode = union(enum) {
         pre: void,
         err: anyerror,
@@ -275,16 +285,26 @@ pub const Page = struct {
         while (true) {
             SW: switch (self.mode) {
                 .pre, .raw => {
+                    if (self.request_intercepted) {
+                        // the page request was intercepted.
+
+                        // there shouldn't be any active requests;
+                        std.debug.assert(http_client.active == 0);
+
+                        // nothing we can do for this, need to kick the can up
+                        // the chain and wait for activity (e.g. a CDP message)
+                        // to unblock this.
+                        return;
+                    }
+
                     // The main page hasn't started/finished navigating.
                     // There's no JS to run, and no reason to run the scheduler.
-
                     if (http_client.active == 0) {
                         // haven't started navigating, I guess.
                         return;
                     }
 
                     // There should only be 1 active http transfer, the main page
-                    std.debug.assert(http_client.active == 1);
                     try http_client.tick(ms_remaining);
                 },
                 .html, .parsed => {
@@ -330,20 +350,35 @@ pub const Page = struct {
 
                     _ = try scheduler.runLowPriority();
 
-                    // We'll block here, waiting for network IO. We know
-                    // when the next timeout is scheduled, and we know how long
-                    // the caller wants to wait for, so we can pick a good wait
-                    // duration
-                    const ms_to_wait = @min(ms_remaining, ms_to_next_task orelse 1000);
+                    const request_intercepted = self.request_intercepted;
+
+                    // We want to prioritize processing intercepted requests
+                    // because, the sooner they get unblocked, the sooner we
+                    // can start the HTTP request. But we still want to advanced
+                    // existing HTTP requests, if possible. So, if we have
+                    // intercepted requests, we'll still look at existing HTTP
+                    // requests, but we won't block waiting for more data.
+                    const ms_to_wait =
+                        if (request_intercepted) 0
+
+                        // But if we have no intercepted requests, we'll wait
+                        // for as long as we can for data to our existing
+                        // inflight requests
+                        else @min(ms_remaining, ms_to_next_task orelse 1000);
+
                     try http_client.tick(ms_to_wait);
 
-                    if (try_catch.hasCaught()) {
-                        const msg = (try try_catch.err(self.arena)) orelse "unknown";
-                        log.warn(.user_script, "page wait", .{ .err = msg, .src = "data" });
-                        return error.JsError;
+                    if (request_intercepted) {
+                        // Again, proritizing intercepted requests. Exit this
+                        // loop so that our caller can hopefully resolve them
+                        // (i.e. continue or abort them);
+                        return;
                     }
                 },
-                .err => |err| return err,
+                .err => |err| {
+                    self.mode = .{ .raw_done = @errorName(err) };
+                    return err;
+                },
                 .raw_done => return,
             }
 
@@ -362,7 +397,7 @@ pub const Page = struct {
             std.debug.print("\nactive requests: {d}\n", .{self.http_client.active});
             var n_ = self.http_client.handles.in_use.first;
             while (n_) |n| {
-                const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| {
+                const transfer = Http.Transfer.fromEasy(n.data.conn.easy) catch |err| {
                     std.debug.print(" - failed to load transfer: {any}\n", .{err});
                     break;
                 };
@@ -435,7 +470,7 @@ pub const Page = struct {
         is_http: bool = true,
         is_navigation: bool = false,
     };
-    pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie {
+    pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie {
         return .{
             .jar = self.cookie_jar,
             .origin = &self.url.uri,
@@ -473,7 +508,7 @@ pub const Page = struct {
         const owned_url = try self.arena.dupeZ(u8, request_url);
         self.url = try URL.parse(owned_url, null);
 
-        var headers = try HttpClient.Headers.init();
+        var headers = try Http.Headers.init();
         if (opts.header) |hdr| try headers.add(hdr);
         try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers);
 
@@ -484,6 +519,7 @@ pub const Page = struct {
             .headers = headers,
             .body = opts.body,
             .cookie_jar = self.cookie_jar,
+            .resource_type = .document,
             .header_done_callback = pageHeaderDoneCallback,
             .data_callback = pageDataCallback,
             .done_callback = pageDoneCallback,
@@ -563,7 +599,7 @@ pub const Page = struct {
         );
     }
 
-    fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
+    fn pageHeaderDoneCallback(transfer: *Http.Transfer) !void {
         var self: *Page = @alignCast(@ptrCast(transfer.ctx));
 
         // would be different than self.url in the case of a redirect
@@ -578,7 +614,7 @@ pub const Page = struct {
         });
     }
 
-    fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn pageDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         var self: *Page = @alignCast(@ptrCast(transfer.ctx));
 
         if (self.mode == .pre) {
@@ -1002,7 +1038,7 @@ pub const NavigateReason = enum {
 pub const NavigateOpts = struct {
     cdp_id: ?i64 = null,
     reason: NavigateReason = .address_bar,
-    method: HttpClient.Method = .GET,
+    method: Http.Method = .GET,
     body: ?[]const u8 = null,
     header: ?[:0]const u8 = null,
 };

src/browser/xhr/xhr.zig

@@ -30,7 +30,7 @@ const URL = @import("../../url.zig").URL;
 const Mime = @import("../mime.zig").Mime;
 const parser = @import("../netsurf.zig");
 const Page = @import("../page.zig").Page;
-const HttpClient = @import("../../http/Client.zig");
+const Http = @import("../../http/Http.zig");
 const CookieJar = @import("../storage/storage.zig").CookieJar;
 
 // XHR interfaces
@@ -80,12 +80,12 @@ const XMLHttpRequestBodyInit = union(enum) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
-    transfer: ?*HttpClient.Transfer = null,
+    transfer: ?*Http.Transfer = null,
     err: ?anyerror = null,
     last_dispatch: i64 = 0,
     send_flag: bool = false,
 
-    method: HttpClient.Method,
+    method: Http.Method,
     state: State,
     url: ?[:0]const u8 = null,
 
@@ -320,7 +320,7 @@ pub const XMLHttpRequest = struct {
     }
 
     const methods = [_]struct {
-        tag: HttpClient.Method,
+        tag: Http.Method,
         name: []const u8,
     }{
         .{ .tag = .DELETE, .name = "DELETE" },
@@ -330,7 +330,7 @@ pub const XMLHttpRequest = struct {
         .{ .tag = .POST, .name = "POST" },
         .{ .tag = .PUT, .name = "PUT" },
     };
-    pub fn validMethod(m: []const u8) DOMError!HttpClient.Method {
+    pub fn validMethod(m: []const u8) DOMError!Http.Method {
         for (methods) |method| {
             if (std.ascii.eqlIgnoreCase(method.name, m)) {
                 return method.tag;
@@ -370,7 +370,7 @@ pub const XMLHttpRequest = struct {
             }
         }
 
-        var headers = try HttpClient.Headers.init();
+        var headers = try Http.Headers.init();
         for (self.headers.items) |hdr| {
             try headers.add(hdr);
         }
@@ -383,6 +383,7 @@ pub const XMLHttpRequest = struct {
             .headers = headers,
             .body = self.request_body,
             .cookie_jar = page.cookie_jar,
+            .resource_type = .xhr,
             .start_callback = httpStartCallback,
             .header_callback = httpHeaderCallback,
             .header_done_callback = httpHeaderDoneCallback,
@@ -392,18 +393,19 @@ pub const XMLHttpRequest = struct {
         });
     }
 
-    fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
+    fn httpStartCallback(transfer: *Http.Transfer) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
         log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" });
         self.transfer = transfer;
     }
 
-    fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: []const u8) !void {
+    fn httpHeaderCallback(transfer: *Http.Transfer, header: Http.Header) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
-        try self.response_headers.append(self.arena, try self.arena.dupe(u8, header));
+        const joined = try std.fmt.allocPrint(self.arena, "{s}: {s}", .{ header.name, header.value });
+        try self.response_headers.append(self.arena, joined);
     }
 
-    fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
+    fn httpHeaderDoneCallback(transfer: *Http.Transfer) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
 
         const header = &transfer.response_header.?;
@@ -433,7 +435,7 @@ pub const XMLHttpRequest = struct {
         self.dispatchEvt("readystatechange");
     }
 
-    fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn httpDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
         try self.response_bytes.appendSlice(self.arena, data);