First version of headers, no cookies yet

Author: sjorsdonkers

src/browser/ScriptManager.zig

@@ -57,7 +57,6 @@ deferreds: OrderList,
 
 shutdown: bool = false,
 
-
 client: *HttpClient,
 allocator: Allocator,
 buffer_pool: BufferPool,
@@ -230,6 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
         .url = remote_url.?,
         .ctx = pending_script,
         .method = .GET,
+        .headers = try HttpClient.Headers.init(),
         .cookie = page.requestCookie(.{}),
         .start_callback = if (log.enabled(.http, .debug)) startCallback else null,
         .header_done_callback = headerCallback,
@@ -293,6 +293,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
     try client.blockingRequest(.{
         .url = url,
         .method = .GET,
+        .headers = try HttpClient.Headers.init(),
         .ctx = &blocking,
         .cookie = self.page.requestCookie(.{}),
         .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null,

src/browser/page.zig

@@ -433,12 +433,15 @@ pub const Page = struct {
         const owned_url = try self.arena.dupeZ(u8, request_url);
         self.url = try URL.parse(owned_url, null);
 
+        var headers = try HttpClient.Headers.init();
+        if (opts.header) |hdr| try headers.add(hdr);
+
         self.http_client.request(.{
             .ctx = self,
             .url = owned_url,
             .method = opts.method,
+            .headers = headers,
             .body = opts.body,
-            .header = opts.header,
             .cookie = self.requestCookie(.{ .is_navigation = true }),
             .header_done_callback = pageHeaderDoneCallback,
             .data_callback = pageDataCallback,

src/browser/xhr/xhr.zig

@@ -370,10 +370,16 @@ pub const XMLHttpRequest = struct {
             }
         }
 
+        var headers = try HttpClient.Headers.init();
+        for (self.headers.items) |hdr| {
+            try headers.add(hdr);
+        }
+
         try page.http_client.request(.{
             .ctx = self,
             .url = self.url.?,
             .method = self.method,
+            .headers = headers,
             .body = self.request_body,
             .cookie = page.requestCookie(.{}),
             .start_callback = httpStartCallback,
@@ -387,11 +393,6 @@ pub const XMLHttpRequest = struct {
 
     fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
-
-        for (self.headers.items) |hdr| {
-            try transfer.addHeader(hdr);
-        }
-
         log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" });
         self.transfer = transfer;
     }

src/cdp/domains/fetch.zig

@@ -149,11 +149,7 @@ pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notificati
     const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{
         .fragment = true,
     });
-    const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
-    // try headers.ensureTotalCapacity(arena, request.headers.items.len);
-    // for (request.headers.items) |header| {
-    //     headers.putAssumeCapacity(header.name, header.value);
-    // }
+    const headers = try intercept.request.headers.asHashMap(arena);
     // End of duped code
 
     try cdp.sendEvent("Fetch.requestPaused", .{

src/cdp/domains/network.zig

@@ -263,12 +263,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
         .fragment = true, // TODO since path is false, this likely does not work as intended
     });
 
-    // @newhttp
-    const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
-    // try headers.ensureTotalCapacity(arena, request.headers.items.len);
-    // for (request.headers.items) |header| {
-    //     headers.putAssumeCapacity(header.name, header.value);
-    // }
+    const headers = try data.request.headers.asHashMap(arena);
 
     // We're missing a bunch of fields, but, for now, this seems like enough
     try cdp.sendEvent("Network.requestWillBeSent", .{

src/http/Client.zig

@@ -20,6 +20,7 @@ const std = @import("std");
 const log = @import("../log.zig");
 const builtin = @import("builtin");
 const Http = @import("Http.zig");
+pub const Headers = Http.Headers;
 const Notification = @import("../notification.zig").Notification;
 
 const c = Http.c;
@@ -260,7 +261,8 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
         return;
     };
 
-    const header_list = blk: {
+    var header_list = req.headers;
+    {
         errdefer self.handles.release(handle);
         try conn.setMethod(req.method);
         try conn.setURL(req.url);
@@ -269,31 +271,23 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
             try conn.setBody(b);
         }
 
-        var header_list = conn.commonHeaders();
-        errdefer c.curl_slist_free_all(header_list);
-
-        if (req.header) |hdr| {
-            header_list = c.curl_slist_append(header_list, hdr);
-        }
-
-        {
-            const aa = self.arena.allocator();
-            var arr: std.ArrayListUnmanaged(u8) = .{};
-            try req.cookie.forRequest(&uri, arr.writer(aa));
-
-            if (arr.items.len > 0) {
-                try arr.append(aa, 0); //null terminate
+        // { // TODO move up to `fn request()`
+        //     const aa = self.arena.allocator();
+        //     var arr: std.ArrayListUnmanaged(u8) = .{};
+        //     try req.cookie.forRequest(&uri, arr.writer(aa));
 
-                // copies the value
-                header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr));
-                defer _ = self.arena.reset(.{ .retain_with_limit = 2048 });
-            }
-        }
+        //     if (arr.items.len > 0) {
+        //         try arr.append(aa, 0); //null terminate
 
-        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
+        //         // copies the value
+        //         header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr));
+        //         defer _ = self.arena.reset(.{ .retain_with_limit = 2048 });
+        //     }
+        // }
 
-        break :blk header_list;
-    };
+        try conn.secretHeaders(&header_list); // Add headers that must be hidden from intercepts
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers));
+    }
 
     {
         errdefer self.handles.release(handle);
@@ -305,7 +299,7 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
             .req = req,
             .ctx = req.ctx,
             .handle = handle,
-            ._request_header_list = header_list,
+            ._request_header_list = header_list.headers,
         };
         errdefer self.transfer_pool.destroy(transfer);
 
@@ -495,8 +489,8 @@ pub const Request = struct {
     id: ?u64 = null,
     method: Method,
     url: [:0]const u8,
+    headers: Headers,
     body: ?[]const u8 = null,
-    header: ?[:0]const u8 = null,
     cookie: RequestCookie,
 
     // arbitrary data that can be associated with this request

src/http/Http.zig

@@ -185,20 +185,20 @@ pub const Connection = struct {
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, body.ptr));
     }
 
-    pub fn commonHeaders(self: *const Connection) *c.curl_slist {
-        var header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0");
+    // These are headers that may not be send to the users for inteception.
+    pub fn secretHeaders(self: *const Connection, headers: *Headers) !void {
         if (self.opts.proxy_bearer_token) |hdr| {
-            header_list = c.curl_slist_append(header_list, hdr);
+            try headers.add(hdr);
         }
-        return header_list;
     }
 
     pub fn request(self: *const Connection) !u16 {
         const easy = self.easy;
 
-        const header_list = self.commonHeaders();
-        defer c.curl_slist_free_all(header_list);
-        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
+        const header_list = try Headers.init();
+        defer header_list.deinit();
+        try self.secretHeaders(&header_list);
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers));
 
         try errorCheck(c.curl_easy_perform(easy));
         var http_code: c_long = undefined;
@@ -210,6 +210,59 @@ pub const Connection = struct {
     }
 };
 
+pub const Headers = struct {
+    headers: *c.curl_slist,
+
+    pub fn init() !Headers {
+        const header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0");
+        if (header_list == null) return error.OutOfMemory;
+        return .{ .headers = header_list };
+    }
+
+    pub fn deinit(self: *Headers) void {
+        c.curl_slist_free_all(self.headers);
+    }
+
+    pub fn add(self: *Headers, header: [*c]const u8) !void {
+        const updated_headers = c.curl_slist_append(self.headers, header);
+        if (updated_headers == null) return error.OutOfMemory;
+        self.headers = updated_headers;
+    }
+
+    pub fn asHashMap(self: *const Headers, allocator: Allocator) !std.StringArrayHashMapUnmanaged([]const u8) {
+        var list: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+        try list.ensureTotalCapacity(allocator, self.count());
+
+        var current: [*c]c.curl_slist = self.headers;
+        while (current) |node| {
+            const str = std.mem.span(@as([*:0]const u8, @ptrCast(node.*.data)));
+            const header = parseHeader(str) orelse return error.InvalidHeader;
+            list.putAssumeCapacity(header.name, header.value);
+            current = node.*.next;
+        }
+        return list;
+    }
+
+    fn parseHeader(header_str: []const u8) ?struct { name: []const u8, value: []const u8 } {
+        const colon_pos = std.mem.indexOf(u8, header_str, ":") orelse return null;
+
+        const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
+        const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");
+
+        return .{ .name = name, .value = value };
+    }
+
+    pub fn count(self: *const Headers) usize {
+        var current: [*c]c.curl_slist = self.headers;
+        var num: usize = 0;
+        while (current) |node| {
+            num += 1;
+            current = node.*.next;
+        }
+        return num;
+    }
+};
+
 pub fn errorCheck(code: c.CURLcode) errors.Error!void {
     if (code == c.CURLE_OK) {
         return;