Add cookie support to browser (not XHR yet) requests

Author: karlseguin

src/browser/browser.zig

@@ -43,7 +43,7 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const HttpClient = @import("../http/client.zig").Client;
+const http = @import("../http/client.zig");
 const UserContext = @import("../user_context.zig").UserContext;
 
 const polyfill = @import("../polyfill/polyfill.zig");
@@ -60,7 +60,7 @@ pub const Browser = struct {
     app: *App,
     session: ?*Session,
     allocator: Allocator,
-    http_client: *HttpClient,
+    http_client: *http.Client,
     session_pool: SessionPool,
     page_arena: std.heap.ArenaAllocator,
 
@@ -123,10 +123,12 @@ pub const Session = struct {
 
     window: Window,
 
-    // TODO move the shed to the browser?
+    // TODO move the shed/jar to the browser?
     storage_shed: storage.Shed,
+    cookie_jar: storage.CookieJar,
+
     page: ?Page = null,
-    http_client: *HttpClient,
+    http_client: *http.Client,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -141,6 +143,7 @@ pub const Session = struct {
             .http_client = browser.http_client,
             .storage_shed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
+            .cookie_jar = storage.CookieJar.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
         };
 
@@ -176,6 +179,7 @@ pub const Session = struct {
         }
         self.env.deinit();
         self.arena.deinit();
+        self.cookie_jar.deinit();
         self.storage_shed.deinit();
     }
 
@@ -364,14 +368,16 @@ pub const Page = struct {
         } });
 
         // load the data
-        var request = try self.session.http_client.request(.GET, self.uri);
+        var request = try self.newHTTPRequest(.GET, self.uri, .{ .navigation = true });
         defer request.deinit();
-        var response = try request.sendSync(.{});
 
+        var response = try request.sendSync(.{});
         const header = response.header;
+        try self.processHTTPResponse(self.uri, &header);
+
         log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        const ct = response.header.get("content-type") orelse {
+        const ct = header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
@@ -607,13 +613,19 @@ pub const Page = struct {
         }
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
-        var request = try self.session.http_client.request(.GET, u);
+        var request = try self.newHTTPRequest(.GET, u, .{
+            .origin = self.uri,
+            .navigation = false,
+        });
         defer request.deinit();
+
         var response = try request.sendSync(.{});
+        var header = response.header;
+        try self.processHTTPResponse(u, &header);
 
-        log.info("fetch {any}: {d}", .{ u, response.header.status });
+        log.info("fetch {any}: {d}", .{ u, header.status });
 
-        if (response.header.status != 200) {
+        if (header.status != 200) {
             return FetchError.BadStatusCode;
         }
 
@@ -638,6 +650,46 @@ pub const Page = struct {
         try s.eval(arena, &self.session.env, body);
     }
 
+    const RequestOpts = struct {
+        origin: ?std.Uri = null,
+        navigation: bool = true,
+    };
+    fn newHTTPRequest(self: *const Page, method: http.Request.Method, uri: std.Uri, opts: RequestOpts) !http.Request {
+        const session = self.session;
+        var request = try session.http_client.request(method, uri);
+        errdefer request.deinit();
+
+        var cookies = try session.cookie_jar.forRequest(
+            self.arena,
+            std.time.timestamp(),
+            opts.origin,
+            uri,
+            opts.navigation,
+        );
+        defer cookies.deinit(self.arena);
+
+        if (cookies.len() > 0) {
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            try cookies.write(arr.writer(self.arena));
+            try request.addHeader("Cookie", arr.items, .{});
+        }
+
+        return request;
+    }
+
+    fn processHTTPResponse(self: *const Page, uri: std.Uri, header: *const http.ResponseHeader) !void {
+        const session = self.session;
+        const now = std.time.timestamp();
+        var it = header.iterate("set-cookie");
+        while (it.next()) |set_cookie| {
+            const c = storage.Cookie.parse(self.arena, uri, set_cookie) catch |err| {
+                log.warn("Couldn't parse cookie '{s}': {}\n", .{ set_cookie, err });
+                continue;
+            };
+            try session.cookie_jar.add(c, now);
+        }
+    }
+
     const Script = struct {
         element: *parser.Element,
         kind: Kind,

src/http/client.zig

@@ -1430,6 +1430,33 @@ pub const ResponseHeader = struct {
     pub fn count(self: *const ResponseHeader) usize {
         return self.headers.items.len;
     }
+
+    pub fn iterate(self: *const ResponseHeader, name: []const u8) HeaderIterator {
+        return .{
+            .index = 0,
+            .name = name,
+            .headers = self.headers,
+        };
+    }
+};
+
+const HeaderIterator = struct {
+    index: usize,
+    name: []const u8,
+    headers: HeaderList,
+
+    pub fn next(self: *HeaderIterator) ?[]const u8 {
+        const name = self.name;
+        const index = self.index;
+        for (self.headers.items[index..], index..) |h, i| {
+            if (std.mem.eql(u8, name, h.name)) {
+                self.index = i + 1;
+                return h.value;
+            }
+        }
+        self.index = self.headers.items.len;
+        return null;
+    }
 };
 
 // What we emit from the AsyncHandler
@@ -2031,6 +2058,52 @@ test "HttpClient: async redirect plaintext to TLS" {
     }
 }
 
+test "HttpClient: HeaderIterator" {
+    var header = ResponseHeader{};
+    defer header.headers.deinit(testing.allocator);
+
+    {
+        var it = header.iterate("nope");
+        try testing.expectEqual(null, it.next());
+        try testing.expectEqual(null, it.next());
+    }
+
+    try header.headers.append(testing.allocator, .{ .name = "h1", .value = "value1" });
+    try header.headers.append(testing.allocator, .{ .name = "h2", .value = "value2" });
+    try header.headers.append(testing.allocator, .{ .name = "h3", .value = "value3" });
+    try header.headers.append(testing.allocator, .{ .name = "h1", .value = "value4" });
+    try header.headers.append(testing.allocator, .{ .name = "h1", .value = "value5" });
+
+    {
+        var it = header.iterate("nope");
+        try testing.expectEqual(null, it.next());
+        try testing.expectEqual(null, it.next());
+    }
+
+    {
+        var it = header.iterate("h2");
+        try testing.expectEqual("value2", it.next());
+        try testing.expectEqual(null, it.next());
+        try testing.expectEqual(null, it.next());
+    }
+
+    {
+        var it = header.iterate("h3");
+        try testing.expectEqual("value3", it.next());
+        try testing.expectEqual(null, it.next());
+        try testing.expectEqual(null, it.next());
+    }
+
+    {
+        var it = header.iterate("h1");
+        try testing.expectEqual("value1", it.next());
+        try testing.expectEqual("value4", it.next());
+        try testing.expectEqual("value5", it.next());
+        try testing.expectEqual(null, it.next());
+        try testing.expectEqual(null, it.next());
+    }
+}
+
 const TestResponse = struct {
     status: u16,
     keepalive: ?bool,

src/storage/cookie.zig

@@ -65,7 +65,7 @@ pub const Jar = struct {
         const same_site = try areSameSite(origin_uri, target_host);
         const is_secure = std.mem.eql(u8, target_uri.scheme, "https");
 
-        var matching: std.ArrayListUnmanaged(*Cookie) = .{};
+        var matching: std.ArrayListUnmanaged(*const Cookie) = .{};
 
         var i: usize = 0;
         var cookies = self.cookies.items;
@@ -146,15 +146,41 @@ pub const Jar = struct {
 };
 
 pub const CookieList = struct {
-    _cookies: std.ArrayListUnmanaged(*Cookie),
+    _cookies: std.ArrayListUnmanaged(*const Cookie) = .{},
 
     pub fn deinit(self: *CookieList, allocator: Allocator) void {
         self._cookies.deinit(allocator);
     }
 
-    pub fn cookies(self: *const CookieList) []*Cookie {
+    pub fn cookies(self: *const CookieList) []*const Cookie {
         return self._cookies.items;
     }
+
+    pub fn len(self: *const CookieList) usize {
+        return self._cookies.items.len;
+    }
+
+    pub fn write(self: *const CookieList, writer: anytype) !void {
+        const all = self._cookies.items;
+        if (all.len == 0) {
+            return;
+        }
+        try writeCookie(all[0], writer);
+        for (all[1..]) |cookie| {
+            try writer.writeAll("; ");
+            try writeCookie(cookie, writer);
+        }
+    }
+
+    fn writeCookie(cookie: *const Cookie, writer: anytype) !void {
+        if (cookie.name.len > 0) {
+            try writer.writeAll(cookie.name);
+            try writer.writeByte('=');
+        }
+        if (cookie.value.len > 0) {
+            try writer.writeAll(cookie.value);
+        }
+    }
 };
 
 fn isCookieExpired(cookie: *const Cookie, now: i64) bool {
@@ -728,6 +754,40 @@ test "Jar: forRequest" {
     // the 'global2' cookie
 }
 
+test "CookieList: write" {
+    var arr: std.ArrayListUnmanaged(u8) = .{};
+    defer arr.deinit(testing.allocator);
+
+    var cookie_list = CookieList{};
+    defer cookie_list.deinit(testing.allocator);
+
+    const c1 = try Cookie.parse(testing.allocator, test_uri, "cookie_name=cookie_value");
+    defer c1.deinit();
+    {
+        try cookie_list._cookies.append(testing.allocator, &c1);
+        try cookie_list.write(arr.writer(testing.allocator));
+        try testing.expectEqual("cookie_name=cookie_value", arr.items);
+    }
+
+    const c2 = try Cookie.parse(testing.allocator, test_uri, "x84");
+    defer c2.deinit();
+    {
+        arr.clearRetainingCapacity();
+        try cookie_list._cookies.append(testing.allocator, &c2);
+        try cookie_list.write(arr.writer(testing.allocator));
+        try testing.expectEqual("cookie_name=cookie_value; x84", arr.items);
+    }
+
+    const c3 = try Cookie.parse(testing.allocator, test_uri, "nope=");
+    defer c3.deinit();
+    {
+        arr.clearRetainingCapacity();
+        try cookie_list._cookies.append(testing.allocator, &c3);
+        try cookie_list.write(arr.writer(testing.allocator));
+        try testing.expectEqual("cookie_name=cookie_value; x84; nope=", arr.items);
+    }
+}
+
 test "Cookie: parse key=value" {
     try expectError(error.Empty, null, "");
     try expectError(error.InvalidByteSequence, null, &.{ 'a', 30, '=', 'b' });

src/storage/storage.zig

@@ -25,6 +25,10 @@ const DOMError = @import("netsurf").DOMError;
 
 const log = std.log.scoped(.storage);
 
+const cookie = @import("cookie.zig");
+pub const Cookie = cookie.Cookie;
+pub const CookieJar = cookie.Jar;
+
 pub const Interfaces = .{
     Bottle,
 };