Cookies

.gitmodules

@@ -23,10 +23,3 @@
 [submodule "vendor/mimalloc"]
 	path = vendor/mimalloc
 	url = https://github.com/microsoft/mimalloc.git/
-[submodule "vendor/tls.zig"]
-	path = vendor/tls.zig
-	url = https://github.com/ianic/tls.zig.git/
-[submodule "vendor/zig-async-io"]
-	path = vendor/zig-async-io
-	url = https://github.com/lightpanda-io/zig-async-io.git/
-	branch = zig-0.14

LICENSING.md

@@ -10,7 +10,6 @@ The default license for this project is [AGPL-3.0-only](LICENSE).
 The following files are licensed under MIT:
 
 ```
-src/http/Client.zig
 src/polyfill/fetch.js
 ```
 

build.zig

@@ -133,8 +133,8 @@ pub fn build(b: *std.Build) !void {
 
     // compile
     const unit_tests = b.addTest(.{
-        .root_source_file = b.path("src/unit_tests.zig"),
-        .test_runner = .{ .path = b.path("src/unit_tests.zig"), .mode = .simple },
+        .root_source_file = b.path("src/main_unit_tests.zig"),
+        .test_runner = .{ .path = b.path("src/test_runner.zig"), .mode = .simple },
         .target = target,
         .optimize = mode,
     });
@@ -177,6 +177,9 @@ fn common(
     options: jsruntime.Options,
 ) !void {
     const target = step.root_module.resolved_target.?;
+    const optimize = step.root_module.optimize.?;
+    const dep_opts = .{ .target = target, .optimize = optimize };
+
     const jsruntimemod = try jsruntime_pkgs.module(
         b,
         options,
@@ -189,15 +192,7 @@ fn common(
     netsurf.addImport("jsruntime", jsruntimemod);
     step.root_module.addImport("netsurf", netsurf);
 
-    const asyncio = b.addModule("asyncio", .{
-        .root_source_file = b.path("vendor/zig-async-io/src/lib.zig"),
-    });
-    step.root_module.addImport("asyncio", asyncio);
-
-    const tlsmod = b.addModule("tls", .{
-        .root_source_file = b.path("vendor/tls.zig/src/root.zig"),
-    });
-    step.root_module.addImport("tls", tlsmod);
+    step.root_module.addImport("tls", b.dependency("tls", dep_opts).module("tls"));
 }
 
 fn moduleNetSurf(b: *std.Build, target: std.Build.ResolvedTarget) !*std.Build.Module {

build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = .browser,
+    .paths = .{""},
+    .version = "0.0.0",
+    .fingerprint = 0xda130f3af836cea0,
+    .dependencies = .{
+        .tls = .{
+            .url = "https://github.com/ianic/tls.zig/archive/b29a8b45fc59fc2d202769c4f54509bb9e17d0a2.tar.gz",
+            .hash = "1220e6fd39920dd6e28b2bc06688787a39430f8856f0597cd77c44ca868c6c54fb86",
+        },
+    },
+}

src/app.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 
 const Loop = @import("jsruntime").Loop;
 const Allocator = std.mem.Allocator;
-const HttpClient = @import("http/Client.zig");
+const HttpClient = @import("http/client.zig").Client;
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 
 const log = std.log.scoped(.app);
@@ -38,7 +38,7 @@ pub const App = struct {
             .allocator = allocator,
             .telemetry = undefined,
             .app_dir_path = app_dir_path,
-            .http_client = .{ .allocator = allocator },
+            .http_client = try HttpClient.init(allocator, 5),
         };
         app.telemetry = Telemetry.init(app, run_mode);
 

src/browser/browser.zig

@@ -24,7 +24,6 @@ const Allocator = std.mem.Allocator;
 const Types = @import("root").Types;
 
 const parser = @import("netsurf");
-const Loader = @import("loader.zig").Loader;
 const Dump = @import("dump.zig");
 const Mime = @import("mime.zig").Mime;
 
@@ -44,10 +43,8 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const FetchResult = @import("../http/Client.zig").Client.FetchResult;
-
+const http = @import("../http/client.zig");
 const UserContext = @import("../user_context.zig").UserContext;
-const HttpClient = @import("asyncio").Client;
 
 const polyfill = @import("../polyfill/polyfill.zig");
 
@@ -63,7 +60,7 @@ pub const Browser = struct {
     app: *App,
     session: ?*Session,
     allocator: Allocator,
-    http_client: *HttpClient,
+    http_client: *http.Client,
     session_pool: SessionPool,
     page_arena: std.heap.ArenaAllocator,
 
@@ -75,7 +72,7 @@ pub const Browser = struct {
             .app = app,
             .session = null,
             .allocator = allocator,
-            .http_client = @ptrCast(&app.http_client),
+            .http_client = &app.http_client,
             .session_pool = SessionPool.init(allocator),
             .page_arena = std.heap.ArenaAllocator.init(allocator),
         };
@@ -121,17 +118,17 @@ pub const Session = struct {
     // all others Session deps use directly self.alloc and not the arena.
     arena: std.heap.ArenaAllocator,
 
-    // TODO handle proxy
-    loader: Loader,
-
     env: Env,
     inspector: jsruntime.Inspector,
 
     window: Window,
 
-    // TODO move the shed to the browser?
+    // TODO move the shed/jar to the browser?
     storage_shed: storage.Shed,
+    cookie_jar: storage.CookieJar,
+
     page: ?Page = null,
+    http_client: *http.Client,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -143,9 +140,10 @@ pub const Session = struct {
             .env = undefined,
             .browser = browser,
             .inspector = undefined,
-            .loader = Loader.init(allocator),
+            .http_client = browser.http_client,
             .storage_shed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
+            .cookie_jar = storage.CookieJar.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
         };
 
@@ -181,7 +179,7 @@ pub const Session = struct {
         }
         self.env.deinit();
         self.arena.deinit();
-        self.loader.deinit();
+        self.cookie_jar.deinit();
         self.storage_shed.deinit();
     }
 
@@ -370,32 +368,16 @@ pub const Page = struct {
         } });
 
         // load the data
-        var resp = try self.session.loader.get(arena, self.uri);
-        defer resp.deinit();
+        var request = try self.newHTTPRequest(.GET, self.uri, .{ .navigation = true });
+        defer request.deinit();
 
-        const req = resp.req;
+        var response = try request.sendSync(.{});
+        const header = response.header;
+        try self.processHTTPResponse(self.uri, &header);
 
-        log.info("GET {any} {d}", .{ self.uri, @intFromEnum(req.response.status) });
+        log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        // TODO handle redirection
-        log.debug("{?} {d} {s}", .{
-            req.response.version,
-            @intFromEnum(req.response.status),
-            req.response.reason,
-            // TODO log headers
-        });
-
-        // TODO handle charset
-        // https://html.spec.whatwg.org/#content-type
-        var it = req.response.iterateHeaders();
-        var ct_: ?[]const u8 = null;
-        while (true) {
-            const h = it.next() orelse break;
-            if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
-                ct_ = try arena.dupe(u8, h.value);
-            }
-        }
-        const ct = ct_ orelse {
+        const ct = header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
@@ -404,14 +386,18 @@ pub const Page = struct {
 
         log.debug("header content-type: {s}", .{ct});
         var mime = try Mime.parse(arena, ct);
+        defer mime.deinit();
 
         if (mime.isHTML()) {
-            try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8", aux_data);
+            try self.loadHTMLDoc(&response, mime.charset orelse "utf-8", aux_data);
         } else {
             log.info("non-HTML document: {s}", .{ct});
-
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            while (try response.next()) |data| {
+                try arr.appendSlice(arena, try arena.dupe(u8, data));
+            }
             // save the body into the page.
-            self.raw_data = try req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
+            self.raw_data = arr.items;
         }
     }
 
@@ -453,7 +439,7 @@ pub const Page = struct {
         // replace the user context document with the new one.
         try session.env.setUserContext(.{
             .document = html_doc,
-            .httpClient = self.session.browser.http_client,
+            .http_client = @ptrCast(self.session.http_client),
         });
 
         // browse the DOM tree to retrieve scripts
@@ -625,39 +611,85 @@ pub const Page = struct {
                 res_src = try std.fs.path.resolve(arena, &.{ _dir, src });
             }
         }
-
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
-        var fetchres = try self.session.loader.get(arena, u);
-        defer fetchres.deinit();
+        var request = try self.newHTTPRequest(.GET, u, .{
+            .origin = self.uri,
+            .navigation = false,
+        });
+        defer request.deinit();
 
-        const resp = fetchres.req.response;
+        var response = try request.sendSync(.{});
+        var header = response.header;
+        try self.processHTTPResponse(u, &header);
 
-        log.info("fetch {any}: {d}", .{ u, resp.status });
+        log.info("fetch {any}: {d}", .{ u, header.status });
 
-        if (resp.status != .ok) {
+        if (header.status != 200) {
             return FetchError.BadStatusCode;
         }
 
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        while (try response.next()) |data| {
+            try arr.appendSlice(arena, try arena.dupe(u8, data));
+        }
+
         // TODO check content-type
-        const body = try fetchres.req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
 
         // check no body
-        if (body.len == 0) {
+        if (arr.items.len == 0) {
             return FetchError.NoBody;
         }
 
-        return body;
+        return arr.items;
     }
 
-    // fetchScript senf a GET request to the src and execute the script
-    // received.
     fn fetchScript(self: *const Page, s: *const Script) !void {
         const arena = self.arena;
         const body = try self.fetchData(arena, s.src, null);
         try s.eval(arena, &self.session.env, body);
     }
 
+    const RequestOpts = struct {
+        origin: ?std.Uri = null,
+        navigation: bool = true,
+    };
+    fn newHTTPRequest(self: *const Page, method: http.Request.Method, uri: std.Uri, opts: RequestOpts) !http.Request {
+        const session = self.session;
+        var request = try session.http_client.request(method, uri);
+        errdefer request.deinit();
+
+        var cookies = try session.cookie_jar.forRequest(
+            self.arena,
+            std.time.timestamp(),
+            opts.origin,
+            uri,
+            opts.navigation,
+        );
+        defer cookies.deinit(self.arena);
+
+        if (cookies.len() > 0) {
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            try cookies.write(arr.writer(self.arena));
+            try request.addHeader("Cookie", arr.items, .{});
+        }
+
+        return request;
+    }
+
+    fn processHTTPResponse(self: *const Page, uri: std.Uri, header: *const http.ResponseHeader) !void {
+        const session = self.session;
+        const now = std.time.timestamp();
+        var it = header.iterate("set-cookie");
+        while (it.next()) |set_cookie| {
+            const c = storage.Cookie.parse(self.arena, uri, set_cookie) catch |err| {
+                log.warn("Couldn't parse cookie '{s}': {}\n", .{ set_cookie, err });
+                continue;
+            };
+            try session.cookie_jar.add(c, now);
+        }
+    }
+
     const Script = struct {
         element: *parser.Element,
         kind: Kind,