replace zig-async-io and std.http.Client with a custom HTTP client

Author: karlseguin

.gitmodules

@@ -23,10 +23,3 @@
 [submodule "vendor/mimalloc"]
 	path = vendor/mimalloc
 	url = https://github.com/microsoft/mimalloc.git/
-[submodule "vendor/tls.zig"]
-	path = vendor/tls.zig
-	url = https://github.com/ianic/tls.zig.git/
-[submodule "vendor/zig-async-io"]
-	path = vendor/zig-async-io
-	url = https://github.com/lightpanda-io/zig-async-io.git/
-	branch = zig-0.14

LICENSING.md

@@ -10,7 +10,6 @@ The default license for this project is [AGPL-3.0-only](LICENSE).
 The following files are licensed under MIT:
 
 ```
-src/http/Client.zig
 src/polyfill/fetch.js
 ```
 

build.zig

@@ -177,6 +177,9 @@ fn common(
     options: jsruntime.Options,
 ) !void {
     const target = step.root_module.resolved_target.?;
+    const optimize = step.root_module.optimize.?;
+    const dep_opts = .{ .target = target, .optimize = optimize };
+
     const jsruntimemod = try jsruntime_pkgs.module(
         b,
         options,
@@ -189,15 +192,7 @@ fn common(
     netsurf.addImport("jsruntime", jsruntimemod);
     step.root_module.addImport("netsurf", netsurf);
 
-    const asyncio = b.addModule("asyncio", .{
-        .root_source_file = b.path("vendor/zig-async-io/src/lib.zig"),
-    });
-    step.root_module.addImport("asyncio", asyncio);
-
-    const tlsmod = b.addModule("tls", .{
-        .root_source_file = b.path("vendor/tls.zig/src/root.zig"),
-    });
-    step.root_module.addImport("tls", tlsmod);
+    step.root_module.addImport("tls", b.dependency("tls", dep_opts).module("tls"));
 }
 
 fn moduleNetSurf(b: *std.Build, target: std.Build.ResolvedTarget) !*std.Build.Module {

build.zig.zon

@@ -0,0 +1,12 @@
+.{
+  .name = .browser,
+  .paths = .{""},
+  .version = "0.0.0",
+  .fingerprint = 0xda130f3af836cea0,
+  .dependencies = .{
+    .tls = .{
+      .url = "https://github.com/karlseguin/tls.zig/archive/e39d40150f10464992da11352fb3955b3345272f.tar.gz",
+      .hash = "122039cd3abe387b69d23930bf12154c2c84fc894874e10129a1fc5e8ac75ca0ddc0"
+    },
+  },
+}

src/browser/browser.zig

@@ -24,7 +24,6 @@ const Allocator = std.mem.Allocator;
 const Types = @import("root").Types;
 
 const parser = @import("netsurf");
-const Loader = @import("loader.zig").Loader;
 const Dump = @import("dump.zig");
 const Mime = @import("mime.zig").Mime;
 
@@ -43,10 +42,8 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const FetchResult = @import("../http/Client.zig").Client.FetchResult;
-
+const http = @import("../http/client.zig");
 const UserContext = @import("../user_context.zig").UserContext;
-const HttpClient = @import("asyncio").Client;
 
 const polyfill = @import("../polyfill/polyfill.zig");
 
@@ -62,19 +59,19 @@ pub const Browser = struct {
     loop: *Loop,
     session: ?*Session,
     allocator: Allocator,
-    http_client: HttpClient,
+    http_client: http.Client,
     session_pool: SessionPool,
     page_arena: std.heap.ArenaAllocator,
 
     const SessionPool = std.heap.MemoryPool(Session);
 
-    pub fn init(allocator: Allocator, loop: *Loop) Browser {
+    pub fn init(allocator: Allocator, loop: *Loop) !Browser {
         return .{
             .loop = loop,
             .session = null,
             .allocator = allocator,
-            .http_client = .{ .allocator = allocator },
             .session_pool = SessionPool.init(allocator),
+            .http_client = try http.Client.init(allocator, 5),
             .page_arena = std.heap.ArenaAllocator.init(allocator),
         };
     }
@@ -118,9 +115,6 @@ pub const Session = struct {
     // all others Session deps use directly self.alloc and not the arena.
     arena: std.heap.ArenaAllocator,
 
-    // TODO handle proxy
-    loader: Loader,
-
     env: Env,
     inspector: jsruntime.Inspector,
 
@@ -129,6 +123,7 @@ pub const Session = struct {
     // TODO move the shed to the browser?
     storage_shed: storage.Shed,
     page: ?Page = null,
+    http_client: *http.Client,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -138,7 +133,7 @@ pub const Session = struct {
             .env = undefined,
             .browser = browser,
             .inspector = undefined,
-            .loader = Loader.init(allocator),
+            .http_client = &browser.http_client,
             .storage_shed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
@@ -176,7 +171,6 @@ pub const Session = struct {
         }
         self.env.deinit();
         self.arena.deinit();
-        self.loader.deinit();
         self.storage_shed.deinit();
     }
 
@@ -360,32 +354,14 @@ pub const Page = struct {
         // TODO handle fragment in url.
 
         // load the data
-        var resp = try self.session.loader.get(arena, self.uri);
-        defer resp.deinit();
+        var request = try self.session.http_client.request(.GET, self.uri);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        const req = resp.req;
+        const header = response.header;
+        log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        log.info("GET {any} {d}", .{ self.uri, @intFromEnum(req.response.status) });
-
-        // TODO handle redirection
-        log.debug("{?} {d} {s}", .{
-            req.response.version,
-            @intFromEnum(req.response.status),
-            req.response.reason,
-            // TODO log headers
-        });
-
-        // TODO handle charset
-        // https://html.spec.whatwg.org/#content-type
-        var it = req.response.iterateHeaders();
-        var ct_: ?[]const u8 = null;
-        while (true) {
-            const h = it.next() orelse break;
-            if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
-                ct_ = try arena.dupe(u8, h.value);
-            }
-        }
-        const ct = ct_ orelse {
+        const ct = response.header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
@@ -394,14 +370,18 @@ pub const Page = struct {
 
         log.debug("header content-type: {s}", .{ct});
         var mime = try Mime.parse(arena, ct);
+        defer mime.deinit();
 
         if (mime.isHTML()) {
-            try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8", aux_data);
+            try self.loadHTMLDoc(&response, mime.charset orelse "utf-8", aux_data);
         } else {
             log.info("non-HTML document: {s}", .{ct});
-
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            while (try response.next()) |data| {
+                try arr.appendSlice(arena, try arena.dupe(u8, data));
+            }
             // save the body into the page.
-            self.raw_data = try req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
+            self.raw_data = arr.items;
         }
     }
 
@@ -443,7 +423,7 @@ pub const Page = struct {
         // replace the user context document with the new one.
         try session.env.setUserContext(.{
             .document = html_doc,
-            .httpClient = &self.session.browser.http_client,
+            .http_client = @ptrCast(&self.session.http_client),
         });
 
         // browse the DOM tree to retrieve scripts
@@ -620,30 +600,32 @@ pub const Page = struct {
 
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
-        var fetchres = try self.session.loader.get(arena, u);
-        defer fetchres.deinit();
-
-        const resp = fetchres.req.response;
+        var request = try self.session.http_client.request(.GET, u);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        log.info("fetch {any}: {d}", .{ u, resp.status });
+        log.info("fetch {any}: {d}", .{ u, response.header.status });
 
-        if (resp.status != .ok) {
+        if (response.header.status != 200) {
             return FetchError.BadStatusCode;
         }
 
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        while (try response.next()) |data| {
+            try arr.appendSlice(arena, try arena.dupe(u8, data));
+        }
+
         // TODO check content-type
-        const body = try fetchres.req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
 
         // check no body
-        if (body.len == 0) {
+        if (arr.items.len == 0) {
             return FetchError.NoBody;
         }
 
-        return body;
+        return arr.items;
     }
 
-    // fetchScript senf a GET request to the src and execute the script
-    // received.
+
     fn fetchScript(self: *const Page, s: *const Script) !void {
         const arena = self.arena;
 

src/browser/loader.zig

@@ -73,13 +73,13 @@ pub fn CDPT(comptime TypeProvider: type) type {
         pub const Browser = TypeProvider.Browser;
         pub const Session = TypeProvider.Session;
 
-        pub fn init(allocator: Allocator, client: TypeProvider.Client, loop: TypeProvider.Loop) Self {
+        pub fn init(allocator: Allocator, client: TypeProvider.Client, loop: TypeProvider.Loop) !Self {
             return .{
                 .loop = loop,
                 .client = client,
                 .allocator = allocator,
                 .browser_context = null,
-                .browser = Browser.init(allocator, loop),
+                .browser = try Browser.init(allocator, loop),
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
                 .browser_context_pool = std.heap.MemoryPool(BrowserContext(Self)).init(allocator),
             };

src/cdp/cdp.zig

@@ -16,7 +16,7 @@ const Browser = struct {
     session: ?*Session = null,
     arena: std.heap.ArenaAllocator,
 
-    pub fn init(allocator: Allocator, loop: anytype) Browser {
+    pub fn init(allocator: Allocator, loop: anytype) !Browser {
         _ = loop;
         return .{
             .arena = std.heap.ArenaAllocator.init(allocator),
@@ -135,7 +135,7 @@ const TestContext = struct {
             self.client = Client.init(self.arena.allocator());
             // Don't use the arena here. We want to detect leaks in CDP.
             // The arena is only for test-specific stuff
-            self.cdp_ = TestCDP.init(std.testing.allocator, &self.client.?, {});
+            self.cdp_ = try TestCDP.init(std.testing.allocator, &self.client.?, {});
         }
         return &self.cdp_.?;
     }