replace zig-async-io and std.http.Client with a custom HTTP client

Author: karlseguin

.gitmodules

@@ -23,10 +23,3 @@
 [submodule "vendor/mimalloc"]
 	path = vendor/mimalloc
 	url = https://github.com/microsoft/mimalloc.git/
-[submodule "vendor/tls.zig"]
-	path = vendor/tls.zig
-	url = https://github.com/ianic/tls.zig.git/
-[submodule "vendor/zig-async-io"]
-	path = vendor/zig-async-io
-	url = https://github.com/lightpanda-io/zig-async-io.git/
-	branch = zig-0.14

LICENSING.md

@@ -10,7 +10,6 @@ The default license for this project is [AGPL-3.0-only](LICENSE).
 The following files are licensed under MIT:
 
 ```
-src/http/Client.zig
 src/polyfill/fetch.js
 ```
 

build.zig

@@ -177,6 +177,9 @@ fn common(
     options: jsruntime.Options,
 ) !void {
     const target = step.root_module.resolved_target.?;
+    const optimize = step.root_module.optimize.?;
+    const dep_opts = .{ .target = target, .optimize = optimize };
+
     const jsruntimemod = try jsruntime_pkgs.module(
         b,
         options,
@@ -189,15 +192,7 @@ fn common(
     netsurf.addImport("jsruntime", jsruntimemod);
     step.root_module.addImport("netsurf", netsurf);
 
-    const asyncio = b.addModule("asyncio", .{
-        .root_source_file = b.path("vendor/zig-async-io/src/lib.zig"),
-    });
-    step.root_module.addImport("asyncio", asyncio);
-
-    const tlsmod = b.addModule("tls", .{
-        .root_source_file = b.path("vendor/tls.zig/src/root.zig"),
-    });
-    step.root_module.addImport("tls", tlsmod);
+    step.root_module.addImport("tls", b.dependency("tls", dep_opts).module("tls"));
 }
 
 fn moduleNetSurf(b: *std.Build, target: std.Build.ResolvedTarget) !*std.Build.Module {

build.zig.zon

@@ -0,0 +1,12 @@
+.{
+  .name = .browser,
+  .paths = .{""},
+  .version = "0.0.0",
+  .fingerprint = 0xda130f3af836cea0,
+  .dependencies = .{
+    .tls = .{
+      .url = "https://github.com/karlseguin/tls.zig/archive/e39d40150f10464992da11352fb3955b3345272f.tar.gz",
+      .hash = "122039cd3abe387b69d23930bf12154c2c84fc894874e10129a1fc5e8ac75ca0ddc0"
+    },
+  },
+}

src/browser/browser.zig

@@ -24,7 +24,6 @@ const Allocator = std.mem.Allocator;
 const Types = @import("root").Types;
 
 const parser = @import("netsurf");
-const Loader = @import("loader.zig").Loader;
 const Dump = @import("dump.zig");
 const Mime = @import("mime.zig").Mime;
 
@@ -43,10 +42,8 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const FetchResult = @import("../http/Client.zig").Client.FetchResult;
-
+const http = @import("../http/client.zig");
 const UserContext = @import("../user_context.zig").UserContext;
-const HttpClient = @import("asyncio").Client;
 
 const polyfill = @import("../polyfill/polyfill.zig");
 
@@ -117,9 +114,6 @@ pub const Session = struct {
 
     uri: []const u8,
 
-    // TODO handle proxy
-    loader: Loader,
-
     env: Env,
     loop: *Loop,
     inspector: jsruntime.Inspector,
@@ -129,7 +123,7 @@ pub const Session = struct {
     // TODO move the shed to the browser?
     storageShed: storage.Shed,
     page: ?Page = null,
-    httpClient: HttpClient,
+    http_client: http.Client,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -139,8 +133,7 @@ pub const Session = struct {
             .env = undefined,
             .inspector = undefined,
             .allocator = allocator,
-            .loader = Loader.init(allocator),
-            .httpClient = .{ .allocator = allocator },
+            .http_client = try http.Client.init(allocator, 5),
             .storageShed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
@@ -179,8 +172,7 @@ pub const Session = struct {
 
         self.env.deinit();
         self.arena.deinit();
-        self.httpClient.deinit();
-        self.loader.deinit();
+        self.http_client.deinit();
         self.storageShed.deinit();
     }
 
@@ -367,50 +359,34 @@ pub const Page = struct {
         // TODO handle fragment in url.
 
         // load the data
-        var resp = try self.session.loader.get(alloc, self.uri);
-        defer resp.deinit();
-
-        const req = resp.req;
-
-        log.info("GET {any} {d}", .{ self.uri, @intFromEnum(req.response.status) });
+        var request = try self.session.http_client.request(.GET, self.uri);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        // TODO handle redirection
-        log.debug("{?} {d} {s}", .{
-            req.response.version,
-            @intFromEnum(req.response.status),
-            req.response.reason,
-            // TODO log headers
-        });
+        const header = response.header;
+        log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        // TODO handle charset
-        // https://html.spec.whatwg.org/#content-type
-        var it = req.response.iterateHeaders();
-        var ct: ?[]const u8 = null;
-        while (true) {
-            const h = it.next() orelse break;
-            if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
-                ct = try alloc.dupe(u8, h.value);
-            }
-        }
-        if (ct == null) {
+        const ct = response.header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
             return;
-        }
-        defer alloc.free(ct.?);
+        };
 
-        log.debug("header content-type: {s}", .{ct.?});
-        var mime = try Mime.parse(alloc, ct.?);
+        log.debug("header content-type: {s}", .{ct});
+        var mime = try Mime.parse(alloc, ct);
         defer mime.deinit();
 
         if (mime.isHTML()) {
-            try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8", auxData);
+            try self.loadHTMLDoc(&response, mime.charset orelse "utf-8", auxData);
         } else {
-            log.info("non-HTML document: {s}", .{ct.?});
-
+            log.info("non-HTML document: {s}", .{ct});
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            while (try response.next()) |data| {
+                try arr.appendSlice(alloc, try alloc.dupe(u8, data));
+            }
             // save the body into the page.
-            self.raw_data = try req.reader().readAllAlloc(alloc, 16 * 1024 * 1024);
+            self.raw_data = arr.items;
         }
     }
 
@@ -453,7 +429,7 @@ pub const Page = struct {
         // replace the user context document with the new one.
         try self.session.env.setUserContext(.{
             .document = html_doc,
-            .httpClient = &self.session.httpClient,
+            .http_client = @ptrCast(&self.session.http_client),
         });
 
         // browse the DOM tree to retrieve scripts
@@ -614,22 +590,29 @@ pub const Page = struct {
         var b: []u8 = buffer[0..];
         const u = try std.Uri.resolve_inplace(self.uri, src, &b);
 
-        var fetchres = try self.session.loader.get(alloc, u);
-        defer fetchres.deinit();
+        var request = try self.session.http_client.request(.GET, u);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        const resp = fetchres.req.response;
+        log.info("fetch {any}: {d}", .{ u, response.header.status });
 
-        log.info("fetch {any}: {d}", .{ u, resp.status });
+        if (response.header.status != 200) {
+            return FetchError.BadStatusCode;
+        }
 
-        if (resp.status != .ok) return FetchError.BadStatusCode;
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        while (try response.next()) |data| {
+            try arr.appendSlice(alloc, try alloc.dupe(u8, data));
+        }
 
         // TODO check content-type
-        const body = try fetchres.req.reader().readAllAlloc(alloc, 16 * 1024 * 1024);
 
         // check no body
-        if (body.len == 0) return FetchError.NoBody;
+        if (arr.items.len == 0) {
+            return FetchError.NoBody;
+        }
 
-        return body;
+        return arr.items;
     }
 
     // fetchScript senf a GET request to the src and execute the script
@@ -638,7 +621,6 @@ pub const Page = struct {
         const alloc = self.arena.allocator();
         const body = try self.fetchData(alloc, s.src);
         defer alloc.free(body);
-
         try s.eval(alloc, self.session.env, body);
     }