Replace zig-async-io with a custom HTTP client

.gitmodules

@@ -23,10 +23,3 @@
 [submodule "vendor/mimalloc"]
 	path = vendor/mimalloc
 	url = https://github.com/microsoft/mimalloc.git/
-[submodule "vendor/tls.zig"]
-	path = vendor/tls.zig
-	url = https://github.com/ianic/tls.zig.git/
-[submodule "vendor/zig-async-io"]
-	path = vendor/zig-async-io
-	url = https://github.com/lightpanda-io/zig-async-io.git/
-	branch = zig-0.14

LICENSING.md

@@ -10,7 +10,6 @@ The default license for this project is [AGPL-3.0-only](LICENSE).
 The following files are licensed under MIT:
 
 ```
-src/http/Client.zig
 src/polyfill/fetch.js
 ```
 

build.zig

@@ -133,8 +133,8 @@ pub fn build(b: *std.Build) !void {
 
     // compile
     const unit_tests = b.addTest(.{
-        .root_source_file = b.path("src/unit_tests.zig"),
-        .test_runner = .{ .path = b.path("src/unit_tests.zig"), .mode = .simple },
+        .root_source_file = b.path("src/main_unit_tests.zig"),
+        .test_runner = .{ .path = b.path("src/test_runner.zig"), .mode = .simple },
         .target = target,
         .optimize = mode,
     });
@@ -177,6 +177,9 @@ fn common(
     options: jsruntime.Options,
 ) !void {
     const target = step.root_module.resolved_target.?;
+    const optimize = step.root_module.optimize.?;
+    const dep_opts = .{ .target = target, .optimize = optimize };
+
     const jsruntimemod = try jsruntime_pkgs.module(
         b,
         options,
@@ -189,15 +192,7 @@ fn common(
     netsurf.addImport("jsruntime", jsruntimemod);
     step.root_module.addImport("netsurf", netsurf);
 
-    const asyncio = b.addModule("asyncio", .{
-        .root_source_file = b.path("vendor/zig-async-io/src/lib.zig"),
-    });
-    step.root_module.addImport("asyncio", asyncio);
-
-    const tlsmod = b.addModule("tls", .{
-        .root_source_file = b.path("vendor/tls.zig/src/root.zig"),
-    });
-    step.root_module.addImport("tls", tlsmod);
+    step.root_module.addImport("tls", b.dependency("tls", dep_opts).module("tls"));
 }
 
 fn moduleNetSurf(b: *std.Build, target: std.Build.ResolvedTarget) !*std.Build.Module {

build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = .browser,
+    .paths = .{""},
+    .version = "0.0.0",
+    .fingerprint = 0xda130f3af836cea0,
+    .dependencies = .{
+        .tls = .{
+            .url = "https://github.com/ianic/tls.zig/archive/b29a8b45fc59fc2d202769c4f54509bb9e17d0a2.tar.gz",
+            .hash = "1220e6fd39920dd6e28b2bc06688787a39430f8856f0597cd77c44ca868c6c54fb86",
+        },
+    },
+}

src/app.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 
 const Loop = @import("jsruntime").Loop;
 const Allocator = std.mem.Allocator;
-const HttpClient = @import("http/Client.zig");
+const HttpClient = @import("http/client.zig").Client;
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 
 const log = std.log.scoped(.app);
@@ -38,7 +38,7 @@ pub const App = struct {
             .allocator = allocator,
             .telemetry = undefined,
             .app_dir_path = app_dir_path,
-            .http_client = .{ .allocator = allocator },
+            .http_client = try HttpClient.init(allocator, 5),
         };
         app.telemetry = Telemetry.init(app, run_mode);
 

src/browser/browser.zig

@@ -24,7 +24,6 @@ const Allocator = std.mem.Allocator;
 const Types = @import("root").Types;
 
 const parser = @import("netsurf");
-const Loader = @import("loader.zig").Loader;
 const Dump = @import("dump.zig");
 const Mime = @import("mime.zig").Mime;
 
@@ -44,10 +43,8 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const FetchResult = @import("../http/Client.zig").Client.FetchResult;
-
+const HttpClient = @import("../http/client.zig").Client;
 const UserContext = @import("../user_context.zig").UserContext;
-const HttpClient = @import("asyncio").Client;
 
 const polyfill = @import("../polyfill/polyfill.zig");
 
@@ -75,7 +72,7 @@ pub const Browser = struct {
             .app = app,
             .session = null,
             .allocator = allocator,
-            .http_client = @ptrCast(&app.http_client),
+            .http_client = &app.http_client,
             .session_pool = SessionPool.init(allocator),
             .page_arena = std.heap.ArenaAllocator.init(allocator),
         };
@@ -121,9 +118,6 @@ pub const Session = struct {
     // all others Session deps use directly self.alloc and not the arena.
     arena: std.heap.ArenaAllocator,
 
-    // TODO handle proxy
-    loader: Loader,
-
     env: Env,
     inspector: jsruntime.Inspector,
 
@@ -132,6 +126,7 @@ pub const Session = struct {
     // TODO move the shed to the browser?
     storage_shed: storage.Shed,
     page: ?Page = null,
+    http_client: *HttpClient,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -143,7 +138,7 @@ pub const Session = struct {
             .env = undefined,
             .browser = browser,
             .inspector = undefined,
-            .loader = Loader.init(allocator),
+            .http_client = browser.http_client,
             .storage_shed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
@@ -181,7 +176,6 @@ pub const Session = struct {
         }
         self.env.deinit();
         self.arena.deinit();
-        self.loader.deinit();
         self.storage_shed.deinit();
     }
 
@@ -370,32 +364,14 @@ pub const Page = struct {
         } });
 
         // load the data
-        var resp = try self.session.loader.get(arena, self.uri);
-        defer resp.deinit();
-
-        const req = resp.req;
+        var request = try self.session.http_client.request(.GET, self.uri);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        log.info("GET {any} {d}", .{ self.uri, @intFromEnum(req.response.status) });
+        const header = response.header;
+        log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        // TODO handle redirection
-        log.debug("{?} {d} {s}", .{
-            req.response.version,
-            @intFromEnum(req.response.status),
-            req.response.reason,
-            // TODO log headers
-        });
-
-        // TODO handle charset
-        // https://html.spec.whatwg.org/#content-type
-        var it = req.response.iterateHeaders();
-        var ct_: ?[]const u8 = null;
-        while (true) {
-            const h = it.next() orelse break;
-            if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
-                ct_ = try arena.dupe(u8, h.value);
-            }
-        }
-        const ct = ct_ orelse {
+        const ct = response.header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
@@ -404,14 +380,18 @@ pub const Page = struct {
 
         log.debug("header content-type: {s}", .{ct});
         var mime = try Mime.parse(arena, ct);
+        defer mime.deinit();
 
         if (mime.isHTML()) {
-            try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8", aux_data);
+            try self.loadHTMLDoc(&response, mime.charset orelse "utf-8", aux_data);
         } else {
             log.info("non-HTML document: {s}", .{ct});
-
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            while (try response.next()) |data| {
+                try arr.appendSlice(arena, try arena.dupe(u8, data));
+            }
             // save the body into the page.
-            self.raw_data = try req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
+            self.raw_data = arr.items;
         }
     }
 
@@ -453,7 +433,7 @@ pub const Page = struct {
         // replace the user context document with the new one.
         try session.env.setUserContext(.{
             .document = html_doc,
-            .httpClient = self.session.browser.http_client,
+            .http_client = @ptrCast(self.session.http_client),
         });
 
         // browse the DOM tree to retrieve scripts
@@ -625,33 +605,33 @@ pub const Page = struct {
                 res_src = try std.fs.path.resolve(arena, &.{ _dir, src });
             }
         }
-
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
-        var fetchres = try self.session.loader.get(arena, u);
-        defer fetchres.deinit();
+        var request = try self.session.http_client.request(.GET, u);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        const resp = fetchres.req.response;
+        log.info("fetch {any}: {d}", .{ u, response.header.status });
 
-        log.info("fetch {any}: {d}", .{ u, resp.status });
-
-        if (resp.status != .ok) {
+        if (response.header.status != 200) {
             return FetchError.BadStatusCode;
         }
 
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        while (try response.next()) |data| {
+            try arr.appendSlice(arena, try arena.dupe(u8, data));
+        }
+
         // TODO check content-type
-        const body = try fetchres.req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
 
         // check no body
-        if (body.len == 0) {
+        if (arr.items.len == 0) {
             return FetchError.NoBody;
         }
 
-        return body;
+        return arr.items;
     }
 
-    // fetchScript senf a GET request to the src and execute the script
-    // received.
     fn fetchScript(self: *const Page, s: *const Script) !void {
         const arena = self.arena;
         const body = try self.fetchData(arena, s.src, null);

src/cdp/testing.zig

@@ -1,3 +1,21 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 const std = @import("std");
 
 const json = std.json;