Start unifying test and code

Depends on #993

There's currently 3 ways to execute a page:
1 - page.navigate (as used in both the 'fetch' and 'serve' commands)
2 - jsRunner as used in unit tests
3 - main_wpt as used in the WPT runner

Both jsRunner and main_wpt replicate the page.navigate code, but in their own
hack-ish way. main_wpt re-implements the DOM walking in order to extract and
execute <script> tags, as well as the needed page lifecycle events.

This PR replaces the existing main_wpt loader with a call to page.navigate. To
support this, a test HTTP server was added. (The test HTTP server is extracted
from the existing unit test test server, and re-used between the two).

There are benefits to this approach:
1 - The code is simpler
2 - More of the actual code and flow is tested
3 - There's 1 way to do things (page.navigate)
4 - Having an HTTP server might unlock some WPT tests

Technically, we're replacing file IO with network IO i.e. http requests). This
has potential downsides:
1 - The tests might be more brittle
2 - The tests might be slower

I think we need to run it for a while to see if we get flaky behavior.

The goal for following PRs is to bring this unification to the jsRunner.

Author: karlseguin

src/TestHTTPServer.zig

@@ -0,0 +1,117 @@
+const std = @import("std");
+
+const TestHTTPServer = @This();
+
+shutdown: bool,
+listener: ?std.net.Server,
+handler: Handler,
+
+const Handler = *const fn (req: *std.http.Server.Request) anyerror!void;
+
+pub fn init(handler: Handler) TestHTTPServer {
+    return .{
+        .shutdown = true,
+        .listener = null,
+        .handler = handler,
+    };
+}
+
+pub fn deinit(self: *TestHTTPServer) void {
+    self.shutdown = true;
+    if (self.listener) |*listener| {
+        listener.deinit();
+    }
+}
+
+pub fn run(self: *TestHTTPServer, wg: *std.Thread.WaitGroup) !void {
+    const address = try std.net.Address.parseIp("127.0.0.1", 9582);
+
+    self.listener = try address.listen(.{ .reuse_address = true });
+    var listener = &self.listener.?;
+
+    wg.finish();
+
+    while (true) {
+        const conn = listener.accept() catch |err| {
+            if (self.shutdown) {
+                return;
+            }
+            return err;
+        };
+        const thrd = try std.Thread.spawn(.{}, handleConnection, .{ self, conn });
+        thrd.detach();
+    }
+}
+
+fn handleConnection(self: *TestHTTPServer, conn: std.net.Server.Connection) !void {
+    defer conn.stream.close();
+
+    var req_buf: [2048]u8 = undefined;
+    var conn_reader = conn.stream.reader(&req_buf);
+    var conn_writer = conn.stream.writer(&req_buf);
+
+    var http_server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface);
+
+    while (true) {
+        var req = http_server.receiveHead() catch |err| switch (err) {
+            error.ReadFailed => continue,
+            error.HttpConnectionClosing => continue,
+            else => {
+                std.debug.print("Test HTTP Server error: {}\n", .{err});
+                return err;
+            },
+        };
+        self.handler(&req) catch |err| {
+            std.debug.print("test http error '{s}': {}\n", .{ req.head.target, err });
+            try req.respond("server error", .{ .status = .internal_server_error });
+            return;
+        };
+    }
+}
+
+pub fn sendFile(req: *std.http.Server.Request, file_path: []const u8) !void {
+    var file = std.fs.cwd().openFile(file_path, .{}) catch |err| switch (err) {
+        error.FileNotFound => return req.respond("server error", .{ .status = .not_found }),
+        else => return err,
+    };
+
+    const stat = try file.stat();
+    var send_buffer: [4096]u8 = undefined;
+
+    var res = try req.respondStreaming(&send_buffer, .{
+        .content_length = stat.size,
+        .respond_options = .{
+            .extra_headers = &.{
+                .{ .name = "content-type", .value = getContentType(file_path) },
+            },
+        },
+    });
+
+    var read_buffer: [4096]u8 = undefined;
+    var reader = file.reader(&read_buffer);
+    _ = try res.writer.sendFileAll(&reader, .unlimited);
+    try res.writer.flush();
+    try res.end();
+}
+
+fn getContentType(file_path: []const u8) []const u8 {
+    if (std.mem.endsWith(u8, file_path, ".js")) {
+        return "application/json";
+    }
+
+    if (std.mem.endsWith(u8, file_path, ".html")) {
+        return "text/html";
+    }
+
+    if (std.mem.endsWith(u8, file_path, ".htm")) {
+        return "text/html";
+    }
+
+    if (std.mem.endsWith(u8, file_path, ".xml")) {
+        // some wpt tests do this
+        return "text/xml";
+    }
+
+    std.debug.print("TestHTTPServer asked to serve an unknown file type: {s}\n", .{file_path});
+    return "text/html";
+}

src/browser/ScriptManager.zig

@@ -403,14 +403,7 @@ fn startCallback(transfer: *Http.Transfer) !void {
 
 fn headerCallback(transfer: *Http.Transfer) !void {
     const script: *PendingScript = @ptrCast(@alignCast(transfer.ctx));
-    script.headerCallback(transfer) catch |err| {
-        log.err(.http, "SM.headerCallback", .{
-            .err = err,
-            .transfer = transfer,
-            .status = transfer.response_header.?.status,
-        });
-        return err;
-    };
+    script.headerCallback(transfer);
 }
 
 fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
@@ -463,18 +456,23 @@ const PendingScript = struct {
         log.debug(.http, "script fetch start", .{ .req = transfer });
     }
 
-    fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
+    fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) void {
         const header = &transfer.response_header.?;
+        if (header.status != 200) {
+            log.info(.http, "script header", .{
+                .req = transfer,
+                .status = header.status,
+                .content_type = header.contentType(),
+            });
+            return;
+        }
+
         log.debug(.http, "script header", .{
             .req = transfer,
             .status = header.status,
             .content_type = header.contentType(),
         });
 
-        if (header.status != 200) {
-            return error.InvalidStatusCode;
-        }
-
         // If this isn't true, then we'll likely leak memory. If you don't
         // set `CURLOPT_SUPPRESS_CONNECT_HEADERS` and CONNECT to a proxy, this
         // will fail. This assertion exists to catch incorrect assumptions about

src/browser/xhr/xhr.zig

@@ -785,8 +785,7 @@ test "Browser.XHR.XMLHttpRequest" {
         .{ "req.statusText", "OK" },
         .{ "req.getResponseHeader('Content-Type')", "text/html; charset=utf-8" },
         .{ "req.getAllResponseHeaders()", "content-length: 100\r\n" ++
-            "Content-Type: text/html; charset=utf-8\r\n" ++
-            "Connection: Close\r\n" },
+            "Content-Type: text/html; charset=utf-8\r\n" },
         .{ "req.responseText.length", "100" },
         .{ "req.response.length == req.responseText.length", "true" },
         .{ "req.responseXML instanceof Document", "true" },

src/main.zig

@@ -719,23 +719,26 @@ test {
     std.testing.refAllDecls(@This());
 }
 
+const TestHTTPServer = @import("TestHTTPServer.zig");
+
 var test_cdp_server: ?Server = null;
+var test_http_server: ?TestHTTPServer = null;
+
 test "tests:beforeAll" {
     log.opts.level = .err;
     log.opts.format = .logfmt;
-
     try testing.setup();
-
     var wg: std.Thread.WaitGroup = .{};
     wg.startMany(2);
 
     {
-        const thread = try std.Thread.spawn(.{}, serveHTTP, .{&wg});
+        const thread = try std.Thread.spawn(.{}, serveCDP, .{&wg});
         thread.detach();
     }
 
+    test_http_server = TestHTTPServer.init(testHTTPHandler);
     {
-        const thread = try std.Thread.spawn(.{}, serveCDP, .{&wg});
+        const thread = try std.Thread.spawn(.{}, TestHTTPServer.run, .{ &test_http_server.?, &wg });
         thread.detach();
     }
 
@@ -748,59 +751,10 @@ test "tests:afterAll" {
     if (test_cdp_server) |*server| {
         server.deinit();
     }
-    testing.shutdown();
-}
-
-fn serveHTTP(wg: *std.Thread.WaitGroup) !void {
-    const address = try std.net.Address.parseIp("127.0.0.1", 9582);
-
-    var listener = try address.listen(.{ .reuse_address = true });
-    defer listener.deinit();
-
-    wg.finish();
-
-    var buf: [1024]u8 = undefined;
-    while (true) {
-        var conn = try listener.accept();
-        defer conn.stream.close();
-        var conn_reader = conn.stream.reader(&buf);
-        var conn_writer = conn.stream.writer(&buf);
-
-        var http_server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface);
-
-        var request = http_server.receiveHead() catch |err| switch (err) {
-            error.HttpConnectionClosing => continue,
-            else => {
-                std.debug.print("Test HTTP Server error: {}\n", .{err});
-                return err;
-            },
-        };
-
-        const path = request.head.target;
-
-        if (std.mem.eql(u8, path, "/loader")) {
-            try request.respond("Hello!", .{
-                .extra_headers = &.{.{ .name = "Connection", .value = "close" }},
-            });
-        } else if (std.mem.eql(u8, path, "/xhr")) {
-            try request.respond("1234567890" ** 10, .{
-                .extra_headers = &.{
-                    .{ .name = "Content-Type", .value = "text/html; charset=utf-8" },
-                    .{ .name = "Connection", .value = "Close" },
-                },
-            });
-        } else if (std.mem.eql(u8, path, "/xhr/json")) {
-            try request.respond("{\"over\":\"9000!!!\"}", .{
-                .extra_headers = &.{
-                    .{ .name = "Content-Type", .value = "application/json" },
-                    .{ .name = "Connection", .value = "Close" },
-                },
-            });
-        } else {
-            // should not have an unknown path
-            unreachable;
-        }
+    if (test_http_server) |*server| {
+        server.deinit();
     }
+    testing.shutdown();
 }
 
 fn serveCDP(wg: *std.Thread.WaitGroup) !void {
@@ -816,3 +770,31 @@ fn serveCDP(wg: *std.Thread.WaitGroup) !void {
         return err;
     };
 }
+
+fn testHTTPHandler(req: *std.http.Server.Request) !void {
+    const path = req.head.target;
+
+    if (std.mem.eql(u8, path, "/loader")) {
+        return req.respond("Hello!", .{
+            .extra_headers = &.{.{ .name = "Connection", .value = "close" }},
+        });
+    }
+
+    if (std.mem.eql(u8, path, "/xhr")) {
+        return req.respond("1234567890" ** 10, .{
+            .extra_headers = &.{
+                .{ .name = "Content-Type", .value = "text/html; charset=utf-8" },
+            },
+        });
+    }
+
+    if (std.mem.eql(u8, path, "/xhr/json")) {
+        return req.respond("{\"over\":\"9000!!!\"}", .{
+            .extra_headers = &.{
+                .{ .name = "Content-Type", .value = "application/json" },
+            },
+        });
+    }
+
+    unreachable;
+}