Add command line options to control HTTP client

http_timeout_ms
http_connect_timeout_ms
http_max_host_open
http_max_concurrent

Author: karlseguin

src/app.zig

@@ -35,6 +35,10 @@ pub const App = struct {
         tls_verify_host: bool = true,
         http_proxy: ?[:0]const u8 = null,
         proxy_bearer_token: ?[:0]const u8 = null,
+        http_timeout_ms: ?u31 = null,
+        http_connect_timeout_ms: ?u31 = null,
+        http_max_host_open: ?u8 = null,
+        http_max_concurrent: ?u8 = null,
     };
 
     pub fn init(allocator: Allocator, config: Config) !*App {
@@ -51,7 +55,10 @@ pub const App = struct {
         errdefer notification.deinit();
 
         var http = try Http.init(allocator, .{
-            .max_concurrent_transfers = 10,
+            .max_host_open = config.http_max_host_open orelse 4,
+            .max_concurrent = config.http_max_concurrent orelse 10,
+            .timeout_ms = config.http_timeout_ms orelse 5000,
+            .connect_timeout_ms = config.http_connect_timeout_ms orelse 0,
             .http_proxy = config.http_proxy,
             .tls_verify_host = config.tls_verify_host,
             .proxy_bearer_token = config.proxy_bearer_token,

src/http/Client.zig

@@ -102,6 +102,8 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie
     const multi = c.curl_multi_init() orelse return error.FailedToInitializeMulti;
     errdefer _ = c.curl_multi_cleanup(multi);
 
+    try errorMCheck(c.curl_multi_setopt(multi, c.CURLMOPT_MAX_HOST_CONNECTIONS, @as(c_long, opts.max_host_open)));
+
     var handles = try Handles.init(allocator, client, ca_blob, &opts);
     errdefer handles.deinit(allocator);
 
@@ -346,8 +348,7 @@ const Handles = struct {
 
     // pointer to opts is not stable, don't hold a reference to it!
     fn init(allocator: Allocator, client: *Client, ca_blob: ?c.curl_blob, opts: *const Http.Opts) !Handles {
-        const count = opts.max_concurrent_transfers;
-        std.debug.assert(count > 0);
+        const count = if (opts.max_concurrent == 0) 1 else opts.max_concurrent;
 
         const handles = try allocator.alloc(Handle, count);
         errdefer allocator.free(handles);

src/http/Http.zig

@@ -230,11 +230,12 @@ pub fn errorMCheck(code: c.CURLMcode) errors.Multi!void {
 }
 
 pub const Opts = struct {
-    timeout_ms: u31 = 0,
+    timeout_ms: u31,
+    max_host_open: u8,
+    max_concurrent: u8,
+    connect_timeout_ms: u31,
     max_redirects: u8 = 10,
     tls_verify_host: bool = true,
-    connect_timeout_ms: u31 = 5000,
-    max_concurrent_transfers: u8 = 5,
     http_proxy: ?[:0]const u8 = null,
     proxy_bearer_token: ?[:0]const u8 = null,
 };

src/main.zig

@@ -87,6 +87,10 @@ fn run(alloc: Allocator) !void {
         .http_proxy = args.httpProxy(),
         .proxy_bearer_token = args.proxyBearerToken(),
         .tls_verify_host = args.tlsVerifyHost(),
+        .http_timeout_ms = args.httpTimeout(),
+        .http_connect_timeout_ms = args.httpConnectTiemout(),
+        .http_max_host_open = args.httpMaxHostOpen(),
+        .http_max_concurrent = args.httpMaxConcurrent(),
     });
     defer app.deinit();
     app.telemetry.record(.{ .run = {} });
@@ -165,6 +169,34 @@ const Command = struct {
         };
     }
 
+    fn httpMaxConcurrent(self: *const Command) ?u8 {
+        return switch (self.mode) {
+            inline .serve, .fetch => |opts| opts.common.http_max_concurrent,
+            else => unreachable,
+        };
+    }
+
+    fn httpMaxHostOpen(self: *const Command) ?u8 {
+        return switch (self.mode) {
+            inline .serve, .fetch => |opts| opts.common.http_max_host_open,
+            else => unreachable,
+        };
+    }
+
+    fn httpConnectTiemout(self: *const Command) ?u31 {
+        return switch (self.mode) {
+            inline .serve, .fetch => |opts| opts.common.http_connect_timeout,
+            else => unreachable,
+        };
+    }
+
+    fn httpTimeout(self: *const Command) ?u31 {
+        return switch (self.mode) {
+            inline .serve, .fetch => |opts| opts.common.http_timeout,
+            else => unreachable,
+        };
+    }
+
     fn logLevel(self: *const Command) ?log.Level {
         return switch (self.mode) {
             inline .serve, .fetch => |opts| opts.common.log_level,
@@ -208,31 +240,52 @@ const Command = struct {
     };
 
     const Common = struct {
-        http_proxy: ?[:0]const u8 = null,
         proxy_bearer_token: ?[:0]const u8 = null,
+        http_proxy: ?[:0]const u8 = null,
+        http_max_concurrent: ?u8 = null,
+        http_max_host_open: ?u8 = null,
+        http_timeout: ?u31 = null,
+        http_connect_timeout: ?u31 = null,
         tls_verify_host: bool = true,
         log_level: ?log.Level = null,
         log_format: ?log.Format = null,
         log_filter_scopes: ?[]log.Scope = null,
     };
 
     fn printUsageAndExit(self: *const Command, success: bool) void {
+        //                                                                MAX_HELP_LEN|
         const common_options =
             \\
             \\--insecure_disable_tls_host_verification
-            \\                Disables host verification on all HTTP requests.
-            \\                This is an advanced option which should only be
-            \\                set if you understand and accept the risk of
-            \\                disabling host verification.
+            \\                Disables host verification on all HTTP requests. This is an
+            \\                advanced option which should only be set if you understand
+            \\                and accept the risk of disabling host verification.
             \\
             \\--http_proxy    The HTTP proxy to use for all HTTP requests.
-            \\                A username:password can be included to use basic
-            \\                authentication.
+            \\                A username:password can be included for basic authentication.
             \\                Defaults to none.
             \\
             \\--proxy_bearer_token
-            \\               The <token> to send for bearer authentication with the proxy
-            \\               Proxy-Authorization: Bearer <token>
+            \\                The <token> to send for bearer authentication with the proxy
+            \\                Proxy-Authorization: Bearer <token>
+            \\
+            \\--http_max_concurrent
+            \\                The maximum number of concurrent HTTP requests.
+            \\                Defaults to 10.
+            \\
+            \\--http_max_host_open
+            \\                The maximum number of open connection to a given host:port.
+            \\                Defaults to 4.
+            \\
+            \\--http_connect_timeout
+            \\                The time, in milliseconds, for establishing an HTTP connection
+            \\                before timing out. 0 means it never times out.
+            \\                Defaults to 0.
+            \\
+            \\--http_timeout
+            \\                The maximum time, in milliseconds, the transfer is allowed
+            \\                to complete. 0 means it never times out.
+            \\                Defaults to 10000.
             \\
             \\--log_level     The log level: debug, info, warn, error or fatal.
             \\                Defaults to
@@ -243,9 +296,9 @@ const Command = struct {
             \\                Defaults to
         ++ (if (builtin.mode == .Debug) " pretty." else " logfmt.") ++
             \\
-            \\
         ;
 
+        //                                                                MAX_HELP_LEN|
         const usage =
             \\usage: {s} command [options] [URL]
             \\
@@ -497,6 +550,58 @@ fn parseCommonArg(
         return true;
     }
 
+    if (std.mem.eql(u8, "--http_max_concurrent", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--http_max_concurrent" });
+            return error.InvalidArgument;
+        };
+
+        common.http_max_concurrent = std.fmt.parseInt(u8, str, 10) catch |err| {
+            log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_concurrent", .err = err });
+            return error.InvalidArgument;
+        };
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--http_max_host_open", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--http_max_host_open" });
+            return error.InvalidArgument;
+        };
+
+        common.http_max_host_open = std.fmt.parseInt(u8, str, 10) catch |err| {
+            log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_host_open", .err = err });
+            return error.InvalidArgument;
+        };
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--http_connect_timeout", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--http_connect_timeout" });
+            return error.InvalidArgument;
+        };
+
+        common.http_connect_timeout = std.fmt.parseInt(u31, str, 10) catch |err| {
+            log.fatal(.app, "invalid argument value", .{ .arg = "--http_connect_timeout", .err = err });
+            return error.InvalidArgument;
+        };
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--http_timeout", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--http_timeout" });
+            return error.InvalidArgument;
+        };
+
+        common.http_timeout = std.fmt.parseInt(u31, str, 10) catch |err| {
+            log.fatal(.app, "invalid argument value", .{ .arg = "--http_timeout", .err = err });
+            return error.InvalidArgument;
+        };
+        return true;
+    }
+
     if (std.mem.eql(u8, "--log_level", opt)) {
         const str = args.next() orelse {
             log.fatal(.app, "missing argument value", .{ .arg = "--log_level" });
@@ -644,6 +749,7 @@ fn serveCDP(address: std.net.Address, platform: *const Platform) !void {
         .run_mode = .serve,
         .tls_verify_host = false,
         .platform = platform,
+        .max_concurrent_transfers = 2,
     });
     defer app.deinit();