casync-http: add max-host-connections option

This commit introduces a new option --max-host-connections=<MAX> that
limits the number of connections to a single host.

See https://curl.haxx.se/libcurl/c/CURLMOPT_MAX_HOST_CONNECTIONS.html.

Author: gportay

doc/casync.rst

@@ -165,6 +165,7 @@ General options:
 --cache-auto, -c                Pick encoder cache directory automatically
 --rate-limit-bps=<LIMIT>        Maximum bandwidth in bytes/s for remote communication
 --max-active-chunks=<MAX>       Maximum number of simultaneously active chunks for remote communication
+--max-host-connections=<MAX>    Maximum number of connections to a single host for remote communication
 --exclude-nodump=no             Don't exclude files with chattr(1)'s +d **nodump** flag when creating archive
 --exclude-submounts=yes         Exclude submounts when creating archive
 --exclude-file=no               Don't respect .caexclude files in the file tree

src/caremote.c

@@ -61,6 +61,7 @@ struct CaRemote {
         int log_level;
         uint64_t rate_limit_bps;
         unsigned max_active_chunks;
+        unsigned max_host_connections;
 
         ReallocBuffer input_buffer;
         ReallocBuffer output_buffer;
@@ -258,6 +259,15 @@ int ca_remote_set_max_active_chunks(CaRemote *rr, unsigned max_active_chunks) {
         return 0;
 }
 
+int ca_remote_set_max_host_connections(CaRemote *rr, unsigned max_host_connections) {
+        if (!rr)
+                return -EINVAL;
+
+        rr->max_host_connections = max_host_connections;
+
+        return 0;
+}
+
 int ca_remote_set_local_feature_flags(CaRemote *rr, uint64_t flags) {
         if (!rr)
                 return -EINVAL;
@@ -1014,6 +1024,9 @@ static int ca_remote_start(CaRemote *rr) {
                         if (rr->max_active_chunks)
                                 argc++;
 
+                        if (rr->max_host_connections)
+                                argc++;
+
                         args = newa(char*, argc + 1);
 
                         if (rr->callout) {
@@ -1067,6 +1080,14 @@ static int ca_remote_start(CaRemote *rr) {
                                 i++;
                         }
 
+                        if (rr->max_host_connections) {
+                                r = asprintf(args + i, "--max-host-connections=%u", rr->max_host_connections);
+                                if (r < 0)
+                                        return log_oom();
+
+                                i++;
+                        }
+
                         args[i + CA_REMOTE_ARG_OPERATION] = (char*) ((rr->local_feature_flags & (CA_PROTOCOL_PUSH_CHUNKS|CA_PROTOCOL_PUSH_INDEX|CA_PROTOCOL_PUSH_ARCHIVE)) ? "push" : "pull");
                         args[i + CA_REMOTE_ARG_BASE_URL] = /* rr->base_url ? rr->base_url + skip :*/ (char*) "-";
                         args[i + CA_REMOTE_ARG_ARCHIVE_URL] = rr->archive_url ? rr->archive_url + skip : (char*) "-";

src/caremote.h

@@ -53,6 +53,7 @@ int ca_remote_get_digest_type(CaRemote *rr, CaDigestType *ret);
 int ca_remote_set_log_level(CaRemote *rr, int log_level);
 int ca_remote_set_rate_limit_bps(CaRemote *rr, uint64_t rate_limit_bps);
 int ca_remote_set_max_active_chunks(CaRemote *rr, unsigned max_active_chunks);
+int ca_remote_set_max_host_connections(CaRemote *rr, unsigned max_max_connections);
 
 int ca_remote_set_io_fds(CaRemote *rr, int input_fd, int output_fd);
 int ca_remote_get_io_fds(CaRemote *rr, int *ret_input_fd, int *ret_output_fd);

src/casync-http.c

@@ -39,12 +39,23 @@
  */
 #define MAX_ACTIVE_CHUNKS 64
 
+/* This is the maximum number of parallel connections per host. This should have
+ * no effect in case we're doing HTTP/2 with one connection and multiplexing.
+ * However, if we're doing HTTP/1, curl will open parallel connections, as HTTP/1
+ * pipelining is no longer supported since libcurl 7.62.
+ *
+ * We want to make sure that we don't open too many parallel connections per host.
+ * It seems that the norm for web browsers ranges from 6 to 8.
+ */
+#define MAX_HOST_CONNECTIONS 8
+
 static volatile sig_atomic_t quit = false;
 
 static int arg_log_level = -1;
 static bool arg_verbose = false;
 static curl_off_t arg_rate_limit_bps = 0;
 static unsigned arg_max_active_chunks = MAX_ACTIVE_CHUNKS;
+static unsigned arg_max_host_connections = MAX_HOST_CONNECTIONS;
 static bool arg_ssl_trust_peer = false;
 
 typedef enum Protocol {
@@ -247,6 +258,8 @@ static int make_curl_multi_handle(CURLM **ret) {
         if (!h)
                 return log_oom();
 
+        CURL_SETOPT_MULTI(h, CURLMOPT_MAX_HOST_CONNECTIONS, arg_max_host_connections);
+
         /* Default since libcurl 7.62.0 */
         CURL_SETOPT_MULTI_CANFAIL(h, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
 
@@ -1231,6 +1244,7 @@ static int parse_argv(int argc, char *argv[]) {
         enum {
                 ARG_RATE_LIMIT_BPS = 0x100,
                 ARG_MAX_ACTIVE_CHUNKS,
+                ARG_MAX_HOST_CONNECTIONS,
                 ARG_SSL_TRUST_PEER,
         };
 
@@ -1240,6 +1254,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "verbose",        no_argument,       NULL, 'v'                },
                 { "rate-limit-bps", required_argument, NULL, ARG_RATE_LIMIT_BPS },
                 { "max-active-chunks",    required_argument, NULL, ARG_MAX_ACTIVE_CHUNKS    },
+                { "max-host-connections", required_argument, NULL, ARG_MAX_HOST_CONNECTIONS },
                 { "ssl-trust-peer", no_argument,       NULL, ARG_SSL_TRUST_PEER },
                 {}
         };
@@ -1298,6 +1313,14 @@ static int parse_argv(int argc, char *argv[]) {
                         }
                         break;
 
+                case ARG_MAX_HOST_CONNECTIONS:
+                        r = safe_atou(optarg, &arg_max_host_connections);
+                        if (r < 0 || arg_max_host_connections == 0) {
+                                log_error("Invalid value for max-host-connections, refusing");
+                                return -EINVAL;
+                        }
+                        break;
+
                 case ARG_SSL_TRUST_PEER:
                         arg_ssl_trust_peer = true;
                         break;

src/casync-tool.c

@@ -67,6 +67,7 @@ static size_t arg_chunk_size_avg = 0;
 static size_t arg_chunk_size_max = 0;
 static uint64_t arg_rate_limit_bps = UINT64_MAX;
 static unsigned arg_max_active_chunks = 0;
+static unsigned arg_max_host_connections = 0;
 static uint64_t arg_with = 0;
 static uint64_t arg_without = 0;
 static uid_t arg_uid_shift = 0, arg_uid_range = 0x10000U;
@@ -110,6 +111,9 @@ static void help(void) {
                "                             communication\n"
                "     --max-active-chunks=MAX Maximum number of simultaneously active chunks for\n"
                "                             remote communication\n"
+               "     --max-host-connections=MAX\n"
+               "                             Maximum number of connections to a single host for\n"
+               "                             remote communication\n"
                "     --exclude-nodump=no     Don't exclude files with chattr(1)'s +d 'nodump'\n"
                "                             flag when creating archive\n"
                "     --exclude-submounts=yes Exclude submounts when creating archive\n"
@@ -332,6 +336,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_CACHE,
                 ARG_RATE_LIMIT_BPS,
                 ARG_MAX_ACTIVE_CHUNKS,
+                ARG_MAX_HOST_CONNECTIONS,
                 ARG_WITH,
                 ARG_WITHOUT,
                 ARG_WHAT,
@@ -367,6 +372,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "cache-auto",        no_argument,       NULL, 'c'                   },
                 { "rate-limit-bps",    required_argument, NULL, ARG_RATE_LIMIT_BPS    },
                 { "max-active-chunks", required_argument, NULL, ARG_MAX_ACTIVE_CHUNKS },
+                { "max-host-connections", required_argument, NULL, ARG_MAX_HOST_CONNECTIONS },
                 { "with",              required_argument, NULL, ARG_WITH              },
                 { "without",           required_argument, NULL, ARG_WITHOUT           },
                 { "what",              required_argument, NULL, ARG_WHAT              },
@@ -488,6 +494,14 @@ static int parse_argv(int argc, char *argv[]) {
                         }
                         break;
 
+                case ARG_MAX_HOST_CONNECTIONS:
+                        r = safe_atou(optarg, &arg_max_host_connections);
+                        if (r < 0) {
+                                log_error("Failed to parse --max-host-connections= value %s", optarg);
+                                return -EINVAL;
+                        }
+                        break;
+
                 case ARG_WITH: {
                         uint64_t u;
 
@@ -1349,6 +1363,12 @@ static int verb_make(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_sync_set_max_host_connections(s, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         r = ca_sync_set_base_fd(s, input_fd);
         if (r < 0)
                 return log_error_errno(r, "Failed to set sync base: %m");
@@ -1660,6 +1680,12 @@ static int verb_extract(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_sync_set_max_host_connections(s, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         if (seek_path) {
                 if (output_fd >= 0)
                         r = ca_sync_set_boundary_fd(s, output_fd);
@@ -2827,6 +2853,12 @@ static int verb_mount(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_sync_set_max_host_connections(s, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         if (operation == MOUNT_ARCHIVE) {
                 if (input_fd >= 0)
                         r = ca_sync_set_archive_fd(s, input_fd);
@@ -2959,6 +2991,12 @@ static int verb_mkdev(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_sync_set_max_host_connections(s, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         if (operation == MKDEV_BLOB) {
                 if (input_fd >= 0)
                         r = ca_sync_set_archive_fd(s, input_fd);
@@ -3532,6 +3570,12 @@ static int verb_pull(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_remote_set_max_host_connections(rr, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         r = ca_remote_set_io_fds(rr, STDIN_FILENO, STDOUT_FILENO);
         if (r < 0)
                 return log_error_errno(r, "Failed to set I/O file descriptors: %m");
@@ -3697,6 +3741,12 @@ static int verb_push(int argc, char *argv[]) {
                         return log_error_errno(r, "Failed to set max active chunks: %m");
         }
 
+        if (arg_max_host_connections) {
+                r = ca_remote_set_max_host_connections(rr, arg_max_host_connections);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to set max host connections: %m");
+        }
+
         r = ca_remote_set_io_fds(rr, STDIN_FILENO, STDOUT_FILENO);
         if (r < 0)
                 log_error_errno(r, "Failed to set I/O file descriptors: %m");

src/casync.c

@@ -113,6 +113,7 @@ struct CaSync {
         int log_level;
         size_t rate_limit_bps;
         unsigned max_active_chunks;
+        unsigned max_host_connections;
 
         uint64_t feature_flags;
         uint64_t feature_flags_mask;
@@ -532,6 +533,15 @@ int ca_sync_set_max_active_chunks(CaSync *s, unsigned max_active_chunks) {
         return 0;
 }
 
+int ca_sync_set_max_host_connections(CaSync *s, unsigned max_host_connections) {
+        if (!s)
+                return -EINVAL;
+
+        s->max_host_connections = max_host_connections;
+
+        return 0;
+}
+
 int ca_sync_set_rate_limit_bps(CaSync *s, uint64_t rate_limit_bps) {
         if (!s)
                 return -EINVAL;
@@ -710,6 +720,12 @@ int ca_sync_set_index_remote(CaSync *s, const char *url) {
                         return r;
         }
 
+        if (s->max_host_connections > 0) {
+                r = ca_remote_set_max_host_connections(s->remote_index, s->max_host_connections);
+                if (r < 0)
+                        return r;
+        }
+
         r = ca_remote_set_index_url(s->remote_index, url);
         if (r < 0)
                 return r;

src/casync.h

@@ -34,6 +34,7 @@ DEFINE_TRIVIAL_CLEANUP_FUNC(CaSync *, ca_sync_unref);
 int ca_sync_set_log_level(CaSync *s, int log_level);
 int ca_sync_set_rate_limit_bps(CaSync *s, uint64_t rate_limit_bps);
 int ca_sync_set_max_active_chunks(CaSync *s, unsigned max_active_chunks);
+int ca_sync_set_max_host_connections(CaSync *s, unsigned max_host_connection);
 
 int ca_sync_set_feature_flags(CaSync *s, uint64_t flags);
 int ca_sync_get_feature_flags(CaSync *s, uint64_t *ret);