scalar clone: support GVFS-enabled remote repositories

With this change, we come a big step closer to feature parity with
Scalar: this allows cloning from Azure Repos (which do not support
partial clones at time of writing).

We use the just-implemented JSON parser to parse the response we got
from the `gvfs/config` endpoint; Please note that this response might,
or might not, contain information about a cache server. The presence or
absence of said cache server, however, has nothing to do with the
ability to speak the GVFS protocol (but the presence of the
`gvfs/config` endpoint does that).

An alternative considered during the development of this patch was to
perform simple string matching instead of parsing the JSON-formatted
data; However, this would have been fragile, as the response contains
free-form text (e.g. the repository's description) which might contain
parts that would confuse a simple string matcher (but not a proper JSON
parser).

Note: we need to limit the re-try logic in `git clone` to handle only
the non-GVFS case: the call to `set_config()` to un-set the partial
clone settings would otherwise fail because those settings would not
exist in the GVFS protocol case. This will at least give us a clearer
reason why such a fetch fails.

The way the `gvfs-helper` command operates is apparently incompatible
with HTTP/2, that's why we need to enforce HTTP/1.1 in Scalar clones
accessing Azure Repos.

Co-authored-by: Derrick Stolee <[email protected]>
Signed-off-by: Johannes Schindelin <[email protected]>
Signed-off-by: Derrick Stolee <[email protected]>

Author: dscho

diagnose.c

@@ -12,6 +12,7 @@
 #include "parse-options.h"
 #include "repository.h"
 #include "write-or-die.h"
+#include "config.h"
 
 struct archive_dir {
 	const char *path;
@@ -185,6 +186,7 @@ int create_diagnostics_archive(struct repository *r,
 	struct strvec archiver_args = STRVEC_INIT;
 	char **argv_copy = NULL;
 	int stdout_fd = -1, archiver_fd = -1;
+	char *cache_server_url = NULL;
 	struct strbuf buf = STRBUF_INIT;
 	int res;
 	struct archive_dir archive_dirs[] = {
@@ -220,6 +222,11 @@ int create_diagnostics_archive(struct repository *r,
 	get_version_info(&buf, 1);
 
 	strbuf_addf(&buf, "Repository root: %s\n", r->worktree);
+
+	repo_config_get_string(r, "gvfs.cache-server", &cache_server_url);
+	strbuf_addf(&buf, "Cache Server: %s\n\n",
+		    cache_server_url ? cache_server_url : "None");
+
 	get_disk_info(&buf);
 	write_or_die(stdout_fd, buf.buf, buf.len);
 	strvec_pushf(&archiver_args,
@@ -277,6 +284,7 @@ int create_diagnostics_archive(struct repository *r,
 	free(argv_copy);
 	strvec_clear(&archiver_args);
 	strbuf_release(&buf);
+	free(cache_server_url);
 
 	return res;
 }

scalar.c

@@ -19,6 +19,7 @@
 #include "help.h"
 #include "setup.h"
 #include "trace2.h"
+#include "json-parser.h"
 
 static void setup_enlistment_directory(int argc, const char **argv,
 				       const char * const *usagestr,
@@ -354,6 +355,85 @@ static int set_config(const char *fmt, ...)
 	return res;
 }
 
+/* Find N for which .CacheServers[N].GlobalDefault == true */
+static int get_cache_server_index(struct json_iterator *it)
+{
+	const char *p;
+	char *q;
+	long l;
+
+	if (it->type == JSON_TRUE &&
+	    skip_iprefix(it->key.buf, ".CacheServers[", &p) &&
+	    (l = strtol(p, &q, 10)) >= 0 && p != q &&
+	    !strcasecmp(q, "].GlobalDefault")) {
+		*(long *)it->fn_data = l;
+		return 1;
+	}
+
+	return 0;
+}
+
+struct cache_server_url_data {
+	char *key, *url;
+};
+
+/* Get .CacheServers[N].Url */
+static int get_cache_server_url(struct json_iterator *it)
+{
+	struct cache_server_url_data *data = it->fn_data;
+
+	if (it->type == JSON_STRING &&
+	    !strcasecmp(data->key, it->key.buf)) {
+		data->url = strbuf_detach(&it->string_value, NULL);
+		return 1;
+	}
+
+	return 0;
+}
+
+/*
+ * If `cache_server_url` is `NULL`, print the list to `stdout`.
+ *
+ * Since `gvfs-helper` requires a Git directory, this _must_ be run in
+ * a worktree.
+ */
+static int supports_gvfs_protocol(const char *url, char **cache_server_url)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+	struct strbuf out = STRBUF_INIT;
+
+	cp.git_cmd = 1;
+	strvec_pushl(&cp.args, "-c", "http.version=HTTP/1.1",
+		     "gvfs-helper", "--remote", url, "config", NULL);
+	if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) {
+		long l = 0;
+		struct json_iterator it =
+			JSON_ITERATOR_INIT(out.buf, get_cache_server_index, &l);
+		struct cache_server_url_data data = { .url = NULL };
+
+		if (iterate_json(&it) < 0) {
+			reset_iterator(&it);
+			strbuf_release(&out);
+			return error("JSON parse error");
+		}
+		data.key = xstrfmt(".CacheServers[%ld].Url", l);
+		it.fn = get_cache_server_url;
+		it.fn_data = &data;
+		if (iterate_json(&it) < 0) {
+			reset_iterator(&it);
+			strbuf_release(&out);
+			return error("JSON parse error");
+		}
+		*cache_server_url = data.url;
+		free(data.key);
+		reset_iterator(&it);
+		strbuf_release(&out);
+		return 1;
+	}
+	strbuf_release(&out);
+	return 0; /* error out quietly */
+}
+
 static char *remote_default_branch(const char *url)
 {
 	struct child_process cp = CHILD_PROCESS_INIT;
@@ -454,6 +534,8 @@ static int cmd_clone(int argc, const char **argv)
 	char *branch_to_free = NULL;
 	int full_clone = 0, single_branch = 0, show_progress = isatty(2);
 	int src = 1, tags = 1, maintenance = 1;
+	const char *cache_server_url = NULL;
+	char *default_cache_server_url = NULL;
 	struct option clone_options[] = {
 		OPT_STRING('b', "branch", &branch, N_("<branch>"),
 			   N_("branch to checkout after clone")),
@@ -468,6 +550,9 @@ static int cmd_clone(int argc, const char **argv)
 			 N_("specify if tags should be fetched during clone")),
 		OPT_BOOL(0, "maintenance", &maintenance,
 			 N_("specify if background maintenance should be enabled")),
+		OPT_STRING(0, "cache-server-url", &cache_server_url,
+			   N_("<url>"),
+			   N_("the url or friendly name of the cache server")),
 		OPT_END(),
 	};
 	const char * const clone_usage[] = {
@@ -479,6 +564,7 @@ static int cmd_clone(int argc, const char **argv)
 	char *enlistment = NULL, *dir = NULL;
 	struct strbuf buf = STRBUF_INIT;
 	int res;
+	int gvfs_protocol;
 
 	argc = parse_options(argc, argv, NULL, clone_options, clone_usage, 0);
 
@@ -544,9 +630,7 @@ static int cmd_clone(int argc, const char **argv)
 	    set_config("remote.origin.fetch="
 		       "+refs/heads/%s:refs/remotes/origin/%s",
 		       single_branch ? branch : "*",
-		       single_branch ? branch : "*") ||
-	    set_config("remote.origin.promisor=true") ||
-	    set_config("remote.origin.partialCloneFilter=blob:none")) {
+		       single_branch ? branch : "*")) {
 		res = error(_("could not configure remote in '%s'"), dir);
 		goto cleanup;
 	}
@@ -556,6 +640,41 @@ static int cmd_clone(int argc, const char **argv)
 		goto cleanup;
 	}
 
+	if (set_config("credential.https://dev.azure.com.useHttpPath=true")) {
+		res = error(_("could not configure credential.useHttpPath"));
+		goto cleanup;
+	}
+
+	gvfs_protocol = cache_server_url ||
+			supports_gvfs_protocol(url, &default_cache_server_url);
+
+	if (gvfs_protocol) {
+		if (!cache_server_url)
+			cache_server_url = default_cache_server_url;
+		if (set_config("core.useGVFSHelper=true") ||
+		    set_config("core.gvfs=150") ||
+		    set_config("http.version=HTTP/1.1")) {
+			res = error(_("could not turn on GVFS helper"));
+			goto cleanup;
+		}
+		if (cache_server_url &&
+		    set_config("gvfs.cache-server=%s", cache_server_url)) {
+			res = error(_("could not configure cache server"));
+			goto cleanup;
+		}
+		if (cache_server_url)
+			fprintf(stderr, "Cache server URL: %s\n",
+				cache_server_url);
+	} else {
+		if (set_config("core.useGVFSHelper=false") ||
+		    set_config("remote.origin.promisor=true") ||
+		    set_config("remote.origin.partialCloneFilter=blob:none")) {
+			res = error(_("could not configure partial clone in "
+				      "'%s'"), dir);
+			goto cleanup;
+		}
+	}
+
 	if (!full_clone &&
 	    (res = run_git("sparse-checkout", "init", "--cone", NULL)))
 		goto cleanup;
@@ -568,6 +687,11 @@ static int cmd_clone(int argc, const char **argv)
 				"origin",
 				(tags ? NULL : "--no-tags"),
 				NULL))) {
+		if (gvfs_protocol) {
+			res = error(_("failed to prefetch commits and trees"));
+			goto cleanup;
+		}
+
 		warning(_("partial clone failed; attempting full clone"));
 
 		if (set_config("remote.origin.promisor") ||
@@ -602,6 +726,7 @@ static int cmd_clone(int argc, const char **argv)
 	free(enlistment);
 	free(dir);
 	strbuf_release(&buf);
+	free(default_cache_server_url);
 	return res;
 }