Merge branch 'scalar-with-gvfs'

Prepare `scalar` to use the GVFS protocol instead of partial clone
(required to support Azure Repos).

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

Documentation/scalar.adoc

@@ -9,14 +9,17 @@ SYNOPSIS
 --------
 [verse]
 scalar clone [--single-branch] [--branch <main-branch>] [--full-clone]
-	[--[no-]src] [--[no-]tags] [--[no-]maintenance] <url> [<enlistment>]
+	[--[no-]src] [--[no-]tags] [--[no-]maintenance]
+	[--[no-]src] [--local-cache-path <path>] [--cache-server-url <url>]
+	<url> [<enlistment>]
 scalar list
 scalar register [--[no-]maintenance] [<enlistment>]
 scalar unregister [<enlistment>]
 scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]
 scalar reconfigure [--maintenance=(enable|disable|keep)] [ --all | <enlistment> ]
 scalar diagnose [<enlistment>]
 scalar delete <enlistment>
+scalar cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]
 
 DESCRIPTION
 -----------
@@ -102,6 +105,37 @@ cloning. If the HEAD at the remote did not point at any branch when
 	background maintenance feature. Use the `--no-maintenance` to skip
 	this configuration.
 
+--local-cache-path <path>::
+	Override the path to the local cache root directory; Pre-fetched objects
+	are stored into a repository-dependent subdirectory of that path.
++
+The default is `<drive>:\.scalarCache` on Windows (on the same drive as the
+clone), and `~/.scalarCache` on macOS.
+
+--cache-server-url <url>::
+	Retrieve missing objects from the specified remote, which is expected to
+	understand the GVFS protocol.
+
+--[no-]gvfs-protocol::
+	When cloning from a `<url>` with either `dev.azure.com` or
+	`visualstudio.com` in the name, `scalar clone` will attempt to use the GVFS
+	Protocol to access Git objects, specifically from a cache server when
+	available, and will fail to clone if there is an error over that protocol.
+
+	To enable the GVFS Protocol regardless of the origin `<url>`, use
+	`--gvfs-protocol`. This will cause `scalar clone` to fail when the origin
+	server fails to provide a valid response to the `gvfs/config` endpoint.
+
+	To disable the GVFS Protocol, use `--no-gvfs-protocol` and `scalar clone`
+	will only use the Git protocol, starting with a partial clone. This can be
+	helpful if your `<url>` points to Azure Repos but the repository does not
+	have GVFS cache servers enabled. It is likely more efficient to use its
+	partial clone functionality through the Git protocol.
+
+	Previous versions of `scalar clone` could fall back to a partial clone over
+	the Git protocol if there is any issue gathering GVFS configuration
+	information from the origin server.
+
 List
 ~~~~
 
@@ -191,6 +225,27 @@ delete <enlistment>::
 	This subcommand lets you delete an existing Scalar enlistment from your
 	local file system, unregistering the repository.
 
+Cache-server
+~~~~~~~~~~~~
+
+cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]::
+    This command lets you query or set the GVFS-enabled cache server used
+    to fetch missing objects.
+
+--get::
+    This is the default command mode: query the currently-configured cache
+    server URL, if any.
+
+--list::
+    Access the `gvfs/info` endpoint of the specified remote (default:
+    `origin`) to figure out which cache servers are available, if any.
++
+In contrast to the `--get` command mode (which only accesses the local
+repository), this command mode triggers a request via the network that
+potentially requires authentication. If authentication is required, the
+configured credential helper is employed (see linkgit:git-credential[1]
+for details).
+
 SEE ALSO
 --------
 linkgit:git-clone[1], linkgit:git-maintenance[1].

Makefile

@@ -2825,6 +2825,7 @@ GIT_OBJS += git.o
 .PHONY: git-objs
 git-objs: $(GIT_OBJS)
 
+SCALAR_OBJS := json-parser.o
 SCALAR_OBJS += scalar.o
 .PHONY: scalar-objs
 scalar-objs: $(SCALAR_OBJS)
@@ -2980,7 +2981,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
 		$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
 
-scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
+scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
 		$(filter %.o,$^) $(LIBS)
 

contrib/buildsystems/CMakeLists.txt

@@ -804,7 +804,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main)
 add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c)
 target_link_libraries(git-shell common-main)
 
-add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c)
+add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c)
 target_link_libraries(scalar common-main)
 
 if(CURL_FOUND)

contrib/scalar/docs/getting-started.md

@@ -18,8 +18,9 @@ Creating a new Scalar clone
 ---------------------------------------------------
 
 The `clone` verb creates a local enlistment of a remote repository using the
-partial clone feature available e.g. on GitHub.
-
+partial clone feature available e.g. on GitHub, or using the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md),
+such as Azure Repos.
 
 ```
 scalar clone [options] <url> [<dir>]
@@ -68,11 +69,26 @@ in `<path>`.
 These options allow a user to customize their initial enlistment.
 
 * `--full-clone`: If specified, do not initialize the sparse-checkout feature.
-  All files will be present in your `src` directory. This uses a Git partial
-  clone: blobs are downloaded on demand.
+  All files will be present in your `src` directory. This behaves very similar
+  to a Git partial clone in that blobs are downloaded on demand. However, it
+  will use the GVFS protocol to download all Git objects.
+
+* `--cache-server-url=<url>`: If specified, set the intended cache server to
+  the specified `<url>`. All object queries will use the GVFS protocol to this
+  `<url>` instead of the origin remote. If the remote supplies a list of
+  cache servers via the `<url>/gvfs/config` endpoint, then the `clone` command
+  will select a nearby cache server from that list.
 
 * `--branch=<ref>`: Specify the branch to checkout after clone.
 
+* `--local-cache-path=<path>`: Use this option to override the path for the
+  local Scalar cache. If not specified, then Scalar will select a default
+  path to share objects with your other enlistments. On Windows, this path
+  is a subdirectory of `<Volume>:\.scalarCache\`. On Mac, this path is a
+  subdirectory of `~/.scalarCache/`. The default cache path is recommended so
+  multiple enlistments of the same remote repository share objects on the
+  same device.
+
 ### Advanced Options
 
 The options below are not intended for use by a typical user. These are

contrib/scalar/docs/index.md

@@ -28,10 +28,14 @@ these features for that repo (except partial clone) and start running suggested
 maintenance in the background using
 [the `git maintenance` feature](https://git-scm.com/docs/git-maintenance).
 
-Repos cloned with the `scalar clone` command use partial clone to significantly
-reduce the amount of data required to get started using a repository. By
-delaying all blob downloads until they are required, Scalar allows you to work
-with very large repositories quickly.
+Repos cloned with the `scalar clone` command use partial clone or the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md)
+to significantly reduce the amount of data required to get started
+using a repository. By delaying all blob downloads until they are required,
+Scalar allows you to work with very large repositories quickly. The GVFS
+protocol allows a network of _cache servers_ to serve objects with lower
+latency and higher throughput. The cache servers also reduce load on the
+central server.
 
 Documentation
 -------------
@@ -42,7 +46,7 @@ Documentation
 
 * [Troubleshooting](troubleshooting.md):
   Collect diagnostic information or update custom settings. Includes
-  `scalar diagnose`.
+  `scalar diagnose` and `scalar cache-server`.
 
 * [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way
   it does, and how do we make decisions about its future?

contrib/scalar/docs/philosophy.md

@@ -13,22 +13,27 @@ Scalar only to configure those new settings. In particular, we ported
 features like background maintenance to Git to make Scalar simpler and
 make Git more powerful.
 
-Services such as GitHub support partial clone , a standard adopted by the Git
-project to download only part of the Git objects when cloning, and fetching
-further objects on demand. If your hosting service supports partial clone, then
-we absolutely recommend it as a way to greatly speed up your clone and fetch
-times and to reduce how much disk space your Git repository requires. Scalar
-will help with this!
+Scalar ships inside [a custom version of Git][microsoft-git], but we are
+working to make it available in other forks of Git. The only feature
+that is not intended to ever reach the standard Git client is Scalar's use
+of [the GVFS Protocol][gvfs-protocol], which is essentially an older
+version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/)
+that was available first in Azure Repos. Services such as GitHub support
+only partial clone instead of the GVFS protocol because that is the
+standard adopted by the Git project. If your hosting service supports
+partial clone, then we absolutely recommend it as a way to greatly speed
+up your clone and fetch times and to reduce how much disk space your Git
+repository requires. Scalar will help with this!
 
-Most of the value of Scalar can be found in the core Git client. However, most
-of the advanced features that really optimize Git's performance are off by
-default for compatibility reasons. To really take advantage of Git's latest and
-greatest features, you either need to study the [`git config`
-documentation](https://git-scm.com/docs/git-config) and regularly read [the Git
-release notes](https://github.com/git/git/tree/master/Documentation/RelNotes).
+If you don't use the GVFS Protocol, then most of the value of Scalar can
+be found in the core Git client. However, most of the advanced features
+that really optimize Git's performance are off by default for compatibility
+reasons. To really take advantage of Git's latest and greatest features,
+you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config)
+and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes).
 Even if you do all that work and customize your Git settings on your machines,
-you likely will want to share those settings with other team members. Or, you
-can just use Scalar!
+you likely will want to share those settings with other team members.
+Or, you can just use Scalar!
 
 Using `scalar register` on an existing Git repository will give you these
 benefits:

contrib/scalar/docs/troubleshooting.md

@@ -18,3 +18,23 @@ files for that repository. This includes:
 
 As the `diagnose` command completes, it provides the path of the resulting
 zip file. This zip can be attached to bug reports to make the analysis easier.
+
+Modifying Configuration Values
+------------------------------
+
+The Scalar-specific configuration is only available for repos using the
+GVFS protocol.
+
+### Cache Server URL
+
+When using an enlistment cloned with `scalar clone` and the GVFS protocol,
+you will have a value called the cache server URL. Cache servers are a feature
+of the GVFS protocol to provide low-latency access to the on-demand object
+requests. This modifies the `gvfs.cache-server` setting in your local Git config
+file.
+
+Run `scalar cache-server --get` to see the current cache server.
+
+Run `scalar cache-server --list` to see the available cache server URLs.
+
+Run `scalar cache-server --set=<url>` to set your cache server to `<url>`.

diagnose.c

@@ -12,6 +12,7 @@
 #include "parse-options.h"
 #include "repository.h"
 #include "write-or-die.h"
+#include "config.h"
 
 struct archive_dir {
 	const char *path;
@@ -71,6 +72,39 @@ static int dir_file_stats(struct odb_source *source, void *data)
 	return 0;
 }
 
+static void dir_stats(struct strbuf *buf, const char *path)
+{
+	DIR *dir = opendir(path);
+	struct dirent *e;
+	struct stat e_stat;
+	struct strbuf file_path = STRBUF_INIT;
+	size_t base_path_len;
+
+	if (!dir)
+		return;
+
+	strbuf_addstr(buf, "Contents of ");
+	strbuf_add_absolute_path(buf, path);
+	strbuf_addstr(buf, ":\n");
+
+	strbuf_add_absolute_path(&file_path, path);
+	strbuf_addch(&file_path, '/');
+	base_path_len = file_path.len;
+
+	while ((e = readdir(dir)) != NULL)
+		if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) {
+			strbuf_setlen(&file_path, base_path_len);
+			strbuf_addstr(&file_path, e->d_name);
+			if (!stat(file_path.buf, &e_stat))
+				strbuf_addf(buf, "%-70s %16"PRIuMAX"\n",
+					    e->d_name,
+					    (uintmax_t)e_stat.st_size);
+		}
+
+	strbuf_release(&file_path);
+	closedir(dir);
+}
+
 static int count_files(struct strbuf *path)
 {
 	DIR *dir = opendir(path->buf);
@@ -185,7 +219,8 @@ int create_diagnostics_archive(struct repository *r,
 	struct strvec archiver_args = STRVEC_INIT;
 	char **argv_copy = NULL;
 	int stdout_fd = -1, archiver_fd = -1;
-	struct strbuf buf = STRBUF_INIT;
+	char *cache_server_url = NULL, *shared_cache = NULL;
+	struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
 	int res;
 	struct archive_dir archive_dirs[] = {
 		{ ".git", 0 },
@@ -220,6 +255,13 @@ int create_diagnostics_archive(struct repository *r,
 	get_version_info(&buf, 1);
 
 	strbuf_addf(&buf, "Repository root: %s\n", r->worktree);
+
+	repo_config_get_string(r, "gvfs.cache-server", &cache_server_url);
+	repo_config_get_string(r, "gvfs.sharedCache", &shared_cache);
+	strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n",
+		    cache_server_url ? cache_server_url : "None",
+		    shared_cache ? shared_cache : "None");
+
 	get_disk_info(&buf);
 	write_or_die(stdout_fd, buf.buf, buf.len);
 	strvec_pushf(&archiver_args,
@@ -250,6 +292,52 @@ int create_diagnostics_archive(struct repository *r,
 		}
 	}
 
+	if (shared_cache) {
+		size_t path_len;
+
+		strbuf_reset(&buf);
+		strbuf_addf(&path, "%s/pack", shared_cache);
+		strbuf_reset(&buf);
+		strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:");
+		dir_stats(&buf, path.buf);
+		strvec_push(&archiver_args, buf.buf);
+
+		strbuf_reset(&buf);
+		strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:");
+		loose_objs_stats(&buf, shared_cache);
+		strvec_push(&archiver_args, buf.buf);
+
+		strbuf_reset(&path);
+		strbuf_addf(&path, "%s/info", shared_cache);
+		path_len = path.len;
+
+		if (is_directory(path.buf)) {
+			DIR *dir = opendir(path.buf);
+			struct dirent *e;
+
+			while ((e = readdir(dir))) {
+				if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name))
+					continue;
+				if (e->d_type == DT_DIR)
+					continue;
+
+				strbuf_reset(&buf);
+				strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name);
+
+				strbuf_setlen(&path, path_len);
+				strbuf_addch(&path, '/');
+				strbuf_addstr(&path, e->d_name);
+
+				if (strbuf_read_file(&buf, path.buf, 0) < 0) {
+					res = error_errno(_("could not read '%s'"), path.buf);
+					goto diagnose_cleanup;
+				}
+				strvec_push(&archiver_args, buf.buf); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand
+			}
+			closedir(dir);
+		}
+	}
+
 	strvec_pushl(&archiver_args, "--prefix=",
 		     oid_to_hex(r->hash_algo->empty_tree), "--", NULL);
 
@@ -277,6 +365,8 @@ int create_diagnostics_archive(struct repository *r,
 	free(argv_copy);
 	strvec_clear(&archiver_args);
 	strbuf_release(&buf);
+	free(cache_server_url);
+	free(shared_cache);
 
 	return res;
 }

dir.c

@@ -3244,6 +3244,8 @@ static int cmp_icase(char a, char b)
 {
 	if (a == b)
 		return 0;
+	if (is_dir_sep(a))
+		return is_dir_sep(b) ? 0 : -1;
 	if (ignore_case)
 		return toupper(a) - toupper(b);
 	return a - b;