Merge branch 'fscache'

Author: dscho

Documentation/config/core.adoc

@@ -690,6 +690,12 @@ relatively high IO latencies.  When enabled, Git will do the
 index comparison to the filesystem data in parallel, allowing
 overlapping IO's.  Defaults to true.
 
+core.fscache::
+	Enable additional caching of file system data for some operations.
++
+Git for Windows uses this to bulk-read and cache lstat data of entire
+directories (instead of doing lstat file by file).
+
 core.unsetenvvars::
 	Windows-only: comma-separated list of environment variables'
 	names that need to be unset before spawning any other process.

builtin/add.c

@@ -491,6 +491,10 @@ int cmd_add(int argc,
 	die_in_unpopulated_submodule(repo->index, prefix);
 	die_path_inside_submodule(repo->index, &pathspec);
 
+	enable_fscache(1);
+	/* We do not really re-read the index but update the up-to-date flags */
+	preload_index(repo->index, &pathspec, 0);
+
 	if (add_new_files) {
 		int baselen;
 
@@ -603,5 +607,6 @@ int cmd_add(int argc,
 	free(ps_matched);
 	dir_clear(&dir);
 	clear_pathspec(&pathspec);
+	enable_fscache(0);
 	return exit_status;
 }

builtin/checkout.c

@@ -415,6 +415,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
 	if (pc_workers > 1)
 		init_parallel_checkout();
 
+	enable_fscache(1);
 	for (pos = 0; pos < the_repository->index->cache_nr; pos++) {
 		struct cache_entry *ce = the_repository->index->cache[pos];
 		if (ce->ce_flags & CE_MATCHED) {
@@ -440,6 +441,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
 		errs |= run_parallel_checkout(&state, pc_workers, pc_threshold,
 					      NULL, NULL);
 	mem_pool_discard(&ce_mem_pool, should_validate_cache_entries());
+	enable_fscache(0);
 	remove_marked_cache_entries(the_repository->index, 1);
 	remove_scheduled_dirs();
 	errs |= finish_delayed_checkout(&state, opts->show_progress);

builtin/commit.c

@@ -1618,6 +1618,7 @@ struct repository *repo UNUSED)
 		       PATHSPEC_PREFER_FULL,
 		       prefix, argv);
 
+	enable_fscache(1);
 	if (status_format != STATUS_FORMAT_PORCELAIN &&
 	    status_format != STATUS_FORMAT_PORCELAIN_V2)
 		progress_flag = REFRESH_PROGRESS;

compat/mingw-posix.h

@@ -343,6 +343,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp)
 	return 0;
 }
 
+/*
+ * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
+ * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
+ */
+static inline long long filetime_to_hnsec(const FILETIME *ft)
+{
+	long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
+	/* Windows to Unix Epoch conversion */
+	return winTime - 116444736000000000LL;
+}
+
 /*
  * Use mingw specific stat()/lstat()/fstat() implementations on Windows,
  * including our own struct stat with 64 bit st_size and nanosecond-precision
@@ -359,6 +370,13 @@ struct timespec {
 #endif
 #endif
 
+static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
+{
+	long long hnsec = filetime_to_hnsec(ft);
+	ts->tv_sec = (time_t)(hnsec / 10000000);
+	ts->tv_nsec = (hnsec % 10000000) * 100;
+}
+
 struct mingw_stat {
     _dev_t st_dev;
     _ino_t st_ino;
@@ -391,7 +409,7 @@ int mingw_fstat(int fd, struct stat *buf);
 #ifdef lstat
 #undef lstat
 #endif
-#define lstat mingw_lstat
+extern int (*lstat)(const char *file_name, struct stat *buf);
 
 
 int mingw_utime(const char *file_name, const struct utimbuf *times);

compat/mingw.c

@@ -248,6 +248,7 @@ enum hide_dotfiles_type {
 
 static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY;
 static char *unset_environment_variables;
+int core_fscache;
 
 int mingw_core_config(const char *var, const char *value,
 		      const struct config_context *ctx UNUSED,
@@ -261,6 +262,11 @@ int mingw_core_config(const char *var, const char *value,
 		return 0;
 	}
 
+	if (!strcmp(var, "core.fscache")) {
+		core_fscache = git_config_bool(var, value);
+		return 0;
+	}
+
 	if (!strcmp(var, "core.unsetenvvars")) {
 		if (!value)
 			return config_error_nonbool(var);
@@ -883,24 +889,6 @@ int mingw_chmod(const char *filename, int mode)
 	return _wchmod(wfilename, mode);
 }
 
-/*
- * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
- * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
- */
-static inline long long filetime_to_hnsec(const FILETIME *ft)
-{
-	long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
-	/* Windows to Unix Epoch conversion */
-	return winTime - 116444736000000000LL;
-}
-
-static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
-{
-	long long hnsec = filetime_to_hnsec(ft);
-	ts->tv_sec = (time_t)(hnsec / 10000000);
-	ts->tv_nsec = (hnsec % 10000000) * 100;
-}
-
 /**
  * Verifies that safe_create_leading_directories() would succeed.
  */
@@ -1040,6 +1028,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf)
 	return do_lstat(follow, alt_name, buf);
 }
 
+int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat;
+
 static int get_file_info_by_handle(HANDLE hnd, struct stat *buf)
 {
 	BY_HANDLE_FILE_INFORMATION fdata;

compat/mingw.h

@@ -1,5 +1,7 @@
 #include "mingw-posix.h"
 
+extern int core_fscache;
+
 struct config_context;
 int mingw_core_config(const char *var, const char *value,
 		      const struct config_context *ctx, void *cb);

compat/win32/dirent.c

@@ -1,15 +1,21 @@
 #include "../../git-compat-util.h"
 
-struct DIR {
-	struct dirent dd_dir; /* includes d_type */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+typedef struct dirent_DIR {
+	struct DIR base_dir;  /* extend base struct DIR */
 	HANDLE dd_handle;     /* FindFirstFile handle */
 	int dd_stat;          /* 0-based index */
-};
+	struct dirent dd_dir; /* includes d_type */
+} dirent_DIR;
+#pragma GCC diagnostic pop
+
+DIR *(*opendir)(const char *dirname) = dirent_opendir;
 
 static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
 {
-	/* convert UTF-16 name to UTF-8 */
-	xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name));
+	/* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */
+	xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3);
 
 	/* Set file type, based on WIN32_FIND_DATA */
 	if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
@@ -18,41 +24,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
 		ent->d_type = DT_REG;
 }
 
-DIR *opendir(const char *name)
-{
-	wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
-	WIN32_FIND_DATAW fdata;
-	HANDLE h;
-	int len;
-	DIR *dir;
-
-	/* convert name to UTF-16 and check length < MAX_PATH */
-	if ((len = xutftowcs_path(pattern, name)) < 0)
-		return NULL;
-
-	/* append optional '/' and wildcard '*' */
-	if (len && !is_dir_sep(pattern[len - 1]))
-		pattern[len++] = '/';
-	pattern[len++] = '*';
-	pattern[len] = 0;
-
-	/* open find handle */
-	h = FindFirstFileW(pattern, &fdata);
-	if (h == INVALID_HANDLE_VALUE) {
-		DWORD err = GetLastError();
-		errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
-		return NULL;
-	}
-
-	/* initialize DIR structure and copy first dir entry */
-	dir = xmalloc(sizeof(DIR));
-	dir->dd_handle = h;
-	dir->dd_stat = 0;
-	finddata2dirent(&dir->dd_dir, &fdata);
-	return dir;
-}
-
-struct dirent *readdir(DIR *dir)
+static struct dirent *dirent_readdir(dirent_DIR *dir)
 {
 	if (!dir) {
 		errno = EBADF; /* No set_errno for mingw */
@@ -79,7 +51,7 @@ struct dirent *readdir(DIR *dir)
 	return &dir->dd_dir;
 }
 
-int closedir(DIR *dir)
+static int dirent_closedir(dirent_DIR *dir)
 {
 	if (!dir) {
 		errno = EBADF;
@@ -90,3 +62,39 @@ int closedir(DIR *dir)
 	free(dir);
 	return 0;
 }
+
+DIR *dirent_opendir(const char *name)
+{
+	wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
+	WIN32_FIND_DATAW fdata;
+	HANDLE h;
+	int len;
+	dirent_DIR *dir;
+
+	/* convert name to UTF-16 and check length < MAX_PATH */
+	if ((len = xutftowcs_path(pattern, name)) < 0)
+		return NULL;
+
+	/* append optional '/' and wildcard '*' */
+	if (len && !is_dir_sep(pattern[len - 1]))
+		pattern[len++] = '/';
+	pattern[len++] = '*';
+	pattern[len] = 0;
+
+	/* open find handle */
+	h = FindFirstFileW(pattern, &fdata);
+	if (h == INVALID_HANDLE_VALUE) {
+		DWORD err = GetLastError();
+		errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
+		return NULL;
+	}
+
+	/* initialize DIR structure and copy first dir entry */
+	dir = xmalloc(sizeof(dirent_DIR) + MAX_PATH);
+	dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir;
+	dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir;
+	dir->dd_handle = h;
+	dir->dd_stat = 0;
+	finddata2dirent(&dir->dd_dir, &fdata);
+	return (DIR*) dir;
+}

compat/win32/dirent.h

@@ -1,20 +1,34 @@
 #ifndef DIRENT_H
 #define DIRENT_H
 
-typedef struct DIR DIR;
-
 #define DT_UNKNOWN 0
 #define DT_DIR     1
 #define DT_REG     2
 #define DT_LNK     3
 
 struct dirent {
-	unsigned char d_type;      /* file type to prevent lstat after readdir */
-	char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */
+	unsigned char d_type; /* file type to prevent lstat after readdir */
+	char d_name[/* FLEX_ARRAY */]; /* file name */
 };
 
-DIR *opendir(const char *dirname);
-struct dirent *readdir(DIR *dir);
-int closedir(DIR *dir);
+/*
+ * Base DIR structure, contains pointers to readdir/closedir implementations so
+ * that opendir may choose a concrete implementation on a call-by-call basis.
+ */
+typedef struct DIR {
+	struct dirent *(*preaddir)(struct DIR *dir);
+	int (*pclosedir)(struct DIR *dir);
+} DIR;
+
+/* default dirent implementation */
+extern DIR *dirent_opendir(const char *dirname);
+
+#define opendir git_opendir
+
+/* current dirent implementation */
+extern DIR *(*opendir)(const char *dirname);
+
+#define readdir(dir) (dir->preaddir(dir))
+#define closedir(dir) (dir->pclosedir(dir))
 
 #endif /* DIRENT_H */