Merge branch 'nd/sparse'

* nd/sparse: (25 commits)
  t7002: test for not using external grep on skip-worktree paths
  t7002: set test prerequisite "external-grep" if supported
  grep: do not do external grep on skip-worktree entries
  commit: correctly respect skip-worktree bit
  ie_match_stat(): do not ignore skip-worktree bit with CE_MATCH_IGNORE_VALID
  tests: rename duplicate t1009
  sparse checkout: inhibit empty worktree
  Add tests for sparse checkout
  read-tree: add --no-sparse-checkout to disable sparse checkout support
  unpack-trees(): ignore worktree check outside checkout area
  unpack_trees(): apply $GIT_DIR/info/sparse-checkout to the final index
  unpack-trees(): "enable" sparse checkout and load $GIT_DIR/info/sparse-checkout
  unpack-trees.c: generalize verify_* functions
  unpack-trees(): add CE_WT_REMOVE to remove on worktree alone
  Introduce "sparse checkout"
  dir.c: export excluded_1() and add_excludes_from_file_1()
  excluded_1(): support exclude files in index
  unpack-trees(): carry skip-worktree bit over in merged_entry()
  Read .gitignore from index if it is skip-worktree
  Avoid writing to buffer in add_excludes_from_file_1()
  ...

Conflicts:
	.gitignore
	Documentation/config.txt
	Documentation/git-update-index.txt
	Makefile
	entry.c
	t/t7002-grep.sh

Author: gitster

.gitignore

@@ -158,6 +158,7 @@
 /test-delta
 /test-dump-cache-tree
 /test-genrandom
+/test-index-version
 /test-match-trees
 /test-parse-options
 /test-path-utils

Documentation/config.txt

@@ -502,6 +502,10 @@ notes should be printed.
 This setting defaults to "refs/notes/commits", and can be overridden by
 the `GIT_NOTES_REF` environment variable.
 
+core.sparseCheckout::
+	Enable "sparse checkout" feature. See section "Sparse checkout" in
+	linkgit:git-read-tree[1] for more information.
+
 add.ignore-errors::
 	Tells 'git-add' to continue adding files when some files cannot be
 	added due to indexing errors. Equivalent to the '--ignore-errors'

Documentation/git-ls-files.txt

@@ -109,6 +109,7 @@ OPTIONS
 	Identify the file status with the following tags (followed by
 	a space) at the start of each line:
 	H::	cached
+	S::	skip-worktree
 	M::	unmerged
 	R::	removed/deleted
 	C::	modified/changed

Documentation/git-read-tree.txt

@@ -10,7 +10,7 @@ SYNOPSIS
 --------
 'git read-tree' [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>]
 		[-u [--exclude-per-directory=<gitignore>] | -i]]
-		[--index-output=<file>]
+		[--index-output=<file>] [--no-sparse-checkout]
 		<tree-ish1> [<tree-ish2> [<tree-ish3>]]
 
 
@@ -110,6 +110,10 @@ OPTIONS
 	directories the index file and index output file are
 	located in.
 
+--no-sparse-checkout::
+	Disable sparse checkout support even if `core.sparseCheckout`
+	is true.
+
 <tree-ish#>::
 	The id of the tree object(s) to be read/merged.
 
@@ -360,6 +364,52 @@ middle of doing, and when your working tree is ready (i.e. you
 have finished your work-in-progress), attempt the merge again.
 
 
+Sparse checkout
+---------------
+
+"Sparse checkout" allows to sparsely populate working directory.
+It uses skip-worktree bit (see linkgit:git-update-index[1]) to tell
+Git whether a file on working directory is worth looking at.
+
+"git read-tree" and other merge-based commands ("git merge", "git
+checkout"...) can help maintaining skip-worktree bitmap and working
+directory update. `$GIT_DIR/info/sparse-checkout` is used to
+define the skip-worktree reference bitmap. When "git read-tree" needs
+to update working directory, it will reset skip-worktree bit in index
+based on this file, which uses the same syntax as .gitignore files.
+If an entry matches a pattern in this file, skip-worktree will be
+set on that entry. Otherwise, skip-worktree will be unset.
+
+Then it compares the new skip-worktree value with the previous one. If
+skip-worktree turns from unset to set, it will add the corresponding
+file back. If it turns from set to unset, that file will be removed.
+
+While `$GIT_DIR/info/sparse-checkout` is usually used to specify what
+files are in. You can also specify what files are _not_ in, using
+negate patterns. For example, to remove file "unwanted":
+
+----------------
+*
+!unwanted
+----------------
+
+Another tricky thing is fully repopulating working directory when you
+no longer want sparse checkout. You cannot just disable "sparse
+checkout" because skip-worktree are still in the index and you working
+directory is still sparsely populated. You should re-populate working
+directory with the `$GIT_DIR/info/sparse-checkout` file content as
+follows:
+
+----------------
+*
+----------------
+
+Then you can disable sparse checkout. Sparse checkout support in "git
+read-tree" and similar commands is disabled by default. You need to
+turn `core.sparseCheckout` on in order to have sparse checkout
+support.
+
+
 SEE ALSO
 --------
 linkgit:git-write-tree[1]; linkgit:git-ls-files[1];

Documentation/git-update-index.txt

@@ -15,6 +15,7 @@ SYNOPSIS
 	     [--cacheinfo <mode> <object> <file>]\*
 	     [--chmod=(+|-)x]
 	     [--assume-unchanged | --no-assume-unchanged]
+	     [--skip-worktree | --no-skip-worktree]
 	     [--ignore-submodules]
 	     [--really-refresh] [--unresolve] [--again | -g]
 	     [--info-only] [--index-info]
@@ -103,6 +104,13 @@ you will need to handle the situation manually.
 	Like '--refresh', but checks stat information unconditionally,
 	without regard to the "assume unchanged" setting.
 
+--skip-worktree::
+--no-skip-worktree::
+	When one of these flags is specified, the object name recorded
+	for the paths are not updated. Instead, these options
+	set and unset the "skip-worktree" bit for the paths. See
+	section "Skip-worktree bit" below for more information.
+
 -g::
 --again::
 	Runs 'git-update-index' itself on the paths whose index
@@ -308,6 +316,27 @@ M foo.c
 <9> now it checks with lstat(2) and finds it has been changed.
 
 
+Skip-worktree bit
+-----------------
+
+Skip-worktree bit can be defined in one (long) sentence: When reading
+an entry, if it is marked as skip-worktree, then Git pretends its
+working directory version is up to date and read the index version
+instead.
+
+To elaborate, "reading" means checking for file existence, reading
+file attributes or file content. The working directory version may be
+present or absent. If present, its content may match against the index
+version or not. Writing is not affected by this bit, content safety
+is still first priority. Note that Git _can_ update working directory
+file, that is marked skip-worktree, if it is safe to do so (i.e.
+working directory version matches index version)
+
+Although this bit looks similar to assume-unchanged bit, its goal is
+different from assume-unchanged bit's. Skip-worktree also takes
+precedence over assume-unchanged bit when both are set.
+
+
 Configuration
 -------------
 

Documentation/technical/api-directory-listing.txt

@@ -58,6 +58,9 @@ The result of the enumeration is left in these fields::
 Calling sequence
 ----------------
 
+Note: index may be looked at for .gitignore files that are CE_SKIP_WORKTREE
+marked. If you to exclude files, make sure you have loaded index first.
+
 * Prepare `struct dir_struct dir` and clear it with `memset(&dir, 0,
   sizeof(dir))`.
 

Makefile

@@ -1782,6 +1782,7 @@ TEST_PROGRAMS_NEED_X += test-parse-options
 TEST_PROGRAMS_NEED_X += test-path-utils
 TEST_PROGRAMS_NEED_X += test-sha1
 TEST_PROGRAMS_NEED_X += test-sigchain
+TEST_PROGRAMS_NEED_X += test-index-version
 
 TEST_PROGRAMS = $(patsubst %,%$X,$(TEST_PROGRAMS_NEED_X))
 

builtin-apply.c

@@ -2666,7 +2666,7 @@ static int verify_index_match(struct cache_entry *ce, struct stat *st)
 			return -1;
 		return 0;
 	}
-	return ce_match_stat(ce, st, CE_MATCH_IGNORE_VALID);
+	return ce_match_stat(ce, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
 }
 
 static int check_preimage(struct patch *patch, struct cache_entry **ce, struct stat *st)

builtin-clean.c

@@ -75,11 +75,13 @@ int cmd_clean(int argc, const char **argv, const char *prefix)
 
 	dir.flags |= DIR_SHOW_OTHER_DIRECTORIES;
 
+	if (read_cache() < 0)
+		die("index file corrupt");
+
 	if (!ignored)
 		setup_standard_excludes(&dir);
 
 	pathspec = get_pathspec(prefix, argv);
-	read_cache();
 
 	fill_directory(&dir, pathspec);
 

builtin-commit.c

@@ -183,11 +183,15 @@ static int list_paths(struct string_list *list, const char *with_tree,
 
 	for (i = 0; i < active_nr; i++) {
 		struct cache_entry *ce = active_cache[i];
+		struct string_list_item *item;
+
 		if (ce->ce_flags & CE_UPDATE)
 			continue;
 		if (!match_pathspec(pattern, ce->name, ce_namelen(ce), 0, m))
 			continue;
-		string_list_insert(ce->name, list);
+		item = string_list_insert(ce->name, list);
+		if (ce_skip_worktree(ce))
+			item->util = item; /* better a valid pointer than a fake one */
 	}
 
 	return report_path_error(m, pattern, prefix ? strlen(prefix) : 0);
@@ -200,6 +204,10 @@ static void add_remove_files(struct string_list *list)
 		struct stat st;
 		struct string_list_item *p = &(list->items[i]);
 
+		/* p->util is skip-worktree */
+		if (p->util)
+			continue;
+
 		if (!lstat(p->string, &st)) {
 			if (add_to_cache(p->string, &st, 0))
 				die("updating files failed");