git on Mac OS and precomposed unicode

Mac OS X mangles file names containing unicode on file systems HFS+,
VFAT or SAMBA.  When a file using unicode code points outside ASCII
is created on a HFS+ drive, the file name is converted into
decomposed unicode and written to disk. No conversion is done if
the file name is already decomposed unicode.

Calling open("\xc3\x84", ...) with a precomposed "Ä" yields the same
result as open("\x41\xcc\x88",...) with a decomposed "Ä".

As a consequence, readdir() returns the file names in decomposed
unicode, even if the user expects precomposed unicode.  Unlike on
HFS+, Mac OS X stores files on a VFAT drive (e.g. an USB drive) in
precomposed unicode, but readdir() still returns file names in
decomposed unicode.  When a git repository is stored on a network
share using SAMBA, file names are send over the wire and written to
disk on the remote system in precomposed unicode, but Mac OS X
readdir() returns decomposed unicode to be compatible with its
behaviour on HFS+ and VFAT.

The unicode decomposition causes many problems:

- The names "git add" and other commands get from the end user may
  often be precomposed form (the decomposed form is not easily input
  from the keyboard), but when the commands read from the filesystem
  to see what it is going to update the index with already is on the
  filesystem, readdir() will give decomposed form, which is different.

- Similarly "git log", "git mv" and all other commands that need to
  compare pathnames found on the command line (often but not always
  precomposed form; a command line input resulting from globbing may
  be in decomposed) with pathnames found in the tree objects (should
  be precomposed form to be compatible with other systems and for
  consistency in general).

- The same for names stored in the index, which should be
  precomposed, that may need to be compared with the names read from
  readdir().

NFS mounted from Linux is fully transparent and does not suffer from
the above.

As Mac OS X treats precomposed and decomposed file names as equal,
we can

 - wrap readdir() on Mac OS X to return the precomposed form, and

 - normalize decomposed form given from the command line also to the
   precomposed form,

to ensure that all pathnames used in Git are always in the
precomposed form.  This behaviour can be requested by setting
"core.precomposedunicode" configuration variable to true.

The code in compat/precomposed_utf8.c implements basically 4 new
functions: precomposed_utf8_opendir(), precomposed_utf8_readdir(),
precomposed_utf8_closedir() and precompose_argv().  The first three
are to wrap opendir(3), readdir(3), and closedir(3) functions.

The argv[] conversion allows to use the TAB filename completion done
by the shell on command line.  It tolerates other tools which use
readdir() to feed decomposed file names into git.

When creating a new git repository with "git init" or "git clone",
"core.precomposedunicode" will be set "false".

The user needs to activate this feature manually.  She typically
sets core.precomposedunicode to "true" on HFS and VFAT, or file
systems mounted via SAMBA.

Helped-by: Junio C Hamano <[email protected]>
Signed-off-by: Torsten Bögershausen <[email protected]>
Signed-off-by: Junio C Hamano <[email protected]>

Author: tboegi

Documentation/config.txt

@@ -210,6 +210,15 @@ The default is false, except linkgit:git-clone[1] or linkgit:git-init[1]
 will probe and set core.ignorecase true if appropriate when the repository
 is created.
 
+core.precomposeunicode::
+	This option is only used by Mac OS implementation of git.
+	When core.precomposeunicode=true, git reverts the unicode decomposition
+	of filenames done by Mac OS. This is useful when sharing a repository
+	between Mac OS and Linux or Windows.
+	(Git for Windows 1.7.10 or higher is needed, or git under cygwin 1.7).
+	When false, file names are handled fully transparent by git,
+	which is backward compatible with older versions of git.
+
 core.trustctime::
 	If false, the ctime differences between the index and the
 	working tree are ignored; useful when the inode change time

Makefile

@@ -600,6 +600,7 @@ LIB_H += compat/bswap.h
 LIB_H += compat/cygwin.h
 LIB_H += compat/mingw.h
 LIB_H += compat/obstack.h
+LIB_H += compat/precompose_utf8.h
 LIB_H += compat/terminal.h
 LIB_H += compat/win32/dirent.h
 LIB_H += compat/win32/poll.h
@@ -990,6 +991,8 @@ ifeq ($(uname_S),Darwin)
 	NO_MEMMEM = YesPlease
 	USE_ST_TIMESPEC = YesPlease
 	HAVE_DEV_TTY = YesPlease
+	COMPAT_OBJS += compat/precompose_utf8.o
+	BASIC_CFLAGS += -DPRECOMPOSE_UNICODE
 endif
 ifeq ($(uname_S),SunOS)
 	NEEDS_SOCKET = YesPlease

builtin/init-db.c

@@ -290,6 +290,7 @@ static int create_default_files(const char *template_path)
 		strcpy(path + len, "CoNfIg");
 		if (!access(path, F_OK))
 			git_config_set("core.ignorecase", "true");
+		probe_utf8_pathname_composition(path, len);
 	}
 
 	return reinit;

cache.h

@@ -560,6 +560,7 @@ extern int read_replace_refs;
 extern int fsync_object_files;
 extern int core_preload_index;
 extern int core_apply_sparse_checkout;
+extern int precomposed_unicode;
 
 enum branch_track {
 	BRANCH_TRACK_UNSPECIFIED = -1,

compat/precompose_utf8.c

@@ -0,0 +1,190 @@
+/*
+ * Converts filenames from decomposed unicode into precomposed unicode.
+ * Used on MacOS X.
+*/
+
+
+#define PRECOMPOSE_UNICODE_C
+
+#include "cache.h"
+#include "utf8.h"
+#include "precompose_utf8.h"
+
+typedef char *iconv_ibp;
+const static char *repo_encoding = "UTF-8";
+const static char *path_encoding = "UTF-8-MAC";
+
+
+static size_t has_utf8(const char *s, size_t maxlen, size_t *strlen_c)
+{
+	const uint8_t *utf8p = (const uint8_t*) s;
+	size_t strlen_chars = 0;
+	size_t ret = 0;
+
+	if ((!utf8p) || (!*utf8p)) {
+		return 0;
+	}
+
+	while((*utf8p) && maxlen) {
+		if (*utf8p & 0x80)
+			ret++;
+		strlen_chars++;
+		utf8p++;
+		maxlen--;
+	}
+	if (strlen_c)
+		*strlen_c = strlen_chars;
+
+	return ret;
+}
+
+
+void probe_utf8_pathname_composition(char *path, int len)
+{
+	const static char *auml_nfc = "\xc3\xa4";
+	const static char *auml_nfd = "\x61\xcc\x88";
+	int output_fd;
+	if (precomposed_unicode != -1)
+		return; /* We found it defined in the global config, respect it */
+	path[len] = 0;
+	strcpy(path + len, auml_nfc);
+	output_fd = open(path, O_CREAT|O_EXCL|O_RDWR, 0600);
+	if (output_fd >=0) {
+		close(output_fd);
+		path[len] = 0;
+		strcpy(path + len, auml_nfd);
+		/* Indicate to the user, that we can configure it to true */
+		if (0 == access(path, R_OK))
+			git_config_set("core.precomposeunicode", "false");
+			/* To be backward compatible, set precomposed_unicode to 0 */
+		precomposed_unicode = 0;
+		path[len] = 0;
+		strcpy(path + len, auml_nfc);
+		unlink(path);
+	}
+}
+
+
+void precompose_argv(int argc, const char **argv)
+{
+	int i = 0;
+	const char *oldarg;
+	char *newarg;
+	iconv_t ic_precompose;
+
+	if (precomposed_unicode != 1)
+		return;
+
+	ic_precompose = iconv_open(repo_encoding, path_encoding);
+	if (ic_precompose == (iconv_t) -1)
+		return;
+
+	while (i < argc) {
+		size_t namelen;
+		oldarg = argv[i];
+		if (has_utf8(oldarg, (size_t)-1, &namelen)) {
+			newarg = reencode_string_iconv(oldarg, namelen, ic_precompose);
+			if (newarg)
+				argv[i] = newarg;
+		}
+		i++;
+	}
+	iconv_close(ic_precompose);
+}
+
+
+PREC_DIR *precompose_utf8_opendir(const char *dirname)
+{
+	PREC_DIR *prec_dir = xmalloc(sizeof(PREC_DIR));
+	prec_dir->dirent_nfc = xmalloc(sizeof(dirent_prec_psx));
+	prec_dir->dirent_nfc->max_name_len = sizeof(prec_dir->dirent_nfc->d_name);
+
+	prec_dir->dirp = opendir(dirname);
+	if (!prec_dir->dirp) {
+		free(prec_dir->dirent_nfc);
+		free(prec_dir);
+		return NULL;
+	} else {
+		int ret_errno = errno;
+		prec_dir->ic_precompose = iconv_open(repo_encoding, path_encoding);
+		/* if iconv_open() fails, die() in readdir() if needed */
+		errno = ret_errno;
+	}
+
+	return prec_dir;
+}
+
+struct dirent_prec_psx *precompose_utf8_readdir(PREC_DIR *prec_dir)
+{
+	struct dirent *res;
+	res = readdir(prec_dir->dirp);
+	if (res) {
+		size_t namelenz = strlen(res->d_name) + 1; /* \0 */
+		size_t new_maxlen = namelenz;
+
+		int ret_errno = errno;
+
+		if (new_maxlen > prec_dir->dirent_nfc->max_name_len) {
+			size_t new_len = sizeof(dirent_prec_psx) + new_maxlen -
+				sizeof(prec_dir->dirent_nfc->d_name);
+
+			prec_dir->dirent_nfc = xrealloc(prec_dir->dirent_nfc, new_len);
+			prec_dir->dirent_nfc->max_name_len = new_maxlen;
+		}
+
+		prec_dir->dirent_nfc->d_ino  = res->d_ino;
+		prec_dir->dirent_nfc->d_type = res->d_type;
+
+		if ((precomposed_unicode == 1) && has_utf8(res->d_name, (size_t)-1, NULL)) {
+			if (prec_dir->ic_precompose == (iconv_t)-1) {
+				die("iconv_open(%s,%s) failed, but needed:\n"
+						"    precomposed unicode is not supported.\n"
+						"    If you wnat to use decomposed unicode, run\n"
+						"    \"git config core.precomposeunicode false\"\n",
+						repo_encoding, path_encoding);
+			} else {
+				iconv_ibp	cp = (iconv_ibp)res->d_name;
+				size_t inleft = namelenz;
+				char *outpos = &prec_dir->dirent_nfc->d_name[0];
+				size_t outsz = prec_dir->dirent_nfc->max_name_len;
+				size_t cnt;
+				errno = 0;
+				cnt = iconv(prec_dir->ic_precompose, &cp, &inleft, &outpos, &outsz);
+				if (errno || inleft) {
+					/*
+					 * iconv() failed and errno could be E2BIG, EILSEQ, EINVAL, EBADF
+					 * MacOS X avoids illegal byte sequemces.
+					 * If they occur on a mounted drive (e.g. NFS) it is not worth to
+					 * die() for that, but rather let the user see the original name
+					*/
+					namelenz = 0; /* trigger strlcpy */
+				}
+			}
+		}
+		else
+			namelenz = 0;
+
+		if (!namelenz)
+			strlcpy(prec_dir->dirent_nfc->d_name, res->d_name,
+							prec_dir->dirent_nfc->max_name_len);
+
+		errno = ret_errno;
+		return prec_dir->dirent_nfc;
+	}
+	return NULL;
+}
+
+
+int precompose_utf8_closedir(PREC_DIR *prec_dir)
+{
+	int ret_value;
+	int ret_errno;
+	ret_value = closedir(prec_dir->dirp);
+	ret_errno = errno;
+	if (prec_dir->ic_precompose != (iconv_t)-1)
+		iconv_close(prec_dir->ic_precompose);
+	free(prec_dir->dirent_nfc);
+	free(prec_dir);
+	errno = ret_errno;
+	return ret_value;
+}

compat/precompose_utf8.h

@@ -0,0 +1,45 @@
+#ifndef PRECOMPOSE_UNICODE_H
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <iconv.h>
+
+
+typedef struct dirent_prec_psx {
+	ino_t d_ino;            /* Posix */
+	size_t max_name_len;    /* See below */
+	unsigned char d_type;   /* available on all systems git runs on */
+
+	/*
+	 * See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dirent.h.html
+	 * NAME_MAX + 1 should be enough, but some systems have
+	 * NAME_MAX=255 and strlen(d_name) may return 508 or 510
+	 * Solution: allocate more when needed, see precompose_utf8_readdir()
+	 */
+	char   d_name[NAME_MAX+1];
+} dirent_prec_psx;
+
+
+typedef struct {
+	iconv_t ic_precompose;
+	DIR *dirp;
+	struct dirent_prec_psx *dirent_nfc;
+} PREC_DIR;
+
+void precompose_argv(int argc, const char **argv);
+void probe_utf8_pathname_composition(char *, int);
+
+PREC_DIR *precompose_utf8_opendir(const char *dirname);
+struct dirent_prec_psx *precompose_utf8_readdir(PREC_DIR *dirp);
+int precompose_utf8_closedir(PREC_DIR *dirp);
+
+#ifndef PRECOMPOSE_UNICODE_C
+#define dirent dirent_prec_psx
+#define opendir(n) precompose_utf8_opendir(n)
+#define readdir(d) precompose_utf8_readdir(d)
+#define closedir(d) precompose_utf8_closedir(d)
+#define DIR PREC_DIR
+#endif /* PRECOMPOSE_UNICODE_C */
+
+#define  PRECOMPOSE_UNICODE_H
+#endif /* PRECOMPOSE_UNICODE_H */

config.c

@@ -758,6 +758,11 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.precomposeunicode")) {
+		precomposed_unicode = git_config_bool(var, value);
+		return 0;
+	}
+
 	/* Add other config variables here and to Documentation/config.txt. */
 	return 0;
 }

environment.c

@@ -58,6 +58,7 @@ char *notes_ref_name;
 int grafts_replace_parents = 1;
 int core_apply_sparse_checkout;
 int merge_log_config = -1;
+int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */
 struct startup_info *startup_info;
 unsigned long pack_size_limit_cfg;
 

git-compat-util.h

@@ -153,6 +153,15 @@
 #endif
 #endif
 
+/* used on Mac OS X */
+#ifdef PRECOMPOSE_UNICODE
+#include "compat/precompose_utf8.h"
+#else
+#define precompose_str(in,i_nfd2nfc)
+#define precompose_argv(c,v)
+#define probe_utf8_pathname_composition(a,b)
+#endif
+
 #ifndef NO_LIBGEN_H
 #include <libgen.h>
 #else

parse-options.c

@@ -476,6 +476,7 @@ int parse_options(int argc, const char **argv, const char *prefix,
 		usage_with_options(usagestr, options);
 	}
 
+	precompose_argv(argc, argv);
 	return parse_options_end(&ctx);
 }