Win32: use low-level memory allocation during initialization

As of d41489a "Add more large blob test cases", git's high-level memory
allocation functions (xmalloc, xmemdupz etc.) access the environment to
simulate limited memory in tests (see 'getenv("GIT_ALLOC_LIMIT")' in
memory_limit_check()). These functions should not be used before the
environment is fully initialized (particularly not to initialize the
environment itself).

The current solution ('environ = NULL; ALLOC_GROW(environ...)') only works
because MSVCRT's getenv() reinitializes environ when it is NULL (i.e. it
leaves us with two sets of unusabe (non-UTF-8) and unfreeable (CRT-
allocated) environments).

Add our own set of malloc-or-die functions to be used in startup code.

Also check the result of __wgetmainargs, which may fail if there's not
enough memory for wide-char arguments and environment.

This patch is in preparation of the sorted environment feature, which
completely replaces MSVCRT's getenv() implementation.

Signed-off-by: Karsten Blees <[email protected]>

Author: kblees

compat/mingw.c

@@ -2001,16 +2001,37 @@ typedef struct {
 extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob,
 		_startupinfo *si);
 
+static NORETURN void die_startup()
+{
+	fputs("fatal: not enough memory for initialization", stderr);
+	exit(128);
+}
+
+static void *malloc_startup(size_t size)
+{
+	void *result = malloc(size);
+	if (!result)
+		die_startup();
+	return result;
+}
+
+static char *wcstoutfdup_startup(char *buffer, const wchar_t *wcs, size_t len)
+{
+	len = xwcstoutf(buffer, wcs, len) + 1;
+	return memcpy(malloc_startup(len), buffer, len);
+}
+
 void mingw_startup()
 {
-	int i, len, maxlen, argc;
+	int i, maxlen, argc;
 	char *buffer;
 	wchar_t **wenv, **wargv;
 	_startupinfo si;
 
 	/* get wide char arguments and environment */
 	si.newmode = 0;
-	__wgetmainargs(&argc, &wargv, &wenv, _CRT_glob, &si);
+	if (__wgetmainargs(&argc, &wargv, &wenv, _CRT_glob, &si) < 0)
+		die_startup();
 
 	/* determine size of argv and environ conversion buffer */
 	maxlen = wcslen(_wpgmptr);
@@ -2019,26 +2040,25 @@ void mingw_startup()
 	for (i = 0; wenv[i]; i++)
 		maxlen = max(maxlen, wcslen(wenv[i]));
 
-	/* nedmalloc can't free CRT memory, allocate resizable environment list */
-	environ = NULL;
+	/*
+	 * nedmalloc can't free CRT memory, allocate resizable environment
+	 * list. Note that xmalloc / xmemdupz etc. call getenv, so we cannot
+	 * use it while initializing the environment itself.
+	 */
 	environ_size = i + 1;
-	ALLOC_GROW(environ, environ_size * sizeof(char*), environ_alloc);
+	environ_alloc = alloc_nr(environ_size * sizeof(char*));
+	environ = malloc_startup(environ_alloc);
 
 	/* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */
 	maxlen = 3 * maxlen + 1;
-	buffer = xmalloc(maxlen);
+	buffer = malloc_startup(maxlen);
 
 	/* convert command line arguments and environment to UTF-8 */
-	len = xwcstoutf(buffer, _wpgmptr, maxlen);
-	__argv[0] = xmemdupz(buffer, len);
-	for (i = 1; i < argc; i++) {
-		len = xwcstoutf(buffer, wargv[i], maxlen);
-		__argv[i] = xmemdupz(buffer, len);
-	}
-	for (i = 0; wenv[i]; i++) {
-		len = xwcstoutf(buffer, wenv[i], maxlen);
-		environ[i] = xmemdupz(buffer, len);
-	}
+	__argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen);
+	for (i = 1; i < argc; i++)
+		__argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen);
+	for (i = 0; wenv[i]; i++)
+		environ[i] = wcstoutfdup_startup(buffer, wenv[i], maxlen);
 	environ[i] = NULL;
 	free(buffer);