Add support for hidden CMD window

Adds support for a hidden CMD window so we can use it to replace the CMD window that flashes when we update the runtime files

Author: LocalIdentity

engine/system/sys_main.h

@@ -78,6 +78,7 @@ class sys_IMain {
 	virtual char*	ClipboardPaste() = 0;
 	virtual bool	SetWorkDir(std::filesystem::path const& newCwd = {}) = 0;
 	virtual void	SpawnProcess(std::filesystem::path cmdName, const char* argList) = 0;
+	virtual void	SpawnProcessHidden(std::filesystem::path cmdName, const char* argList) = 0;
 	virtual std::optional<std::string> OpenURL(const char* url) = 0;
 	virtual void	Error(const char* fmt, ...) = 0;
 	virtual void	Exit(const char* msg = NULL) = 0;

engine/system/win/sys_local.h

@@ -37,6 +37,7 @@ class sys_main_c: public sys_IMain {
 	char*	ClipboardPaste();
 	bool	SetWorkDir(std::filesystem::path const& newCwd = {});
 	void	SpawnProcess(std::filesystem::path cmdName, const char* argList);
+	void	SpawnProcessHidden(std::filesystem::path cmdName, const char* argList);
 	std::optional<std::string> OpenURL(const char* url); // return value has failure reason
 	void	Error(const char* fmt, ...);
 	void	Exit(const char* msg = NULL);

engine/system/win/sys_main.cpp

@@ -33,6 +33,7 @@
 #include <map>
 #include <set>
 #include <thread>
+#include <vector>
 
 #include <fmt/core.h>
 
@@ -434,6 +435,61 @@ void sys_main_c::SpawnProcess(std::filesystem::path cmdName, const char* argList
 #endif
 }
 
+void sys_main_c::SpawnProcessHidden(std::filesystem::path cmdName, const char* argList)
+{
+#ifdef _WIN32
+	if (!cmdName.has_extension()) {
+		cmdName.replace_extension(".exe");
+	}
+	auto fileStr = cmdName.wstring();
+	const char* safeArgs = argList ? argList : "";
+	wchar_t* wideArgs = (*safeArgs != '\0') ? WidenUTF8String(safeArgs) : nullptr;
+	std::wstring commandLine = L"\"" + fileStr + L"\"";
+	if (wideArgs) {
+		commandLine.push_back(L' ');
+		commandLine.append(wideArgs);
+	}
+	std::vector<wchar_t> commandBuffer(commandLine.begin(), commandLine.end());
+	commandBuffer.push_back(L'\0');
+	std::wstring workingDir;
+	if (cmdName.has_parent_path()) {
+		workingDir = cmdName.parent_path().wstring();
+	}
+	STARTUPINFOW si{};
+	PROCESS_INFORMATION pi{};
+	si.cb = sizeof(si);
+	si.dwFlags = STARTF_USESHOWWINDOW;
+	si.wShowWindow = SW_HIDE;
+	BOOL created = CreateProcessW(
+		nullptr,
+		commandBuffer.data(),
+		nullptr,
+		nullptr,
+		FALSE,
+		CREATE_NO_WINDOW,
+		nullptr,
+		workingDir.empty() ? nullptr : workingDir.data(),
+		&si,
+		&pi
+	);
+	if (wideArgs) {
+		FreeWideString(wideArgs);
+	}
+	if (!created) {
+		DWORD err = GetLastError();
+		if (con) {
+			con->Printf("^1CreateProcessW failed with code %lu, falling back to ShellExecute.^0\n", static_cast<unsigned long>(err));
+		}
+		SpawnProcess(cmdName, argList);
+		return;
+	}
+	CloseHandle(pi.hThread);
+	CloseHandle(pi.hProcess);
+#else
+	SpawnProcess(cmdName, argList);
+#endif
+}
+
 std::string GetWineHostVersion()
 {
 #ifdef _WIN32

ui_api.cpp

@@ -101,6 +101,7 @@
 ** ConPrintTable(table[, noRecurse])
 ** ConExecute("<cmd>")
 ** SpawnProcess("<cmdName>"[, "<args>"])
+** SpawnProcessHidden("<cmdName>"[, "<args>"])
 ** err = OpenURL("<url>")
 ** SetProfiling(isEnabled)
 ** Restart()
@@ -2017,6 +2018,18 @@ static int l_SpawnProcess(lua_State* L)
 	return 0;
 }
 
+static int l_SpawnProcessHidden(lua_State* L)
+{
+	ui_main_c* ui = GetUIPtr(L);
+	int n = lua_gettop(L);
+	ui->LAssert(L, n >= 1, "Usage: SpawnProcessHidden(cmdName[, args])");
+	ui->LAssert(L, lua_isstring(L, 1), "SpawnProcessHidden() argument 1: expected string, got %s", luaL_typename(L, 1));
+	auto cmdPath = std::filesystem::u8path(lua_tostring(L, 1));
+	auto args = lua_tostring(L, 2);
+	ui->sys->SpawnProcessHidden(cmdPath, args);
+	return 0;
+}
+
 static int l_OpenURL(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
@@ -2249,6 +2262,7 @@ int ui_main_c::InitAPI(lua_State* L)
 	ADDFUNC(ConClear);
 	ADDFUNC(print);
 	ADDFUNC(SpawnProcess);
+	ADDFUNC(SpawnProcessHidden);
 	ADDFUNC(OpenURL);
 	ADDFUNC(SetProfiling);
 	ADDFUNC(TakeScreenshot);