Replace proprietary dxil.dll code signing blob with FOSS alternative

Normal usage of dxcompiler requires that you use a proprietary `dxil.dll`
distributed by Microsoft. If this is not found next to the dxcompiler binary
when compiling shaders, then the final shader binary will not be signed and
as a result will not run on others' machines.

DXC is also written with the assumption that dxil.dll and dxcompiler.dll itself
are loaded as dynamic libraries. This, combined with dxil.dll being a proprietary
blob, is annoying for users of Mach engine and [many others](https://bevy-cheatbook.github.io/platforms/windows.html#dxc-compiler-support)
as it just complicates application distribution further.

This commit replaces dxcompiler.dll runtime loading behavior with an 'emulated'
dynamic library, effectively completing our effort to make dxcompiler a static
library.

dxil.dll is closed-source, so we cannot simply patch it in the same way. To fix
this, we outright disable runtime loading of dxil.dll and silence warnings related
to it not being present / the final binary not being code signed. Instead, once
the compiler would emit a compiled shader blob, we perform our own code signing
algorithm (Mach Siegbert Vogt DXCSA) which results in a bitwise identical compiled
shader, and thus dxil.dll is no longer needed to perform code signing of shaders.

Ultimately, this means mach-dxcompiler can be a fully static library, code signing
included.

Signed-off-by: Stephen Gutekanst <[email protected]>

Author: Stephen Gutekanst

include/dxc/Support/dxcapi.use.h

@@ -17,21 +17,26 @@
 namespace dxc {
 
 extern const char *kDxCompilerLib;
-// Mach change start: static dxil
-// extern const char *kDxilLib;
+extern const char *kDxilLib;
+
+// Mach change start: static dxcompiler/dxil
+extern "C" BOOL MachDxcompilerInvokeDllMain();
+extern "C" void MachDxcompilerInvokeDllShutdown();
+static bool dxcompiler_dll_loaded = false;
 // Mach change end
 
 // Helper class to dynamically load the dxcompiler or a compatible libraries.
 class DxcDllSupport {
 protected:
-  // Mach change start: static dxcompiler
+  // Mach change start: static dxcompiler/dxil
   BOOL m_initialized;
+  std::string m_dllName;
   // Mach change end
   HMODULE m_dll;
   DxcCreateInstanceProc m_createFn;
   DxcCreateInstance2Proc m_createFn2;
 
-// Mach change start: static dxcompiler
+// Mach change start: static dxcompiler/dxil
 //   HRESULT InitializeInternal(LPCSTR dllName, LPCSTR fnName) {
 //     if (m_dll != nullptr)
 //       return S_OK;
@@ -79,10 +84,44 @@ class DxcDllSupport {
 //     return S_OK;
 //   }
   HRESULT InitializeInternal(LPCSTR dllName, LPCSTR fnName) {
+    // The compilation process occurs as follows:
+    //
+    // 1. Compilation begins
+    // 2. InitializeInternal(kDxCompilerLib, "DxcCreateInstance") is called
+    // 3. MachDxcompilerInvokeDllMain() is invoked..
+    //   3a. which calls dxcompiler.dll's DllMain entrypoint
+    //   3b. which triggers loading of dxil.dll
+    // 4. InitializeInternal(kDxilLib, "DxcCreateInstance") is called
+    //   4a. E_FAIL is returned, indicating dxil.dll is not present
+    //   4b. We silence the warning related to dxil.dll not being present and
+    //       code signing not occuring (commented out)
+    // 5. We perform Mach Siegbert Vogt DXCSA on the final container/blob before the
+    //    compiler writes it to disk.
+    //
+    // Look for "DXCSA" in the codebase to see where signing happens.
+
+    // Store which DLL this is for later, so we can MachDxcompilerInvokeDllShutdown later.
+    m_dllName = dllName;
+
+    // If this is dxil.dll, emulate that we do not have it.
+    if (strcmp(dllName, kDxilLib) == 0) {
+      return E_FAIL;
+    }
+
+    // If this is dxcompiler.dll, emulate as if we loaded it in-process.
     if (strcmp(fnName, "DxcCreateInstance") == 0) {
       m_initialized = true;
       m_createFn = &DxcCreateInstance;
       m_createFn2 = &DxcCreateInstance2;
+
+      // If this is the first time this is called, invoke DllMain() 
+      if (!dxcompiler_dll_loaded) {
+        if (!MachDxcompilerInvokeDllMain()) {
+          fprintf(stderr, "mach-dxcompiler: MachDxcompilerInvokeDllMain failed\n");
+          return E_FAIL;
+        }
+        dxcompiler_dll_loaded = true;
+      }
       return S_OK;
     }
     fprintf(stderr, "mach-dxcompiler: InitializeInternal: unknown GetProcAddress name: %s\n", fnName);
@@ -91,15 +130,17 @@ class DxcDllSupport {
 // Mach change end
 
 public:
-  // Mach change start: static dxcompiler
+  // Mach change start: static dxcompiler/dxil
   // DxcDllSupport() : m_dll(nullptr), m_createFn(nullptr), m_createFn2(nullptr) {}
   DxcDllSupport() : m_initialized(false), m_dll(nullptr), m_createFn(nullptr), m_createFn2(nullptr) {}
   // Mach change end
 
   DxcDllSupport(DxcDllSupport &&other) {
-    // Mach change start: static dxcompiler
+    // Mach change start: static dxcompiler/dxil
     m_initialized = other.m_initialized;
     other.m_initialized = false;
+    m_dllName = other.m_dllName;
+    other.m_dllName = nullptr;
     // Mach change end
     m_dll = other.m_dll;
     other.m_dll = nullptr;
@@ -127,7 +168,7 @@ class DxcDllSupport {
   HRESULT CreateInstance(REFCLSID clsid, REFIID riid, IUnknown **pResult) {
     if (pResult == nullptr)
       return E_POINTER;
-    // Mach change start: static dxcompiler
+    // Mach change start: static dxcompiler/dxil
     // if (m_dll == nullptr)
     if (!m_initialized)
     // Mach change end
@@ -147,7 +188,7 @@ class DxcDllSupport {
                           IUnknown **pResult) {
     if (pResult == nullptr)
       return E_POINTER;
-    // Mach change start: static dxcompiler
+    // Mach change start: static dxcompiler/dxil
     // if (m_dll == nullptr)
     if (!m_initialized)
     // Mach change end
@@ -160,12 +201,18 @@ class DxcDllSupport {
 
   bool HasCreateWithMalloc() const { return m_createFn2 != nullptr; }
 
-  // Mach change start: static dxcompiler
+  // Mach change start: static dxcompiler/dxil
   // bool IsEnabled() const { return m_dll != nullptr; }
   bool IsEnabled() const { return m_initialized; }
   // Mach change start
 
   void Cleanup() {
+    // Mach change start: static dxcompiler/dxil
+    if (m_dllName == kDxCompilerLib && dxcompiler_dll_loaded) {
+      dxcompiler_dll_loaded = false;
+      MachDxcompilerInvokeDllShutdown();
+    }
+    // Mach change end
     if (m_dll != nullptr) {
       m_createFn = nullptr;
       m_createFn2 = nullptr;
@@ -178,12 +225,13 @@ class DxcDllSupport {
     }
   }
 
-  // Mach change start: static dxcompiler
+  // Mach change start: static dxcompiler/dxil
   // HMODULE Detach() {
   //   HMODULE hModule = m_dll;
   //   m_dll = nullptr;
   //   return hModule;
   // }
+  void Detach() { m_initialized = false; }
   // Mach change end
 };
 

lib/DxcSupport/dxcapi.use.cpp

@@ -29,13 +29,11 @@ namespace dxc {
 #define CMAKE_SHARED_LIBRARY_PREFIX
 #define CMAKE_SHARED_LIBRARY_SUFFIX
 // Mach change end
+
 const char *kDxCompilerLib =
     CMAKE_SHARED_LIBRARY_PREFIX "dxcompiler" CMAKE_SHARED_LIBRARY_SUFFIX;
-// Mach change start: static dxil
-// const char *kDxilLib =
-//     CMAKE_SHARED_LIBRARY_PREFIX "dxil" CMAKE_SHARED_LIBRARY_SUFFIX;
-//
-// Mach change end
+const char *kDxilLib =
+    CMAKE_SHARED_LIBRARY_PREFIX "dxil" CMAKE_SHARED_LIBRARY_SUFFIX;
 
 #ifdef _WIN32
 static void TrimEOL(char *pMsg) {

tools/clang/tools/dxclib/dxc.cpp

@@ -1339,12 +1339,10 @@ void DxcContext::GetCompilerVersionInfo(llvm::raw_string_ostream &OS) {
       m_Opts.ExternalFn.empty() ? nullptr : m_Opts.ExternalFn.data(),
       m_dxcSupport);
 
-  // Mach change start: static dxil
-  // // Print validator if exists
-  // DxcDllSupport DxilSupport;
-  // DxilSupport.InitializeForDll(kDxilLib, "DxcCreateInstance");
-  // WriteDXILVersionInfo(OS, DxilSupport);
-  // Mach change end
+  // Print validator if exists
+  DxcDllSupport DxilSupport;
+  DxilSupport.InitializeForDll(kDxilLib, "DxcCreateInstance");
+  WriteDXILVersionInfo(OS, DxilSupport);
 }
 
 #ifndef VERSION_STRING_SUFFIX

tools/clang/tools/dxcompiler/DXCompiler.cpp

@@ -65,9 +65,7 @@ static HRESULT InitMaybeFail() throw() {
   fsSetup = true;
   IFC(hlsl::SetupRegistryPassForHLSL());
   IFC(hlsl::SetupRegistryPassForPIX());
-  // Mach change start: static dxil
-  // IFC(DxilLibInitialize());
-  // Mach change end
+  IFC(DxilLibInitialize());
   if (hlsl::options::initHlslOptTable()) {
     hr = E_FAIL;
     goto Cleanup;
@@ -87,18 +85,34 @@ static HRESULT InitMaybeFail() throw() {
   return hr;
 }
 #if defined(LLVM_ON_UNIX)
-HRESULT __attribute__((constructor)) DllMain() { return InitMaybeFail(); }
+// Mach change start: static dxcompiler/dxil
+// HRESULT __attribute__((constructor)) DllMain() { return InitMaybeFail(); }
+HRESULT MachDxcompiler_DllMain() { return InitMaybeFail(); }
+// Mach change end
 
-void __attribute__((destructor)) DllShutdown() {
+// Mach change start: static dxcompiler/dxil
+// void __attribute__((destructor)) DllShutdown() {
+void MachDxcompiler_DllShutdown() {
+// Mach change end
   DxcSetThreadMallocToDefault();
   ::hlsl::options::cleanupHlslOptTable();
   ::llvm::sys::fs::CleanupPerThreadFileSystem();
   ::llvm::llvm_shutdown();
   DxcClearThreadMalloc();
   DxcCleanupThreadMalloc();
 }
+// Mach change start: static dxcompiler/dxil
+extern "C" {
+BOOL MachDxcompilerInvokeDllMain() {
+  return SUCCEEDED(MachDxcompiler_DllMain());
+}
+void MachDxcompilerInvokeDllShutdown() {
+  MachDxcompiler_DllShutdown();
+}
+}
+// Mach change end
 #else  // LLVM_ON_UNIX
-BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD Reason, LPVOID reserved) {
+BOOL WINAPI MachDxcompiler_DllMain(HINSTANCE hinstDLL, DWORD Reason, LPVOID reserved) {
   BOOL result = TRUE;
   if (Reason == DLL_PROCESS_ATTACH) {
     EventRegisterMicrosoft_Windows_DXCompiler_API();
@@ -112,14 +126,12 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD Reason, LPVOID reserved) {
     ::hlsl::options::cleanupHlslOptTable();
     ::llvm::sys::fs::CleanupPerThreadFileSystem();
     ::llvm::llvm_shutdown();
-    // Mach change start: static dxil
-    // if (reserved ==
-    //     NULL) { // FreeLibrary has been called or the DLL load failed
-    //   DxilLibCleanup(DxilLibCleanUpType::UnloadLibrary);
-    // } else { // Process termination. We should not call FreeLibrary()
-    //   DxilLibCleanup(DxilLibCleanUpType::ProcessTermination);
-    // }
-    // Mach change end
+    if (reserved ==
+        NULL) { // FreeLibrary has been called or the DLL load failed
+      DxilLibCleanup(DxilLibCleanUpType::UnloadLibrary);
+    } else { // Process termination. We should not call FreeLibrary()
+      DxilLibCleanup(DxilLibCleanUpType::ProcessTermination);
+    }
     DxcClearThreadMalloc();
     DxcCleanupThreadMalloc();
     DxcEtw_DXCompilerShutdown_Stop(S_OK);
@@ -128,4 +140,14 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD Reason, LPVOID reserved) {
 
   return result;
 }
+// Mach change start: static dxcompiler/dxil
+extern "C" {
+BOOL MachDxcompilerInvokeDllMain() {
+  return MachDxcompiler_DllMain(nullptr, DLL_PROCESS_ATTACH, nullptr);
+}
+void MachDxcompilerInvokeDllShutdown() {
+  MachDxcompiler_DllMain(nullptr, DLL_PROCESS_DETACH, nullptr);
+}
+}
+// Mach change end
 #endif // LLVM_ON_UNIX