Create new class to handle external validation, and rename existing dll loading class.

include/dxc/Support/dxcapi.extval.h

@@ -0,0 +1,38 @@
+#include "dxc/Support/dxcapi.use.h"
+#include <string>
+
+class DxcDllExtValidationSupport : public dxc::DxcDllSupport {
+  // DxcDllExtValidationSupport manages the
+  // lifetime of dxcompiler.dll, while the member, m_DxilSupport,
+  // manages the lifetime of dxil.dll
+protected:
+  dxc::DxcDllSupport DxilSupport;
+
+  std::string DxilDllPath;
+
+  // override DxcDllSupport's implementation of InitializeInternal,
+  // adding the environment variable value check for a path to a dxil.dll
+  HRESULT InitializeInternal(LPCSTR dllName, LPCSTR fnName) override;
+
+public:
+  std::string GetDxilDllPath() { return DxilDllPath; }
+  bool DxilDllFailedToLoad() {
+    return !DxilDllPath.empty() && !DxilSupport.IsEnabled();
+  }
+
+  void Cleanup() override {
+    DxilSupport.Cleanup();
+    DxcDllSupport::Cleanup();
+  }
+
+  HMODULE Detach() override {
+    // Can't Detach and return a handle for DxilSupport. Cleanup() instead.
+    DxilSupport.Cleanup();
+    return DxcDllSupport::Detach();
+  }
+
+  HRESULT CreateInstance(REFCLSID clsid, REFIID riid,
+                         IUnknown **pResult) override;
+  HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
+                          IUnknown **pResult) override;
+};

include/dxc/Support/dxcapi.use.h

@@ -26,7 +26,7 @@ class DxcDllSupport {
   DxcCreateInstanceProc m_createFn;
   DxcCreateInstance2Proc m_createFn2;
 
-  HRESULT InitializeInternal(LPCSTR dllName, LPCSTR fnName) {
+  virtual HRESULT InitializeInternal(LPCSTR dllName, LPCSTR fnName) {
     if (m_dll != nullptr)
       return S_OK;
 
@@ -85,7 +85,7 @@ class DxcDllSupport {
     other.m_createFn2 = nullptr;
   }
 
-  ~DxcDllSupport() { Cleanup(); }
+  virtual ~DxcDllSupport() { Cleanup(); }
 
   HRESULT Initialize() {
     return InitializeInternal(kDxCompilerLib, "DxcCreateInstance");
@@ -100,7 +100,8 @@ class DxcDllSupport {
     return CreateInstance(clsid, __uuidof(TInterface), (IUnknown **)pResult);
   }
 
-  HRESULT CreateInstance(REFCLSID clsid, REFIID riid, IUnknown **pResult) {
+  virtual HRESULT CreateInstance(REFCLSID clsid, REFIID riid,
+                                 IUnknown **pResult) {
     if (pResult == nullptr)
       return E_POINTER;
     if (m_dll == nullptr)
@@ -116,8 +117,8 @@ class DxcDllSupport {
                            (IUnknown **)pResult);
   }
 
-  HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
-                          IUnknown **pResult) {
+  virtual HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
+                                  IUnknown **pResult) {
     if (pResult == nullptr)
       return E_POINTER;
     if (m_dll == nullptr)
@@ -132,7 +133,16 @@ class DxcDllSupport {
 
   bool IsEnabled() const { return m_dll != nullptr; }
 
-  void Cleanup() {
+  bool GetCreateInstanceProcs(DxcCreateInstanceProc *pCreateFn,
+                              DxcCreateInstance2Proc *pCreateFn2) const {
+    if (pCreateFn == nullptr || pCreateFn2 == nullptr || m_createFn == nullptr)
+      return false;
+    *pCreateFn = m_createFn;
+    *pCreateFn2 = m_createFn2;
+    return true;
+  }
+
+  virtual void Cleanup() {
     if (m_dll != nullptr) {
       m_createFn = nullptr;
       m_createFn2 = nullptr;
@@ -145,7 +155,7 @@ class DxcDllSupport {
     }
   }
 
-  HMODULE Detach() {
+  virtual HMODULE Detach() {
     HMODULE hModule = m_dll;
     m_dll = nullptr;
     return hModule;

lib/DxcSupport/CMakeLists.txt

@@ -10,6 +10,7 @@ add_llvm_library(LLVMDxcSupport
   WinAdapter.cpp
   WinIncludes.cpp
   WinFunctions.cpp
+  dxcapi.extval.cpp
   )
 
 #generate header with platform-specific library name

lib/DxcSupport/dxcapi.extval.cpp

@@ -0,0 +1,54 @@
+#include "dxc/Support/WinIncludes.h"
+#include <dxc/Support/Global.h> // for hresult handling with DXC_FAILED
+#include <filesystem>           // C++17 and later
+// WinIncludes must come before dxcapi.extval.h
+#include "dxc/Support/dxcapi.extval.h"
+
+HRESULT DxcDllExtValidationSupport::InitializeInternal(LPCSTR dllName,
+                                                       LPCSTR fnName) {
+  // Load dxcompiler.dll
+  HRESULT Result = DxcDllSupport::InitializeInternal(dllName, fnName);
+  // if dxcompiler.dll fails to load, return the failed HRESULT
+  if (DXC_FAILED(Result)) {
+    return Result;
+  }
+
+  // now handle external dxil.dll
+  const char *EnvVarVal = std::getenv("DXC_DXIL_DLL_PATH");
+  if (!EnvVarVal || std::string(EnvVarVal).empty()) {
+    return S_OK;
+  }
+
+  std::string DllPathStr(EnvVarVal);
+  DxilDllPath = DllPathStr;
+  std::filesystem::path DllPath(DllPathStr);
+
+  // Check if path is absolute and exists
+  if (!DllPath.is_absolute() || !std::filesystem::exists(DllPath)) {
+    return S_OK;
+  }
+  // code that calls this function is responsible for checking
+  // to see if dxil.dll is successfully loaded.
+  // the CheckDxilDLLLoaded function can determine whether there were any
+  // problems loading dxil.dll or not
+  DxilSupport.InitializeForDll(DllPathStr.data(), fnName);
+
+  return S_OK;
+}
+
+HRESULT DxcDllExtValidationSupport::CreateInstance(REFCLSID clsid, REFIID riid,
+                                                   IUnknown **pResult) {
+  if (DxilSupport.IsEnabled() && clsid == CLSID_DxcValidator)
+    return DxilSupport.CreateInstance(clsid, riid, pResult);
+
+  return DxcDllSupport::CreateInstance(clsid, riid, pResult);
+}
+
+HRESULT DxcDllExtValidationSupport::CreateInstance2(IMalloc *pMalloc,
+                                                    REFCLSID clsid, REFIID riid,
+                                                    IUnknown **pResult) {
+  if (DxilSupport.IsEnabled() && clsid == CLSID_DxcValidator)
+    return DxilSupport.CreateInstance2(pMalloc, clsid, riid, pResult);
+
+  return DxcDllSupport::CreateInstance2(pMalloc, clsid, riid, pResult);
+}

tools/clang/unittests/HLSL/ValidationTest.cpp

@@ -20,10 +20,13 @@
 #include "dxc/DxilContainer/DxilContainerAssembler.h"
 #include "dxc/DxilContainer/DxilPipelineStateValidation.h"
 #include "dxc/DxilHash/DxilHash.h"
+#include "dxc/Support/Unicode.h" // for wstring conversions like WideToUtf8String
 #include "dxc/Support/WinIncludes.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Regex.h"
+#include <cstdlib> // for setenv(), getenv(), unsetenv()
+#include <wchar.h> // for _wgetenv
 
 #ifdef _WIN32
 #include <atlbase.h>
@@ -32,6 +35,7 @@
 #include "dxc/Support/Global.h"
 
 #include "dxc/DXIL/DxilShaderModel.h"
+#include "dxc/Support/dxcapi.extval.h"
 #include "dxc/Test/DxcTestUtils.h"
 #include "dxc/Test/HlslTestUtils.h"
 
@@ -323,6 +327,7 @@ class ValidationTest : public ::testing::Test {
   TEST_METHOD(PSVContentValidationCS)
   TEST_METHOD(PSVContentValidationMS)
   TEST_METHOD(PSVContentValidationAS)
+  TEST_METHOD(UnitTestExtValidationSupport)
   TEST_METHOD(WrongPSVSize)
   TEST_METHOD(WrongPSVSizeOnZeros)
   TEST_METHOD(WrongPSVVersion)
@@ -4207,6 +4212,110 @@ TEST_F(ValidationTest, ValidateWithHash) {
   VERIFY_ARE_EQUAL(memcmp(Result, pHeader->Hash.Digest, sizeof(Result)), 0);
 }
 
+std::wstring GetEnvVarW(const std::wstring &VarName) {
+#ifdef _WIN32
+  if (const wchar_t *Result = _wgetenv(VarName.c_str()))
+    return std::wstring(Result);
+#else
+  std::string VarNameUtf8;
+  Unicode::WideToUTF8String(VarName.c_str(), &VarNameUtf8);
+  if (const char *Result = std::getenv(VarNameUtf8.c_str())) {
+    std::wstring ResultWide;
+    Unicode::UTF8ToWideString(Result, &ResultWide);
+    return std::wstring(ResultWide);
+  }
+#endif
+  return std::wstring();
+}
+
+void SetEnvVarW(const std::wstring &VarName, const std::wstring &VarValue) {
+#ifdef _WIN32
+  _wputenv_s(VarName.c_str(), VarValue.c_str());
+#else
+  std::string VarNameUtf8;
+  std::string VarValueUtf8;
+  Unicode::WideToUTF8String(VarName.c_str(), &VarNameUtf8);
+  Unicode::WideToUTF8String(VarValue.c_str(), &VarValueUtf8);
+  setenv(VarNameUtf8.c_str(), VarValueUtf8.c_str(), 1);
+#endif
+}
+
+// For now, 3 things are tested:
+// 1. The environment variable is not set. GetDxilDllPath() is empty and
+// DxilDllFailedToLoad() returns false
+// 2. Given a bogus path in the environment variable, GetDxilDllPath()
+// retrieves the path but fails to load it as a dll, and returns true
+// for DxilDllFailedToLoad()
+// 3. CLSID_DxcCompiler, CLSID_DxcLinker, CLSID_DxcValidator
+// may be created through DxcDllExtValidationSupport.
+// This is all to simply test that the new class, DxcDllExtValidationSupport,
+// works as intended.
+
+TEST_F(ValidationTest, UnitTestExtValidationSupport) {
+  DxcDllExtValidationSupport ExtSupportEmpty;
+  DxcDllExtValidationSupport ExtSupportBogus;
+
+  // capture any existing value in the environment variable,
+  // so that it can be restored after the test
+  std::wstring OldEnvVal = GetEnvVarW(L"DXC_DXIL_DLL_PATH");
+
+  // 1. with no env var set, test GetDxilDllPath() and DxilDllFailedToLoad()
+
+  // make sure the variable is cleared, in case other tests may have set it
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", L"");
+
+  // empty initialization should succeed
+  VERIFY_SUCCEEDED(ExtSupportEmpty.Initialize());
+
+  VERIFY_IS_FALSE(ExtSupportEmpty.DxilDllFailedToLoad());
+  std::string EmptyPath = ExtSupportBogus.GetDxilDllPath();
+  VERIFY_ARE_EQUAL_STR(EmptyPath.c_str(), "");
+
+  // 2. Test with a bogus path in the environment variable
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", L"bogus");
+
+  if (!ExtSupportBogus.IsEnabled()) {
+    VERIFY_SUCCEEDED(ExtSupportBogus.Initialize());
+  }
+
+  // validate that m_dllExtSupport2 was able to capture the environment
+  // variable's value, and that loading the bogus path was unsuccessful
+  std::string BogusPath = ExtSupportBogus.GetDxilDllPath();
+  VERIFY_ARE_EQUAL_STR(BogusPath.c_str(), "bogus");
+  VERIFY_IS_TRUE(ExtSupportBogus.DxilDllFailedToLoad());
+
+  // 3. Test production of class IDs CLSID_DxcCompiler, CLSID_DxcLinker,
+  // and CLSID_DxcValidator through DxcDllExtValidationSupport.
+  CComPtr<IDxcCompiler> Compiler;
+  CComPtr<IDxcLinker> Linker;
+  CComPtr<IDxcValidator> Validator;
+
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcCompiler, __uuidof(IDxcCompiler), (IUnknown **)&Compiler));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcLinker, __uuidof(IDxcLinker), (IUnknown **)&Linker));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcValidator, __uuidof(IDxcValidator), (IUnknown **)&Validator));
+
+  CComPtr<IMalloc> Malloc;
+  CComPtr<IDxcCompiler2> Compiler2;
+  Linker.Release();
+  Validator.Release();
+  VERIFY_SUCCEEDED(DxcCoGetMalloc(1, &Malloc));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(Malloc, CLSID_DxcCompiler,
+                                                   __uuidof(IDxcCompiler),
+                                                   (IUnknown **)&Compiler2));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(
+      Malloc, CLSID_DxcLinker, __uuidof(IDxcLinker), (IUnknown **)&Linker));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(Malloc, CLSID_DxcValidator,
+                                                   __uuidof(IDxcValidator),
+                                                   (IUnknown **)&Validator));
+
+  // reset the environment variable to its previous value,
+  // or the empty string if there was no previous value
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", OldEnvVal);
+}
+
 TEST_F(ValidationTest, ValidatePreviewBypassHash) {
   if (m_ver.SkipDxilVersion(1, ShaderModel::kHighestMinor))
     return;