Create new class to handle external validation, and rename existing dll loading class.

include/dxc/Support/dxcapi.extval.h

@@ -0,0 +1,51 @@
+#include "dxc/Support/dxcapi.use.h"
+#include <string>
+
+namespace dxc {
+class DxcDllExtValidationSupport : public IDllSupport {
+  // DxCompilerSupport manages the
+  // lifetime of dxcompiler.dll, while DxilExtValSupport
+  // manages the lifetime of dxil.dll
+  dxc::DxcDllSupport DxCompilerSupport;
+  dxc::DxcDllSupport DxilExtValSupport;
+
+  DxcCreateInstanceProc m_createFn;
+  DxcCreateInstance2Proc m_createFn2;
+
+  std::string DxilDllPath;
+  HRESULT InitializeInternal(LPCSTR fnName);
+
+public:
+  std::string GetDxilDllPath() { return DxilDllPath; }
+  bool DxilDllFailedToLoad() {
+    return !DxilDllPath.empty() && !DxilExtValSupport.IsEnabled();
+  }
+
+  HRESULT CreateInstance(REFCLSID clsid, REFIID riid, IUnknown **pResult);
+  HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
+                          IUnknown **pResult);
+
+  HRESULT Initialize() { return InitializeInternal("DxcCreateInstance"); }
+  HRESULT InitializeForDll(LPCSTR dll, LPCSTR entryPoint) {
+    return InitializeInternal(dll);
+  }
+
+  bool HasCreateWithMalloc() const { return m_createFn2 != nullptr; }
+
+  bool IsEnabled() const { return DxCompilerSupport.IsEnabled(); }
+
+  bool GetCreateInstanceProcs(DxcCreateInstanceProc *pCreateFn,
+                              DxcCreateInstance2Proc *pCreateFn2) const;
+
+  void Cleanup() {
+    DxilExtValSupport.Cleanup();
+    DxCompilerSupport.Cleanup();
+  }
+
+  HMODULE Detach() {
+    // Can't Detach and return a handle for DxilSupport. Cleanup() instead.
+    DxilExtValSupport.Cleanup();
+    return DxCompilerSupport.Detach();
+  }
+};
+} // namespace dxc

include/dxc/Support/dxcapi.use.h

@@ -19,9 +19,52 @@ namespace dxc {
 extern const char *kDxCompilerLib;
 extern const char *kDxilLib;
 
+// Interface for common dll operations
+class IDllSupport {
+public:
+  IDllSupport() = default;
+
+  IDllSupport(IDllSupport &&other) = default;
+
+  virtual ~IDllSupport() = default;
+
+  virtual HRESULT Initialize() = 0;
+
+  virtual HRESULT InitializeForDll(LPCSTR dll, LPCSTR entryPoint) = 0;
+
+  template <typename TInterface>
+  HRESULT CreateInstance(REFCLSID clsid, TInterface **pResult) {
+    return CreateInstance(clsid, __uuidof(TInterface), (IUnknown **)pResult);
+  }
+
+  virtual HRESULT CreateInstance(REFCLSID clsid, REFIID riid,
+                                 IUnknown **pResult) = 0;
+
+  template <typename TInterface>
+  HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid,
+                          TInterface **pResult) {
+    return CreateInstance2(pMalloc, clsid, __uuidof(TInterface),
+                           (IUnknown **)pResult);
+  }
+
+  virtual HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
+                                  IUnknown **pResult) = 0;
+
+  virtual bool HasCreateWithMalloc() const = 0;
+
+  virtual bool IsEnabled() const = 0;
+
+  virtual bool
+  GetCreateInstanceProcs(DxcCreateInstanceProc *pCreateFn,
+                         DxcCreateInstance2Proc *pCreateFn2) const = 0;
+
+  virtual void Cleanup() = 0;
+  virtual HMODULE Detach() = 0;
+};
+
 // Helper class to dynamically load the dxcompiler or a compatible libraries.
-class DxcDllSupport {
-protected:
+class DxcDllSupport : public IDllSupport {
+
   HMODULE m_dll;
   DxcCreateInstanceProc m_createFn;
   DxcCreateInstance2Proc m_createFn2;
@@ -95,11 +138,9 @@ class DxcDllSupport {
     return InitializeInternal(dll, entryPoint);
   }
 
-  template <typename TInterface>
-  HRESULT CreateInstance(REFCLSID clsid, TInterface **pResult) {
-    return CreateInstance(clsid, __uuidof(TInterface), (IUnknown **)pResult);
-  }
-
+  // Also bring visibility into the interface definition of this function
+  // which takes 2 args
+  using IDllSupport::CreateInstance;
   HRESULT CreateInstance(REFCLSID clsid, REFIID riid, IUnknown **pResult) {
     if (pResult == nullptr)
       return E_POINTER;
@@ -109,13 +150,9 @@ class DxcDllSupport {
     return hr;
   }
 
-  template <typename TInterface>
-  HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid,
-                          TInterface **pResult) {
-    return CreateInstance2(pMalloc, clsid, __uuidof(TInterface),
-                           (IUnknown **)pResult);
-  }
-
+  // Also bring visibility into the interface definition of this function
+  // which takes 3 args
+  using IDllSupport::CreateInstance2;
   HRESULT CreateInstance2(IMalloc *pMalloc, REFCLSID clsid, REFIID riid,
                           IUnknown **pResult) {
     if (pResult == nullptr)
@@ -132,6 +169,15 @@ class DxcDllSupport {
 
   bool IsEnabled() const { return m_dll != nullptr; }
 
+  bool GetCreateInstanceProcs(DxcCreateInstanceProc *pCreateFn,
+                              DxcCreateInstance2Proc *pCreateFn2) const {
+    if (pCreateFn == nullptr || pCreateFn2 == nullptr || m_createFn == nullptr)
+      return false;
+    *pCreateFn = m_createFn;
+    *pCreateFn2 = m_createFn2;
+    return true;
+  }
+
   void Cleanup() {
     if (m_dll != nullptr) {
       m_createFn = nullptr;

lib/DxcSupport/CMakeLists.txt

@@ -10,6 +10,7 @@ add_llvm_library(LLVMDxcSupport
   WinAdapter.cpp
   WinIncludes.cpp
   WinFunctions.cpp
+  dxcapi.extval.cpp
   )
 
 #generate header with platform-specific library name

lib/DxcSupport/dxcapi.extval.cpp

@@ -0,0 +1,61 @@
+#include "dxc/Support/WinIncludes.h"
+#include <dxc/Support/Global.h> // for hresult handling with DXC_FAILED
+#include <filesystem>           // C++17 and later
+// WinIncludes must come before dxcapi.extval.h
+#include "dxc/Support/dxcapi.extval.h"
+
+namespace dxc {
+
+HRESULT DxcDllExtValidationSupport::CreateInstance(REFCLSID clsid, REFIID riid,
+                                                   IUnknown **pResult) {
+  if (DxilExtValSupport.IsEnabled() && clsid == CLSID_DxcValidator)
+    return DxilExtValSupport.CreateInstance(clsid, riid, pResult);
+
+  return DxCompilerSupport.CreateInstance(clsid, riid, pResult);
+}
+
+HRESULT DxcDllExtValidationSupport::CreateInstance2(IMalloc *pMalloc,
+                                                    REFCLSID clsid, REFIID riid,
+                                                    IUnknown **pResult) {
+  if (DxilExtValSupport.IsEnabled() && clsid == CLSID_DxcValidator)
+    return DxilExtValSupport.CreateInstance2(pMalloc, clsid, riid, pResult);
+
+  return DxCompilerSupport.CreateInstance2(pMalloc, clsid, riid, pResult);
+}
+
+HRESULT DxcDllExtValidationSupport::InitializeInternal(LPCSTR fnName) {
+  // Load dxcompiler.dll
+  HRESULT Result = DxCompilerSupport.InitializeForDll(kDxCompilerLib, fnName);
+  // if dxcompiler.dll fails to load, return the failed HRESULT
+  if (DXC_FAILED(Result)) {
+    return Result;
+  }
+
+  // now handle external dxil.dll
+  const char *EnvVarVal = std::getenv("DXC_DXIL_DLL_PATH");
+  if (!EnvVarVal || std::string(EnvVarVal).empty()) {
+    return S_OK;
+  }
+
+  DxilDllPath = std::string(EnvVarVal);
+  std::filesystem::path DllPath(DxilDllPath);
+
+  // Check if path is absolute and exists
+  if (!DllPath.is_absolute() || !std::filesystem::exists(DllPath)) {
+    return E_INVALIDARG;
+  }
+
+  return DxilExtValSupport.InitializeForDll(DxilDllPath.c_str(), fnName);
+}
+
+bool DxcDllExtValidationSupport::GetCreateInstanceProcs(
+    DxcCreateInstanceProc *pCreateFn,
+    DxcCreateInstance2Proc *pCreateFn2) const {
+  if (pCreateFn == nullptr || pCreateFn2 == nullptr || m_createFn == nullptr)
+    return false;
+  *pCreateFn = m_createFn;
+  *pCreateFn2 = m_createFn2;
+  return true;
+}
+
+} // namespace dxc

tools/clang/unittests/HLSL/ValidationTest.cpp

@@ -20,10 +20,12 @@
 #include "dxc/DxilContainer/DxilContainerAssembler.h"
 #include "dxc/DxilContainer/DxilPipelineStateValidation.h"
 #include "dxc/DxilHash/DxilHash.h"
+#include "dxc/Support/Unicode.h" // for wstring conversions like WideToUtf8String
 #include "dxc/Support/WinIncludes.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Regex.h"
+#include <wchar.h> // for _wgetenv
 
 #ifdef _WIN32
 #include <atlbase.h>
@@ -32,6 +34,7 @@
 #include "dxc/Support/Global.h"
 
 #include "dxc/DXIL/DxilShaderModel.h"
+#include "dxc/Support/dxcapi.extval.h"
 #include "dxc/Test/DxcTestUtils.h"
 #include "dxc/Test/HlslTestUtils.h"
 
@@ -323,6 +326,7 @@ class ValidationTest : public ::testing::Test {
   TEST_METHOD(PSVContentValidationCS)
   TEST_METHOD(PSVContentValidationMS)
   TEST_METHOD(PSVContentValidationAS)
+  TEST_METHOD(UnitTestExtValidationSupport)
   TEST_METHOD(WrongPSVSize)
   TEST_METHOD(WrongPSVSizeOnZeros)
   TEST_METHOD(WrongPSVVersion)
@@ -4207,6 +4211,97 @@ TEST_F(ValidationTest, ValidateWithHash) {
   VERIFY_ARE_EQUAL(memcmp(Result, pHeader->Hash.Digest, sizeof(Result)), 0);
 }
 
+#ifdef _WIN32
+std::wstring GetEnvVarW(const std::wstring &VarName) {
+  if (const wchar_t *Result = _wgetenv(VarName.c_str()))
+    return std::wstring(Result);
+  return std::wstring();
+}
+
+void SetEnvVarW(const std::wstring &VarName, const std::wstring &VarValue) {
+  _wputenv_s(VarName.c_str(), VarValue.c_str());
+}
+
+// For now, 3 things are tested:
+// 1. The environment variable is not set. GetDxilDllPath() is empty and
+// DxilDllFailedToLoad() returns false
+// 2. Given a bogus path in the environment variable, and an initialized
+// DxcDllExtValidationSupport object, GetDxilDllPath()
+// retrieves the bogus path despite DxcDllExtValidationSupport failing to load
+// it as a dll, and DxilDllFailedToLoad() returns true.
+// 3. CLSID_DxcCompiler, CLSID_DxcLinker, CLSID_DxcValidator
+// may be created through DxcDllExtValidationSupport.
+// This is all to simply test that the new class, DxcDllExtValidationSupport,
+// works as intended.
+
+TEST_F(ValidationTest, UnitTestExtValidationSupport) {
+  dxc::DxcDllExtValidationSupport ExtSupportEmpty;
+  dxc::DxcDllExtValidationSupport ExtSupportBogus;
+
+  // capture any existing value in the environment variable,
+  // so that it can be restored after the test
+  std::wstring OldEnvVal = GetEnvVarW(L"DXC_DXIL_DLL_PATH");
+
+  // 1. with no env var set, test GetDxilDllPath() and DxilDllFailedToLoad()
+
+  // make sure the variable is cleared, in case other tests may have set it
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", L"");
+
+  // empty initialization should succeed
+  VERIFY_SUCCEEDED(ExtSupportEmpty.Initialize());
+
+  VERIFY_IS_FALSE(ExtSupportEmpty.DxilDllFailedToLoad());
+  std::string EmptyPath = ExtSupportBogus.GetDxilDllPath();
+  VERIFY_ARE_EQUAL_STR(EmptyPath.c_str(), "");
+
+  // 2. Test with a bogus path in the environment variable
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", L"bogus");
+
+  if (!ExtSupportBogus.IsEnabled()) {
+    VERIFY_FAILED(ExtSupportBogus.Initialize());
+  }
+
+  // validate that m_dllExtSupport2 was able to capture the environment
+  // variable's value, and that loading the bogus path was unsuccessful
+  std::string BogusPath = ExtSupportBogus.GetDxilDllPath();
+  VERIFY_ARE_EQUAL_STR(BogusPath.c_str(), "bogus");
+  VERIFY_IS_TRUE(ExtSupportBogus.DxilDllFailedToLoad());
+
+  // 3. Test production of class IDs CLSID_DxcCompiler, CLSID_DxcLinker,
+  // and CLSID_DxcValidator through DxcDllExtValidationSupport.
+  CComPtr<IDxcCompiler> Compiler;
+  CComPtr<IDxcLinker> Linker;
+  CComPtr<IDxcValidator> Validator;
+
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcCompiler, __uuidof(IDxcCompiler), (IUnknown **)&Compiler));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcLinker, __uuidof(IDxcLinker), (IUnknown **)&Linker));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance(
+      CLSID_DxcValidator, __uuidof(IDxcValidator), (IUnknown **)&Validator));
+
+  Linker.Release();
+  Validator.Release();
+  Compiler.Release();
+
+  CComPtr<IMalloc> Malloc;
+  CComPtr<IDxcCompiler2> Compiler2;
+  VERIFY_SUCCEEDED(DxcCoGetMalloc(1, &Malloc));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(Malloc, CLSID_DxcCompiler,
+                                                   __uuidof(IDxcCompiler),
+                                                   (IUnknown **)&Compiler2));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(
+      Malloc, CLSID_DxcLinker, __uuidof(IDxcLinker), (IUnknown **)&Linker));
+  VERIFY_SUCCEEDED(ExtSupportBogus.CreateInstance2(Malloc, CLSID_DxcValidator,
+                                                   __uuidof(IDxcValidator),
+                                                   (IUnknown **)&Validator));
+
+  // reset the environment variable to its previous value,
+  // or the empty string if there was no previous value
+  SetEnvVarW(L"DXC_DXIL_DLL_PATH", OldEnvVal);
+}
+#endif
+
 TEST_F(ValidationTest, ValidatePreviewBypassHash) {
   if (m_ver.SkipDxilVersion(1, ShaderModel::kHighestMinor))
     return;