Dawn: Support streaming ShaderModuleParseResult

This CL add StreamOut support for std::optional, and add a wrapper
UnsafeUnserializedValue to replace CacheKey::UnsafeUnkeyedValue and hold
unserializable value that will do nothing when streaming in and will set
to default constructed value when streamed out to.
These help making ShaderModuleParseResult serializable for shader
module blob cache.

Bug: 42240459, 402772740
Change-Id: Ic6199e46a1be34c4d03047af361dfab1ef807817
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/244094
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Zhaoming Jiang <zhaoming.jiang@microsoft.com>

Author: jiangzhaoming

src/dawn/native/CacheKey.h

@@ -40,34 +40,8 @@ class CacheKey : public stream::ByteVectorSink {
     using stream::ByteVectorSink::ByteVectorSink;
 
     enum class Type { ComputePipeline, RenderPipeline, Shader };
-
-    template <typename T>
-    class UnsafeUnkeyedValue {
-      public:
-        UnsafeUnkeyedValue() = default;
-        // NOLINTNEXTLINE(runtime/explicit) allow implicit construction to decrease verbosity
-        UnsafeUnkeyedValue(T&& value) : mValue(std::forward<T>(value)) {}
-
-        const T& UnsafeGetValue() const { return mValue; }
-
-        // Friend definition of StreamIn which can be found by ADL to override
-        // stream::StreamIn<T>.
-        friend constexpr void StreamIn(stream::Sink*, const UnsafeUnkeyedValue&) {}
-
-        // Enabling DAWN_SERIALIZABLE classes with UnsafeUnkeyedValue member to use default equality
-        // operator. Equality comparison always returns true for the same type UnsafeUnkeyedValues.
-        bool operator==(const UnsafeUnkeyedValue<T>& other) const { return true; }
-
-      private:
-        T mValue;
-    };
 };
 
-template <typename T>
-CacheKey::UnsafeUnkeyedValue<T> UnsafeUnkeyedValue(T&& value) {
-    return CacheKey::UnsafeUnkeyedValue<T>(std::forward<T>(value));
-}
-
 }  // namespace dawn::native
 
 #endif  // SRC_DAWN_NATIVE_CACHEKEY_H_

src/dawn/native/Serializable.h

@@ -63,6 +63,46 @@ class Serializable {
         return CreateBlob(std::move(sink));
     }
 };
+
+// UnsafeUnserializedValue holds a value of type T that does nothing when StreamIn to a sink, calls
+// default constructor when StreamOut from a source, and always compares as equal between objects of
+// the same type UnsafeUnserializedValue<T>. This is used for members in DAWN_SERIALIZABLE or
+// DAWN_MAKE_CACHE_REQUEST to prevent a member to get streamed into cache key, or enable having
+// unserializabled fields that get computed by cache missed function and returned together with
+// cached fields.
+template <typename T>
+class UnsafeUnserializedValue {
+  public:
+    UnsafeUnserializedValue() = default;
+    explicit UnsafeUnserializedValue(T&& value) : mValue(std::forward<T>(value)) {}
+    UnsafeUnserializedValue(const UnsafeUnserializedValue<T>& other)
+        : mValue(other.UnsafeGetValue()) {}
+    UnsafeUnserializedValue<T>& operator=(UnsafeUnserializedValue<T>&& other) {
+        mValue = std::move(other.UnsafeGetValue());
+        return *this;
+    }
+
+    constexpr const T& UnsafeGetValue() const { return mValue; }
+    constexpr T& UnsafeGetValue() { return mValue; }
+
+    friend constexpr void StreamIn(stream::Sink*, const UnsafeUnserializedValue<T>&) {}
+    friend MaybeError StreamOut(stream::Source*, UnsafeUnserializedValue<T>* out) {
+        // Call default constructor to initialize the value.
+        out->mValue = T();
+        return {};
+    }
+    // Enabling DAWN_SERIALIZABLE classes with UnsafeUnserializedOptional member to use default
+    // equality operator. Equality comparison always returns true for the same type.
+    bool operator==(const UnsafeUnserializedValue<T>& other) const { return true; }
+
+  protected:
+    T mValue;
+};
+
+// Template deduction guide for UnsafeUnserializedValue to enable deducting type of
+// UnsafeUnserializedValue(T{}) to UnsafeUnserializedValue<T>.
+template <typename T>
+UnsafeUnserializedValue(T&& value) -> UnsafeUnserializedValue<std::decay_t<T>>;
 }  // namespace dawn::native
 
 // Helper macro to define a struct or class along with VisitAll methods to call

src/dawn/native/d3d/D3DCompilationRequest.h

@@ -60,38 +60,38 @@ enum class Compiler { FXC, DXC };
 using InterStageShaderVariablesMask = std::bitset<tint::hlsl::writer::kMaxInterStageLocations>;
 using SubstituteOverrideConfig = std::unordered_map<tint::OverrideId, double>;
 
-#define HLSL_COMPILATION_REQUEST_MEMBERS(X)                                               \
-    X(ShaderModuleBase::ShaderModuleHash, shaderModuleHash)                               \
-    X(CacheKey::UnsafeUnkeyedValue<ShaderModuleBase::ScopedUseTintProgram>, inputProgram) \
-    X(std::string_view, entryPointName)                                                   \
-    X(SingleShaderStage, stage)                                                           \
-    X(uint32_t, shaderModel)                                                              \
-    X(uint32_t, compileFlags)                                                             \
-    X(Compiler, compiler)                                                                 \
-    X(uint64_t, compilerVersion)                                                          \
-    X(std::wstring_view, dxcShaderProfile)                                                \
-    X(std::string_view, fxcShaderProfile)                                                 \
-    X(uint32_t, firstIndexOffsetShaderRegister)                                           \
-    X(uint32_t, firstIndexOffsetRegisterSpace)                                            \
-    X(tint::hlsl::writer::Options, tintOptions)                                           \
-    X(SubstituteOverrideConfig, substituteOverrideConfig)                                 \
-    X(LimitsForCompilationRequest, limits)                                                \
-    X(CacheKey::UnsafeUnkeyedValue<LimitsForCompilationRequest>, adapterSupportedLimits)  \
-    X(uint32_t, maxSubgroupSize)                                                          \
-    X(bool, disableSymbolRenaming)                                                        \
-    X(bool, dumpShaders)                                                                  \
+#define HLSL_COMPILATION_REQUEST_MEMBERS(X)                                          \
+    X(ShaderModuleBase::ShaderModuleHash, shaderModuleHash)                          \
+    X(UnsafeUnserializedValue<ShaderModuleBase::ScopedUseTintProgram>, inputProgram) \
+    X(std::string_view, entryPointName)                                              \
+    X(SingleShaderStage, stage)                                                      \
+    X(uint32_t, shaderModel)                                                         \
+    X(uint32_t, compileFlags)                                                        \
+    X(Compiler, compiler)                                                            \
+    X(uint64_t, compilerVersion)                                                     \
+    X(std::wstring_view, dxcShaderProfile)                                           \
+    X(std::string_view, fxcShaderProfile)                                            \
+    X(uint32_t, firstIndexOffsetShaderRegister)                                      \
+    X(uint32_t, firstIndexOffsetRegisterSpace)                                       \
+    X(tint::hlsl::writer::Options, tintOptions)                                      \
+    X(SubstituteOverrideConfig, substituteOverrideConfig)                            \
+    X(LimitsForCompilationRequest, limits)                                           \
+    X(UnsafeUnserializedValue<LimitsForCompilationRequest>, adapterSupportedLimits)  \
+    X(uint32_t, maxSubgroupSize)                                                     \
+    X(bool, disableSymbolRenaming)                                                   \
+    X(bool, dumpShaders)                                                             \
     X(bool, useTintIR)
 
-#define D3D_BYTECODE_COMPILATION_REQUEST_MEMBERS(X)           \
-    X(bool, hasShaderF16Feature)                              \
-    X(uint32_t, compileFlags)                                 \
-    X(Compiler, compiler)                                     \
-    X(uint64_t, compilerVersion)                              \
-    X(std::wstring_view, dxcShaderProfile)                    \
-    X(std::string_view, fxcShaderProfile)                     \
-    X(CacheKey::UnsafeUnkeyedValue<pD3DCompile>, d3dCompile)  \
-    X(CacheKey::UnsafeUnkeyedValue<IDxcLibrary*>, dxcLibrary) \
-    X(CacheKey::UnsafeUnkeyedValue<IDxcCompiler3*>, dxcCompiler)
+#define D3D_BYTECODE_COMPILATION_REQUEST_MEMBERS(X)      \
+    X(bool, hasShaderF16Feature)                         \
+    X(uint32_t, compileFlags)                            \
+    X(Compiler, compiler)                                \
+    X(uint64_t, compilerVersion)                         \
+    X(std::wstring_view, dxcShaderProfile)               \
+    X(std::string_view, fxcShaderProfile)                \
+    X(UnsafeUnserializedValue<pD3DCompile>, d3dCompile)  \
+    X(UnsafeUnserializedValue<IDxcLibrary*>, dxcLibrary) \
+    X(UnsafeUnserializedValue<IDxcCompiler3*>, dxcCompiler)
 
 DAWN_SERIALIZABLE(struct, HlslCompilationRequest, HLSL_COMPILATION_REQUEST_MEMBERS){};
 #undef HLSL_COMPILATION_REQUEST_MEMBERS
@@ -104,7 +104,7 @@ DAWN_SERIALIZABLE(struct,
 #define D3D_COMPILATION_REQUEST_MEMBERS(X)     \
     X(HlslCompilationRequest, hlsl)            \
     X(D3DBytecodeCompilationRequest, bytecode) \
-    X(CacheKey::UnsafeUnkeyedValue<dawn::platform::Platform*>, tracePlatform)
+    X(UnsafeUnserializedValue<dawn::platform::Platform*>, tracePlatform)
 
 DAWN_MAKE_CACHE_REQUEST(D3DCompilationRequest, D3D_COMPILATION_REQUEST_MEMBERS);
 #undef D3D_COMPILATION_REQUEST_MEMBERS

src/dawn/native/d3d/ShaderUtils.cpp

@@ -228,7 +228,7 @@ ResultOrError<ComPtr<ID3DBlob>> CompileShaderFXC(const d3d::D3DBytecodeCompilati
 }
 
 MaybeError TranslateToHLSL(d3d::HlslCompilationRequest r,
-                           CacheKey::UnsafeUnkeyedValue<dawn::platform::Platform*> tracePlatform,
+                           UnsafeUnserializedValue<dawn::platform::Platform*> tracePlatform,
                            CompiledShader* compiledShader) {
     tint::ast::transform::Manager transformManager;
     tint::ast::transform::DataMap transformInputs;

src/dawn/native/d3d11/ShaderModuleD3D11.cpp

@@ -92,7 +92,7 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
     const bool useTintIR = device->IsToggleEnabled(Toggle::UseTintIR);
 
     d3d::D3DCompilationRequest req = {};
-    req.tracePlatform = UnsafeUnkeyedValue(device->GetPlatform());
+    req.tracePlatform = UnsafeUnserializedValue(device->GetPlatform());
     req.hlsl.shaderModel = 50;
     req.hlsl.disableSymbolRenaming = device->IsToggleEnabled(Toggle::DisableSymbolRenaming);
     req.hlsl.dumpShaders = device->IsToggleEnabled(Toggle::DumpShaders);
@@ -104,7 +104,8 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
 
     // D3D11 only supports FXC.
     req.bytecode.compiler = d3d::Compiler::FXC;
-    req.bytecode.d3dCompile = std::move(pD3DCompile{device->GetFunctions()->d3dCompile});
+    req.bytecode.d3dCompile =
+        UnsafeUnserializedValue(pD3DCompile{device->GetFunctions()->d3dCompile});
     req.bytecode.compilerVersion = D3D_COMPILER_VERSION;
     DAWN_ASSERT(device->GetDeviceInfo().shaderModel == 50);
     switch (stage) {
@@ -192,7 +193,7 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
     }
 
     req.hlsl.shaderModuleHash = GetHash();
-    req.hlsl.inputProgram = UseTintProgram();
+    req.hlsl.inputProgram = UnsafeUnserializedValue(UseTintProgram());
     req.hlsl.entryPointName = programmableStage.entryPoint.c_str();
     req.hlsl.stage = stage;
 
@@ -228,8 +229,8 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
 
     req.hlsl.substituteOverrideConfig = BuildSubstituteOverridesTransformConfig(programmableStage);
     req.hlsl.limits = LimitsForCompilationRequest::Create(device->GetLimits().v1);
-    req.hlsl.adapterSupportedLimits =
-        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1);
+    req.hlsl.adapterSupportedLimits = UnsafeUnserializedValue(
+        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1));
     req.hlsl.maxSubgroupSize = device->GetAdapter()->GetPhysicalDevice()->GetSubgroupMaxSize();
 
     req.hlsl.tintOptions.disable_robustness = !device->IsRobustnessEnabled();

src/dawn/native/d3d12/ShaderModuleD3D12.cpp

@@ -134,7 +134,7 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
     const bool useTintIR = device->IsToggleEnabled(Toggle::UseTintIR);
 
     d3d::D3DCompilationRequest req = {};
-    req.tracePlatform = UnsafeUnkeyedValue(device->GetPlatform());
+    req.tracePlatform = UnsafeUnserializedValue(device->GetPlatform());
     req.hlsl.shaderModel = ToBackend(device->GetPhysicalDevice())
                                ->GetAppliedShaderModelUnderToggles(device->GetTogglesState());
     req.hlsl.disableSymbolRenaming = device->IsToggleEnabled(Toggle::DisableSymbolRenaming);
@@ -154,13 +154,14 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
             ToBackend(device->GetPhysicalDevice())->GetBackend()->GetDxcVersion();
 
         req.bytecode.compiler = d3d::Compiler::DXC;
-        req.bytecode.dxcLibrary = device->GetDxcLibrary().Get();
-        req.bytecode.dxcCompiler = device->GetDxcCompiler().Get();
+        req.bytecode.dxcLibrary = UnsafeUnserializedValue(device->GetDxcLibrary().Get());
+        req.bytecode.dxcCompiler = UnsafeUnserializedValue(device->GetDxcCompiler().Get());
         req.bytecode.compilerVersion = dxcVersionInfo.DxcCompilerVersion;
         req.bytecode.dxcShaderProfile = device->GetDxcShaderProfiles()[stage];
     } else {
         req.bytecode.compiler = d3d::Compiler::FXC;
-        req.bytecode.d3dCompile = std::move(pD3DCompile{device->GetFunctions()->d3dCompile});
+        req.bytecode.d3dCompile =
+            UnsafeUnserializedValue(pD3DCompile{device->GetFunctions()->d3dCompile});
         req.bytecode.compilerVersion = D3D_COMPILER_VERSION;
         switch (stage) {
             case SingleShaderStage::Vertex:
@@ -322,7 +323,7 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
     }
 
     req.hlsl.shaderModuleHash = GetHash();
-    req.hlsl.inputProgram = UseTintProgram();
+    req.hlsl.inputProgram = UnsafeUnserializedValue(UseTintProgram());
     req.hlsl.entryPointName = programmableStage.entryPoint.c_str();
     req.hlsl.stage = stage;
     if (!useTintIR) {
@@ -380,8 +381,8 @@ ResultOrError<d3d::CompiledShader> ShaderModule::Compile(
         device->IsToggleEnabled(Toggle::EnableIntegerRangeAnalysisInRobustness);
 
     req.hlsl.limits = LimitsForCompilationRequest::Create(device->GetLimits().v1);
-    req.hlsl.adapterSupportedLimits =
-        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1);
+    req.hlsl.adapterSupportedLimits = UnsafeUnserializedValue(
+        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1));
     req.hlsl.maxSubgroupSize = device->GetAdapter()->GetPhysicalDevice()->GetSubgroupMaxSize();
 
     CacheResult<d3d::CompiledShader> compiledShader;

src/dawn/native/metal/ShaderModuleMTL.mm

@@ -55,19 +55,19 @@
 using OptionalVertexPullingTransformConfig = std::optional<tint::VertexPullingConfig>;
 using SubstituteOverrideConfig = std::unordered_map<tint::OverrideId, double>;
 
-#define MSL_COMPILATION_REQUEST_MEMBERS(X)                                                \
-    X(SingleShaderStage, stage)                                                           \
-    X(ShaderModuleBase::ShaderModuleHash, shaderModuleHash)                               \
-    X(CacheKey::UnsafeUnkeyedValue<ShaderModuleBase::ScopedUseTintProgram>, inputProgram) \
-    X(SubstituteOverrideConfig, substituteOverrideConfig)                                 \
-    X(LimitsForCompilationRequest, limits)                                                \
-    X(CacheKey::UnsafeUnkeyedValue<LimitsForCompilationRequest>, adapterSupportedLimits)  \
-    X(uint32_t, maxSubgroupSize)                                                          \
-    X(std::string, entryPointName)                                                        \
-    X(bool, usesSubgroupMatrix)                                                           \
-    X(bool, disableSymbolRenaming)                                                        \
-    X(tint::msl::writer::Options, tintOptions)                                            \
-    X(CacheKey::UnsafeUnkeyedValue<dawn::platform::Platform*>, platform)
+#define MSL_COMPILATION_REQUEST_MEMBERS(X)                                           \
+    X(SingleShaderStage, stage)                                                      \
+    X(ShaderModuleBase::ShaderModuleHash, shaderModuleHash)                          \
+    X(UnsafeUnserializedValue<ShaderModuleBase::ScopedUseTintProgram>, inputProgram) \
+    X(SubstituteOverrideConfig, substituteOverrideConfig)                            \
+    X(LimitsForCompilationRequest, limits)                                           \
+    X(UnsafeUnserializedValue<LimitsForCompilationRequest>, adapterSupportedLimits)  \
+    X(uint32_t, maxSubgroupSize)                                                     \
+    X(std::string, entryPointName)                                                   \
+    X(bool, usesSubgroupMatrix)                                                      \
+    X(bool, disableSymbolRenaming)                                                   \
+    X(tint::msl::writer::Options, tintOptions)                                       \
+    X(UnsafeUnserializedValue<dawn::platform::Platform*>, platform)
 
 DAWN_MAKE_CACHE_REQUEST(MslCompilationRequest, MSL_COMPILATION_REQUEST_MEMBERS);
 #undef MSL_COMPILATION_REQUEST_MEMBERS
@@ -275,12 +275,12 @@
     MslCompilationRequest req = {};
     req.stage = stage;
     req.shaderModuleHash = programmableStage.module->GetHash();
-    req.inputProgram = programmableStage.module->UseTintProgram();
+    req.inputProgram = UnsafeUnserializedValue(programmableStage.module->UseTintProgram());
     req.substituteOverrideConfig = BuildSubstituteOverridesTransformConfig(programmableStage);
     req.entryPointName = programmableStage.entryPoint.c_str();
     req.disableSymbolRenaming = device->IsToggleEnabled(Toggle::DisableSymbolRenaming);
     req.usesSubgroupMatrix = programmableStage.metadata->usesSubgroupMatrix;
-    req.platform = UnsafeUnkeyedValue(device->GetPlatform());
+    req.platform = UnsafeUnserializedValue(device->GetPlatform());
 
     req.tintOptions.strip_all_names = !req.disableSymbolRenaming;
     req.tintOptions.remapped_entry_point_name = device->GetIsolatedEntryPointName();
@@ -305,8 +305,8 @@
         device->IsToggleEnabled(Toggle::EnableIntegerRangeAnalysisInRobustness);
 
     req.limits = LimitsForCompilationRequest::Create(device->GetLimits().v1);
-    req.adapterSupportedLimits =
-        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1);
+    req.adapterSupportedLimits = UnsafeUnserializedValue(
+        LimitsForCompilationRequest::Create(device->GetAdapter()->GetLimits().v1));
     req.maxSubgroupSize = device->GetAdapter()->GetPhysicalDevice()->GetSubgroupMaxSize();
 
     CacheResult<MslCompilation> mslCompilation;