Skip to content

Commit 71924d1

Browse files
anishmgoyalanishmgoyal
committed
Make VulkanProgram handles thread safe
With the introduction of cache prewarming, handles for vulkan programs will be managed on a separate thread in the backend along with the main backend thread. This requires an atomic refcount to ensure accuracy and prevent race conditions.
1 parent 35f501b commit 71924d1

File tree

7 files changed

+285
-267
lines changed

7 files changed

+285
-267
lines changed

‎filament/backend/src/vulkan/VulkanAsyncHandles.cpp‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616

1717
#include "VulkanAsyncHandles.h"
1818

19+
#include "VulkanConstants.h"
20+
21+
#include "vulkan/utils/Spirv.h"
22+
1923
#include <backend/DriverEnums.h>
2024

2125
#include <utils/debug.h>
@@ -29,6 +33,154 @@ using namespace bluevk;
2933

3034
namespace filament::backend {
3135

36+
namespace {
37+
38+
inline VkShaderStageFlags getVkStage(backend::ShaderStage stage) {
39+
switch(stage) {
40+
case backend::ShaderStage::VERTEX:
41+
return VK_SHADER_STAGE_VERTEX_BIT;
42+
case backend::ShaderStage::FRAGMENT:
43+
return VK_SHADER_STAGE_FRAGMENT_BIT;
44+
case backend::ShaderStage::COMPUTE:
45+
PANIC_POSTCONDITION("Unsupported stage");
46+
}
47+
}
48+
49+
} // namespace
50+
51+
PushConstantDescription::PushConstantDescription(backend::Program const& program) {
52+
mRangeCount = 0;
53+
uint32_t offset = 0;
54+
55+
// The range is laid out so that the vertex constants are defined as the first set of bytes,
56+
// followed by fragment and compute. This means we need to keep track of the offset for each
57+
// stage. We do the bookeeping in mDescriptions.
58+
for (auto stage: { ShaderStage::VERTEX, ShaderStage::FRAGMENT, ShaderStage::COMPUTE }) {
59+
auto const& constants = program.getPushConstants(stage);
60+
if (constants.empty()) {
61+
continue;
62+
}
63+
64+
auto& description = mDescriptions[(uint8_t) stage];
65+
// We store the type of the constant for type-checking when writing.
66+
description.types.reserve(constants.size());
67+
std::for_each(constants.cbegin(), constants.cend(),
68+
[&description](Program::PushConstant t) { description.types.push_back(t.type); });
69+
70+
uint32_t const constantsSize = (uint32_t) constants.size() * ENTRY_SIZE;
71+
mRanges[mRangeCount++] = {
72+
.stageFlags = getVkStage(stage),
73+
.offset = offset,
74+
.size = constantsSize,
75+
};
76+
description.offset = offset;
77+
offset += constantsSize;
78+
}
79+
}
80+
81+
void PushConstantDescription::write(VkCommandBuffer cmdbuf, VkPipelineLayout layout,
82+
backend::ShaderStage stage, uint8_t index, backend::PushConstantVariant const& value) {
83+
84+
uint32_t binaryValue = 0;
85+
auto const& description = mDescriptions[(uint8_t) stage];
86+
UTILS_UNUSED_IN_RELEASE auto const& types = description.types;
87+
uint32_t const offset = description.offset;
88+
89+
if (std::holds_alternative<bool>(value)) {
90+
assert_invariant(types[index] == ConstantType::BOOL);
91+
bool const bval = std::get<bool>(value);
92+
binaryValue = static_cast<uint32_t const>(bval ? VK_TRUE : VK_FALSE);
93+
} else if (std::holds_alternative<float>(value)) {
94+
assert_invariant(types[index] == ConstantType::FLOAT);
95+
float const fval = std::get<float>(value);
96+
binaryValue = *reinterpret_cast<uint32_t const*>(&fval);
97+
} else {
98+
assert_invariant(types[index] == ConstantType::INT);
99+
int const ival = std::get<int>(value);
100+
binaryValue = *reinterpret_cast<uint32_t const*>(&ival);
101+
}
102+
103+
vkCmdPushConstants(cmdbuf, layout, getVkStage(stage), offset + index * ENTRY_SIZE, ENTRY_SIZE,
104+
&binaryValue);
105+
}
106+
107+
VulkanProgram::VulkanProgram(VkDevice device, Program const& builder) noexcept
108+
: HwProgram(builder.getName()),
109+
mInfo(new(std::nothrow) PipelineInfo(builder)),
110+
mDevice(device) {
111+
112+
Program::ShaderSource const& blobs = builder.getShadersSource();
113+
auto& modules = mInfo->shaders;
114+
auto const& specializationConstants = builder.getSpecializationConstants();
115+
std::vector<uint32_t> shader;
116+
117+
static_assert(static_cast<ShaderStage>(0) == ShaderStage::VERTEX &&
118+
static_cast<ShaderStage>(1) == ShaderStage::FRAGMENT &&
119+
MAX_SHADER_MODULES == 2);
120+
121+
for (size_t i = 0; i < MAX_SHADER_MODULES; i++) {
122+
Program::ShaderBlob const& blob = blobs[i];
123+
124+
uint32_t* data = (uint32_t*) blob.data();
125+
size_t dataSize = blob.size();
126+
127+
if (!specializationConstants.empty()) {
128+
fvkutils::workaroundSpecConstant(blob, specializationConstants, shader);
129+
data = (uint32_t*) shader.data();
130+
dataSize = shader.size() * 4;
131+
}
132+
133+
VkShaderModule& module = modules[i];
134+
VkShaderModuleCreateInfo moduleInfo = {
135+
.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
136+
.codeSize = dataSize,
137+
.pCode = data,
138+
};
139+
VkResult result = vkCreateShaderModule(mDevice, &moduleInfo, VKALLOC, &module);
140+
FILAMENT_CHECK_POSTCONDITION(result == VK_SUCCESS)
141+
<< "Unable to create shader module."
142+
<< " error=" << static_cast<int32_t>(result);
143+
144+
#if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS)
145+
utils::CString name{ builder.getName().c_str(), builder.getName().size() };
146+
switch (static_cast<ShaderStage>(i)) {
147+
case ShaderStage::VERTEX:
148+
name += "_vs";
149+
break;
150+
case ShaderStage::FRAGMENT:
151+
name += "_fs";
152+
break;
153+
default:
154+
PANIC_POSTCONDITION("Unexpected stage");
155+
break;
156+
}
157+
VulkanDriver::DebugUtils::setName(VK_OBJECT_TYPE_SHADER_MODULE,
158+
reinterpret_cast<uint64_t>(module), name.c_str());
159+
#endif
160+
}
161+
162+
#if FVK_ENABLED(FVK_DEBUG_SHADER_MODULE)
163+
FVK_LOGD << "Created VulkanProgram " << builder << ", shaders = (" << modules[0]
164+
<< ", " << modules[1] << ")";
165+
#endif
166+
}
167+
168+
VulkanProgram::~VulkanProgram() {
169+
for (auto shader: mInfo->shaders) {
170+
vkDestroyShaderModule(mDevice, shader, VKALLOC);
171+
}
172+
delete mInfo;
173+
}
174+
175+
void VulkanProgram::flushPushConstants(VkPipelineLayout layout) {
176+
// At this point, we really ought to have a VkPipelineLayout.
177+
assert_invariant(layout != VK_NULL_HANDLE);
178+
for (const auto& c : mQueuedPushConstants) {
179+
mInfo->pushConstantDescription.write(c.cmdbuf, layout, c.stage, c.index, c.value);
180+
}
181+
mQueuedPushConstants.clear();
182+
}
183+
32184
std::shared_ptr<VulkanCmdFence> VulkanCmdFence::completed() noexcept {
33185
auto cmdFence = std::make_shared<VulkanCmdFence>(VK_NULL_HANDLE);
34186
cmdFence->mStatus = VK_SUCCESS;

‎filament/backend/src/vulkan/VulkanAsyncHandles.h‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
#include "backend/Platform.h"
2525

2626
#include "vulkan/memory/Resource.h"
27+
#include "vulkan/utils/StaticVector.h"
28+
29+
#include <backend/Program.h>
2730

2831
#include <chrono>
2932
#include <cstdint>
@@ -35,6 +38,118 @@
3538

3639
namespace filament::backend {
3740

41+
using PushConstantNameArray = utils::FixedCapacityVector<char const*>;
42+
using PushConstantNameByStage = std::array<PushConstantNameArray, Program::SHADER_TYPE_COUNT>;
43+
44+
struct PushConstantDescription {
45+
explicit PushConstantDescription(backend::Program const& program);
46+
47+
VkPushConstantRange const* getVkRanges() const noexcept { return mRanges; }
48+
uint32_t getVkRangeCount() const noexcept { return mRangeCount; }
49+
void write(VkCommandBuffer cmdbuf, VkPipelineLayout layout, backend::ShaderStage stage,
50+
uint8_t index, backend::PushConstantVariant const& value);
51+
52+
private:
53+
static constexpr uint32_t ENTRY_SIZE = sizeof(uint32_t);
54+
55+
struct ConstantDescription {
56+
utils::FixedCapacityVector<backend::ConstantType> types;
57+
uint32_t offset = 0;
58+
};
59+
60+
// Describes the constants in each shader stage.
61+
ConstantDescription mDescriptions[Program::SHADER_TYPE_COUNT];
62+
VkPushConstantRange mRanges[Program::SHADER_TYPE_COUNT];
63+
uint32_t mRangeCount;
64+
};
65+
66+
struct VulkanProgram : public HwProgram, fvkmemory::ThreadSafeResource {
67+
using BindingList = fvkutils::StaticVector<uint16_t, MAX_SAMPLER_COUNT>;
68+
69+
VulkanProgram(VkDevice device, Program const& builder) noexcept;
70+
~VulkanProgram();
71+
72+
/**
73+
* Cancels any parallel compilation jobs that have not yet run for this
74+
* program.
75+
*/
76+
inline void cancelParallelCompilation() {
77+
mParallelCompilationCanceled.store(true, std::memory_order_release);
78+
}
79+
80+
/**
81+
* Writes out any queued push constants using the provided VkPipelineLayout.
82+
*
83+
* @param layout The layout that is to be used along with these push constants,
84+
* in the next draw call.
85+
*/
86+
void flushPushConstants(VkPipelineLayout layout);
87+
88+
inline VkShaderModule getVertexShader() const {
89+
return mInfo->shaders[0];
90+
}
91+
92+
inline VkShaderModule getFragmentShader() const { return mInfo->shaders[1]; }
93+
94+
inline uint32_t getPushConstantRangeCount() const {
95+
return mInfo->pushConstantDescription.getVkRangeCount();
96+
}
97+
98+
inline VkPushConstantRange const* getPushConstantRanges() const {
99+
return mInfo->pushConstantDescription.getVkRanges();
100+
}
101+
102+
/**
103+
* Returns true if parallel compilation is canceled, false if not. Parallel
104+
* compilation will be canceled if this program is destroyed before relevant
105+
* pipelines are created.
106+
*
107+
* @return true if parallel compilation should run for this program, false if not
108+
*/
109+
inline bool isParallelCompilationCanceled() const {
110+
return mParallelCompilationCanceled.load(std::memory_order_acquire);
111+
}
112+
113+
inline void writePushConstant(VkCommandBuffer cmdbuf, VkPipelineLayout layout,
114+
backend::ShaderStage stage, uint8_t index, backend::PushConstantVariant const& value) {
115+
// It's possible that we don't have the layout yet. When external samplers are used, bindPipeline()
116+
// in VulkanDriver returns early, without binding a layout. If that happens, the layout is not
117+
// set until draw time. Any push constants that are written during that time should be saved for
118+
// later, and flushed when the layout is set.
119+
if (layout != VK_NULL_HANDLE) {
120+
mInfo->pushConstantDescription.write(cmdbuf, layout, stage, index, value);
121+
} else {
122+
mQueuedPushConstants.push_back({cmdbuf, stage, index, value});
123+
}
124+
}
125+
126+
// TODO: handle compute shaders.
127+
// The expected order of shaders - from frontend to backend - is vertex, fragment, compute.
128+
static constexpr uint8_t const MAX_SHADER_MODULES = 2;
129+
130+
private:
131+
struct PipelineInfo {
132+
explicit PipelineInfo(backend::Program const& program) noexcept
133+
: pushConstantDescription(program)
134+
{}
135+
136+
VkShaderModule shaders[MAX_SHADER_MODULES] = { VK_NULL_HANDLE };
137+
PushConstantDescription pushConstantDescription;
138+
};
139+
140+
struct PushConstantInfo {
141+
VkCommandBuffer cmdbuf;
142+
backend::ShaderStage stage;
143+
uint8_t index;
144+
backend::PushConstantVariant value;
145+
};
146+
147+
PipelineInfo* mInfo;
148+
VkDevice mDevice = VK_NULL_HANDLE;
149+
std::atomic<bool> mParallelCompilationCanceled { false };
150+
std::vector<PushConstantInfo> mQueuedPushConstants;
151+
};
152+
38153
// Wrapper to enable use of shared_ptr for implementing shared ownership of low-level Vulkan fences.
39154
struct VulkanCmdFence {
40155
explicit VulkanCmdFence(VkFence fence) : mFence(fence) { }

‎filament/backend/src/vulkan/VulkanCommands.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@
3434
#include <utils/Mutex.h>
3535

3636
#include <atomic>
37-
3837
#include <chrono>
3938
#include <list>
4039
#include <utility>
40+
#include <variant>
4141

4242
namespace filament::backend {
4343

@@ -76,6 +76,10 @@ struct VulkanCommandBuffer {
7676
mResources.push_back(resource);
7777
}
7878

79+
inline void acquire(fvkmemory::resource_ptr<fvkmemory::ThreadSafeResource> resource) {
80+
mResources.push_back(resource);
81+
}
82+
7983
void reset() noexcept;
8084

8185
inline void insertWait(VkSemaphore sem, VkPipelineStageFlags waitStage) {
@@ -115,6 +119,9 @@ struct VulkanCommandBuffer {
115119
}
116120

117121
private:
122+
using HeldResource = std::variant<fvkmemory::resource_ptr<Resource>,
123+
fvkmemory::resource_ptr<ThreadSafeResource>>;
124+
118125
static uint32_t sAgeCounter;
119126

120127
VulkanContext const& mContext;
@@ -129,7 +136,7 @@ struct VulkanCommandBuffer {
129136
fvkmemory::resource_ptr<VulkanSemaphore> mSubmission;
130137
VkFence mFence;
131138
std::shared_ptr<VulkanCmdFence> mFenceStatus;
132-
std::vector<fvkmemory::resource_ptr<Resource>> mResources;
139+
std::vector<HeldResource> mResources;
133140
uint32_t mAge;
134141
};
135142

0 commit comments

Comments
 (0)