User-configurable gpuav indices_count. This can be used by apps using large amounts of draw commands to not run into the internal limit.

Author: MennoVink

layers/gpuav/core/gpuav_constants.h

@@ -23,12 +23,6 @@ namespace gpuav {
 // constant
 namespace cst {
 
-// Number of indices held in the buffer used to index commands and validation resources
-inline constexpr uint32_t indices_count = 1u << 13;  // 8192
-// If we hit our limit, we will use this to signal to the app what they are seeing is likely garbage.
-// This is required because we still need to bind our descriptors regardless.
-inline constexpr uint32_t invalid_index_command = indices_count - 1;
-
 // TODO - Need to develop a proper way to gather ALL binding between shader instrumentation and command validation
 // For now we get enough for DebugPrintf
 inline constexpr uint32_t total_internal_descriptors = 64;

layers/gpuav/core/gpuav_settings.cpp

@@ -36,6 +36,7 @@ bool GpuAVSettings::IsShaderInstrumentationEnabled() const {
            shader_instrumentation.sanitizer;
 }
 bool GpuAVSettings::IsSpirvModified() const { return IsShaderInstrumentationEnabled() || debug_printf_enabled; }
+uint32_t GpuAVSettings::GetInvalidIndexCommand() const { return indices_count - 1; }
 
 // Also disables shader caching and select shader instrumentation
 void GpuAVSettings::DisableShaderInstrumentationAndOptions() {

layers/gpuav/core/gpuav_settings.h

@@ -45,6 +45,7 @@ struct GpuAVSettings {
     bool debug_dump_instrumented_shaders = false;
     uint32_t debug_max_instrumentations_count = 0;  // zero is same as "unlimited"
     bool debug_print_instrumentation_info = false;
+    uint32_t indices_count = 1u << 13;
 
     bool descriptor_buffer_override = false;
 
@@ -64,6 +65,7 @@ struct GpuAVSettings {
 
     bool IsShaderInstrumentationEnabled() const;
     bool IsSpirvModified() const;
+    uint32_t GetInvalidIndexCommand() const;
 
     // Also disables shader caching and select shader instrumentation
     void DisableShaderInstrumentationAndOptions();

layers/gpuav/core/gpuav_setup.cpp

@@ -285,7 +285,7 @@ void Validator::FinishDeviceSetup(const VkDeviceCreateInfo *pCreateInfo, const L
 
         VkBufferCreateInfo buffer_info = vku::InitStructHelper();
         buffer_info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
-        buffer_info.size = cst::indices_count * indices_buffer_alignment_;
+        buffer_info.size = gpuav_settings.indices_count * indices_buffer_alignment_;
         VmaAllocationCreateInfo alloc_info = {};
         alloc_info.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
         alloc_info.preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
@@ -296,7 +296,7 @@ void Validator::FinishDeviceSetup(const VkDeviceCreateInfo *pCreateInfo, const L
 
         uint32_t stride = indices_buffer_alignment_ / sizeof(uint32_t);
         uint32_t *indices_ptr = (uint32_t *)global_indices_buffer_.GetMappedPtr();
-        for (uint32_t i = 0; i < cst::indices_count; ++i) {
+        for (uint32_t i = 0; i < gpuav_settings.indices_count; ++i) {
             const uint32_t offset = i * stride;
             indices_ptr[offset] = i;
         }

layers/gpuav/instrumentation/gpuav_instrumentation.cpp

@@ -393,8 +393,8 @@ void PreCallSetupShaderInstrumentationResourcesClassic(Validator &gpuav, Command
 
     const uint32_t error_logger_index = cb_state.GetErrorLoggerIndex();
 
-    assert(error_logger_index < cst::indices_count);
-    assert(error_info.action_command_index < cst::indices_count);
+    assert(error_logger_index < gpuav.gpuav_settings.indices_count);
+    assert(error_info.action_command_index < gpuav.gpuav_settings.indices_count);
     const std::array<uint32_t, 2> dynamic_offsets = {
         {error_info.action_command_index * gpuav.indices_buffer_alignment_, error_logger_index * gpuav.indices_buffer_alignment_}};
 

layers/gpuav/instrumentation/gpuav_shader_instrumentor.cpp

@@ -820,31 +820,30 @@ void GpuShaderInstrumentor::PostCallRecordCreateRayTracingPipelinesKHR(
             return;
         }
 
-        deferred_op_post_checks.emplace_back(
-            [this, held_chassis_state = chassis_state](std::pair<uint32_t, VkPipeline *> pipelines) mutable {
-                for (const auto [pipe_i, pipe] : vvl::enumerate(pipelines.second, pipelines.first)) {
-                    std::shared_ptr<vvl::Pipeline> pipeline_state = ((GpuShaderInstrumentor *)this)->Get<vvl::Pipeline>(pipe);
-                    ASSERT_AND_CONTINUE(pipeline_state);
-                    if (pipeline_state->ray_tracing_library_ci) {
-                        for (VkPipeline lib : vvl::make_span(pipeline_state->ray_tracing_library_ci->pLibraries,
-                                                             pipeline_state->ray_tracing_library_ci->libraryCount)) {
-                            auto lib_state = ((GpuShaderInstrumentor *)this)->Get<vvl::Pipeline>(lib);
-                            ASSERT_AND_CONTINUE(lib_state);
-                            pipeline_state->instrumentation_data.was_instrumented |=
-                                lib_state->instrumentation_data.was_instrumented;
-                        }
+        deferred_op_post_checks.emplace_back([this, held_chassis_state =
+                                                        chassis_state](std::pair<uint32_t, VkPipeline *> pipelines) mutable {
+            for (const auto [pipe_i, pipe] : vvl::enumerate(pipelines.second, pipelines.first)) {
+                std::shared_ptr<vvl::Pipeline> pipeline_state = ((GpuShaderInstrumentor *)this)->Get<vvl::Pipeline>(pipe);
+                ASSERT_AND_CONTINUE(pipeline_state);
+                if (pipeline_state->ray_tracing_library_ci) {
+                    for (VkPipeline lib : vvl::make_span(pipeline_state->ray_tracing_library_ci->pLibraries,
+                                                         pipeline_state->ray_tracing_library_ci->libraryCount)) {
+                        auto lib_state = ((GpuShaderInstrumentor *)this)->Get<vvl::Pipeline>(lib);
+                        ASSERT_AND_CONTINUE(lib_state);
+                        pipeline_state->instrumentation_data.was_instrumented |= lib_state->instrumentation_data.was_instrumented;
                     }
-                    auto &shader_instrumentation_metadata = held_chassis_state->shader_instrumentations_metadata[pipe_i];
-                    // Ray tracing pipelines can be made of libraries, but contrary to GPL instrumentation is not postponed
-                    // to final link time, and done at ray tracing library creation time.
-                    // => No need to iterate over shader stages coming from libraries,
-                    // stop at VkRayTracingPipelineCreateInfoKHR::stageCount
-                    // Note: This code implicitly relies on the fact that in pipeline_state->stage_states,
-                    // stages coming from libraries are added last.
-                    PostCallRecordPipelineCreationShaderInstrumentation(
-                        *pipeline_state, pipeline_state->RayTracingCreateInfo().stageCount, shader_instrumentation_metadata);
                 }
-            });
+                auto &shader_instrumentation_metadata = held_chassis_state->shader_instrumentations_metadata[pipe_i];
+                // Ray tracing pipelines can be made of libraries, but contrary to GPL instrumentation is not postponed
+                // to final link time, and done at ray tracing library creation time.
+                // => No need to iterate over shader stages coming from libraries,
+                // stop at VkRayTracingPipelineCreateInfoKHR::stageCount
+                // Note: This code implicitly relies on the fact that in pipeline_state->stage_states,
+                // stages coming from libraries are added last.
+                PostCallRecordPipelineCreationShaderInstrumentation(
+                    *pipeline_state, pipeline_state->RayTracingCreateInfo().stageCount, shader_instrumentation_metadata);
+            }
+        });
         dispatch_device_->deferred_operation_post_check.insert(deferredOperation, std::move(deferred_op_post_checks));
     } else {
         for (uint32_t i = 0; i < count; ++i) {
@@ -1129,7 +1128,7 @@ bool GpuShaderInstrumentor::PreCallRecordPipelineCreationShaderInstrumentation(
         auto modified_module_state = std::const_pointer_cast<vvl::ShaderModule>(stage_state.module_state);
         ASSERT_AND_CONTINUE(modified_module_state);
         if (!modified_module_state->spirv) {
-            continue; // Hit when using VK_KHR_pipeline_binary
+            continue;  // Hit when using VK_KHR_pipeline_binary
         }
         std::unique_lock<std::mutex> module_lock(modified_module_state->module_mutex_);
 
@@ -1818,8 +1817,9 @@ std::string GpuShaderInstrumentor::GenerateDebugInfoMessage(VkCommandBuffer comm
             ss << "Unknown Pipeline Operation ";
         }
 
-        if (action_command_index == cst::invalid_index_command) {
-            ss << "Index Unknown (After " << cst::invalid_index_command << " commands, we stop tracking) \n";
+        uint32_t invalid_index_command = dispatch_instance_->settings.gpuav_settings.GetInvalidIndexCommand();
+        if (action_command_index == invalid_index_command) {
+            ss << "Index Unknown (After " << invalid_index_command << " commands, we stop tracking) \n";
         } else {
             ss << "Index " << action_command_index << '\n';
         }

layers/gpuav/instrumentation/post_process_descriptor_indexing.cpp

@@ -197,12 +197,13 @@ void RegisterPostProcessingValidation(Validator& gpuav, CommandBufferSubState& c
 
                 context.SetOriginalSpirv(&it->second.original_spirv);
 
+                uint32_t invalid_index_command = gpuav.gpuav_settings.GetInvalidIndexCommand();
                 for (const DescriptorAccess& descriptor_access : descriptor_accesses) {
-                    if (descriptor_access.error_logger_i == cst::invalid_index_command) {
+                    if (descriptor_access.error_logger_i == invalid_index_command) {
                         gpuav.LogError("GPUAV-Overflow-Unknown", LogObjectList(), Location(vvl::Func::Empty),
                                        "Cannot perform runtime descriptor access validation, access was done in a command past the "
                                        "internal limit of %" PRIu32 " draw/dispatch/traceRays in a command buffer.",
-                                       cst::indices_count);
+                                       gpuav.gpuav_settings.indices_count);
                         continue;
                     }
 

layers/gpuav/resources/gpuav_state_trackers.cpp

@@ -152,7 +152,7 @@ void CommandBufferSubState::RecordEndRenderPass(const VkSubpassEndInfo *, const
 // For things like vkCmdCopyImage there is no "last bound" as not shaders are attached to it
 void CommandBufferSubState::AddCommandErrorLogger(const Location &loc, const LastBound *last_bound,
                                                   ErrorLoggerFunc error_logger_func) {
-    if (command_error_loggers_.size() == cst::invalid_index_command) {
+    if (command_error_loggers_.size() == gpuav_.gpuav_settings.GetInvalidIndexCommand()) {
         return;
     }
 
@@ -203,18 +203,18 @@ void CommandBufferSubState::ResetCBState(bool should_destroy) {
 void CommandBufferSubState::IncrementActionCommandCount(VkPipelineBindPoint bind_point) {
     if (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS) {
         draw_index++;
-        if (draw_index > cst::invalid_index_command) {
-            draw_index = cst::invalid_index_command;
+        if (draw_index > gpuav_.gpuav_settings.GetInvalidIndexCommand()) {
+            draw_index = gpuav_.gpuav_settings.GetInvalidIndexCommand();
         }
     } else if (bind_point == VK_PIPELINE_BIND_POINT_COMPUTE) {
         compute_index++;
-        if (compute_index > cst::invalid_index_command) {
-            compute_index = cst::invalid_index_command;
+        if (compute_index > gpuav_.gpuav_settings.GetInvalidIndexCommand()) {
+            compute_index = gpuav_.gpuav_settings.GetInvalidIndexCommand();
         }
     } else if (bind_point == VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR) {
         trace_rays_index++;
-        if (trace_rays_index > cst::invalid_index_command) {
-            trace_rays_index = cst::invalid_index_command;
+        if (trace_rays_index > gpuav_.gpuav_settings.GetInvalidIndexCommand()) {
+            trace_rays_index = gpuav_.gpuav_settings.GetInvalidIndexCommand();
         }
     }
 }
@@ -351,14 +351,14 @@ void CommandBufferSubState::OnCompletion(VkQueue queue, const std::vector<std::s
                 const uint32_t error_logger_i =
                     error_record_ptr[glsl::kHeader_ActionIdErrorLoggerIdOffset] & glsl::kErrorLoggerId_Mask;
 
-                assert(error_logger_i < cst::indices_count);
-                if (error_logger_i == cst::invalid_index_command) {
+                assert(error_logger_i < gpuav_.gpuav_settings.indices_count);
+                if (error_logger_i == gpuav_.gpuav_settings.GetInvalidIndexCommand()) {
                     const LogObjectList objlist(queue, VkHandle());
                     gpuav_.LogError(
                         "GPUAV-Overflow-Unknown", queue, loc,
                         "An error was detected, but after internal limit of %" PRIu32
                         " draw/dispatch/traceRays in a command buffer, we are unable to track which validation error occured.",
-                        cst::indices_count);
+                        gpuav_.gpuav_settings.indices_count);
                 } else {
                     // normal case
                     const CommandErrorLogger &error_logger = GetErrorLogger(error_logger_i);

layers/gpuav/validation_cmd/gpuav_validation_cmd_common.cpp

@@ -30,8 +30,8 @@ namespace valcmd {
 namespace internal {
 static void BindErrorLoggingDescSet(Validator &gpuav, CommandBufferSubState &cb_state, VkPipelineBindPoint bind_point,
                                     VkPipelineLayout pipeline_layout, uint32_t cmd_index, uint32_t error_logger_index) {
-    assert(cmd_index < cst::indices_count);
-    assert(error_logger_index < cst::indices_count);
+    assert(cmd_index < gpuav.gpuav_settings.indices_count);
+    assert(error_logger_index < gpuav.gpuav_settings.indices_count);
     std::array<uint32_t, 2> dynamic_offsets = {
         {cmd_index * gpuav.indices_buffer_alignment_, error_logger_index * gpuav.indices_buffer_alignment_}};
 

layers/layer_options.cpp

@@ -218,6 +218,7 @@ const char *VK_LAYER_GPUAV_DEBUG_VALIDATE_INSTRUMENTED_SHADERS = "gpuav_debug_va
 const char *VK_LAYER_GPUAV_DEBUG_DUMP_INSTRUMENTED_SHADERS = "gpuav_debug_dump_instrumented_shaders";
 const char *VK_LAYER_GPUAV_DEBUG_MAX_INSTRUMENTATIONS_COUNT = "gpuav_debug_max_instrumentations_count";
 const char *VK_LAYER_GPUAV_DEBUG_PRINT_INSTRUMENTATION_INFO = "gpuav_debug_print_instrumentation_info";
+const char *VK_LAYER_GPUAV_INDICES_COUNT = "gpuav_indices_count";
 
 // SyncVal
 // ---
@@ -1084,6 +1085,11 @@ void ProcessConfigAndEnvSettings(ConfigAndEnvSettings *settings_data) {
                                 gpuav_settings.debug_print_instrumentation_info);
     }
 
+    if (vkuHasLayerSetting(layer_setting_set, VK_LAYER_GPUAV_INDICES_COUNT)) {
+        vkuGetLayerSettingValue(layer_setting_set, VK_LAYER_GPUAV_INDICES_COUNT,
+                                gpuav_settings.indices_count);
+    }
+
     if (vkuHasLayerSetting(layer_setting_set, VK_LAYER_PRINTF_TO_STDOUT)) {
         vkuGetLayerSettingValue(layer_setting_set, VK_LAYER_PRINTF_TO_STDOUT, gpuav_settings.debug_printf_to_stdout);
     }