Add SPM AQL packet construction and HSA queue integration

Author: SrirakshaNag

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/aql/packet_construct.cpp

@@ -42,6 +42,13 @@ namespace rocprofiler
 {
 namespace aql
 {
+struct AQLProfileMetric
+{
+    counters::Metric                    metric;
+    std::vector<aqlprofile_pmc_event_t> instances;
+    std::vector<aqlprofile_pmc_event_t> events;
+};
+
 CounterPacketConstruct::CounterPacketConstruct(rocprofiler_agent_id_t               agent,
                                                const std::vector<counters::Metric>& metrics)
 : _agent(agent)
@@ -50,7 +57,7 @@ CounterPacketConstruct::CounterPacketConstruct(rocprofiler_agent_id_t
     // for the counter.
     for(const auto& x : metrics)
     {
-        auto query_info                = get_query_info(_agent, x);
+        auto query_info                = get_query_info(_agent, x.block(), x.name());
         _metrics.emplace_back().metric = x;
         uint64_t event_id              = 0;
         if(!x.event().empty()) event_id = std::stoul(x.event(), nullptr);
@@ -278,5 +285,150 @@ CounterPacketConstruct::can_collect()
     }
     return ROCPROFILER_STATUS_SUCCESS;
 }
+
+/** @brief Constructs the packet using the contained input parameters.
+ * Writes into ID map and spm descriptor used to decode SPM data
+ */
+std::unique_ptr<hsa::SPMPacket>
+spm_construct_packet(const rocprofiler_agent_id_t         agent_id,
+                     const std::vector<counters::Metric>& metrics,
+                     double                               sample_freq,
+                     uint64_t                             buffer_size,
+                     uint64_t                             timeout)
+{
+    auto events = std::vector<aqlprofile_pmc_event_t>{};
+    auto params = std::vector<aqlprofile_spm_parameter_t>{};
+    auto id_map = std::vector<spm::spm_counter_instance_t>{};
+
+    const auto* agent     = CHECK_NOTNULL(rocprofiler::agent::get_agent(agent_id));
+    const auto* aql_cache = CHECK_NOTNULL(rocprofiler::agent::get_agent_cache(agent));
+    auto        pool      = std::make_shared<hsa::SPMMemoryPool>(
+        *aql_cache, *hsa::get_amd_ext_table(), hsa::get_core_table()->hsa_memory_copy_fn);
+    const auto* aql_agent = rocprofiler::agent::get_aql_agent(agent->id);
+
+    const double sclk_freq   = agent->max_engine_clk_fcompute * 1E9;  // GHz
+    const size_t sclk_period = static_cast<size_t>(std::roundf(sclk_freq / ((sample_freq) *1E9)));
+
+    params.push_back({AQLPROFILE_SPM_PARAMETER_TYPE_BUFFER_SIZE, buffer_size * 1024});
+    params.push_back({AQLPROFILE_SPM_PARAMETER_TYPE_SAMPLE_INTERVAL, sclk_period});
+    params.push_back({AQLPROFILE_SPM_PARAMETER_TYPE_TIMEOUT, timeout});
+
+    for(const auto& metric : metrics)
+    {
+        auto query_info = get_query_info(agent_id, metric.block(), metric.name());
+
+        for(unsigned block_index = 0; block_index < query_info.instance_count; ++block_index)
+        {
+            auto event = aqlprofile_pmc_event_t{
+                .block_index = block_index,
+                .event_id =
+                    static_cast<uint32_t>(std::stoul(metric.event().c_str(), nullptr) & 0xFFFFFFFF),
+                .flags      = aqlprofile_pmc_event_flags_t{metric.flags()},
+                .block_name = static_cast<hsa_ven_amd_aqlprofile_block_name_t>(query_info.id)};
+
+            events.push_back(event);
+            id_map.push_back({rocprofiler_counter_id_t{.handle = metric.id()}, block_index});
+        }
+    }
+
+    aqlprofile_spm_profile_t profile{.aql_agent       = *aql_agent,
+                                     .hsa_agent       = pool->gpu_agent,
+                                     .events          = events.data(),
+                                     .event_count     = events.size(),
+                                     .parameters      = params.data(),
+                                     .parameter_count = params.size(),
+                                     .reserved        = 0,
+                                     .alloc_cb        = &(hsa::SPMMemoryPool::Alloc),
+                                     .dealloc_cb      = &(hsa::SPMMemoryPool::Free),
+                                     .memcpy_cb       = &(hsa::SPMMemoryPool::Copy),
+                                     .userdata        = pool.get()};
+
+    auto pkt = std::make_unique<hsa::SPMPacket>(*aql_agent, profile);
+    ROCP_FATAL_IF(!pkt->valid()) << "SPM Packet creation failed";
+
+    pool->delete_packets_fn = pkt->sym.delete_packets_fn;
+    pool->handle            = pkt->handle;
+    pkt->pool               = std::move(pool);
+
+    pkt->spm_desc.size =
+        sizeof(spm::spm_desc_v0_t) + id_map.size() * sizeof(id_map[0]) + pkt->aql_desc.size;
+
+    pkt->container_desc_data = std::make_shared<std::vector<char>>(pkt->spm_desc.size);
+    pkt->spm_desc.data       = pkt->container_desc_data->data();
+
+    auto* desc = static_cast<spm::spm_desc_v0_t*>(pkt->spm_desc.data);
+
+    *desc               = spm::spm_desc_v0_t{};
+    desc->aql_desc_size = pkt->aql_desc.size;
+    desc->num_events    = id_map.size();
+
+    std::memcpy(desc->aqlprofile_desc(), pkt->aql_desc.data, pkt->aql_desc.size);
+    std::memcpy(desc->events(), id_map.data(), id_map.size() * sizeof(id_map[0]));
+
+    pkt->clear();
+    return pkt;
+}
+
+// Following the PMC check for now
+// ToDO: change this to SPM
+rocprofiler_status_t
+spm_can_collect(const rocprofiler_agent_id_t agent_id, const std::vector<counters::Metric>& metrics)
+{
+    // Verify that the counters fit within harrdware limits
+    auto counter_count =
+        std::map<std::pair<hsa_ven_amd_aqlprofile_block_name_t, uint32_t>, int64_t>{};
+    auto max_allowed =
+        std::map<std::pair<hsa_ven_amd_aqlprofile_block_name_t, uint32_t>, int64_t>{};
+    auto _metrics = std::vector<AQLProfileMetric>{};
+
+    for(const auto& metric : metrics)
+    {
+        auto query_info                = get_query_info(agent_id, metric.block(), metric.name());
+        _metrics.emplace_back().metric = metric;
+
+        auto event_id =
+            static_cast<uint32_t>(std::stoul(metric.event().c_str(), nullptr) & 0xFFFFFFFF);
+
+        for(unsigned block_index = 0; block_index < query_info.instance_count; ++block_index)
+        {
+            _metrics.back().instances.push_back(
+                {.block_index = block_index,
+                 .event_id    = event_id,
+                 .flags       = aqlprofile_pmc_event_flags_t{metric.flags()},
+                 .block_name  = static_cast<hsa_ven_amd_aqlprofile_block_name_t>(query_info.id)});
+
+            _metrics.back().events.push_back(
+                {.block_index = block_index,
+                 .event_id    = event_id,
+                 .flags       = aqlprofile_pmc_event_flags_t{metric.flags()},
+                 .block_name  = static_cast<hsa_ven_amd_aqlprofile_block_name_t>(query_info.id)});
+        }
+    }
+
+    for(auto& metric : _metrics)
+    {
+        for(auto& instance : metric.events)
+        {
+            auto block_pair       = std::make_pair(instance.block_name, instance.block_index);
+            auto [iter, inserted] = counter_count.emplace(block_pair, 0);
+            iter->second++;
+            if(inserted)
+            {
+                max_allowed.emplace(block_pair, get_block_counters(agent_id, instance));
+            }
+        }
+    }
+
+    // Check if the block count > max count
+    for(auto& [block_name, count] : counter_count)
+    {
+        if(auto* max = CHECK_NOTNULL(common::get_val(max_allowed, block_name)); count > *max)
+        {
+            return ROCPROFILER_STATUS_ERROR_EXCEEDS_HW_LIMIT;
+        }
+    }
+    return ROCPROFILER_STATUS_SUCCESS;
+}
+
 }  // namespace aql
 }  // namespace rocprofiler

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/aql/packet_construct.hpp

@@ -26,6 +26,7 @@
 #include "lib/rocprofiler-sdk/aql/helpers.hpp"
 #include "lib/rocprofiler-sdk/counters/metrics.hpp"
 #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp"
+#include "lib/rocprofiler-sdk/spm/decode.hpp"
 #include "lib/rocprofiler-sdk/thread_trace/core.hpp"
 
 #include <rocprofiler-sdk/fwd.h>
@@ -126,5 +127,16 @@ class ThreadTraceAQLPacketFactory
     hsa::TraceMemoryPool tracepool;
 };
 
+std::unique_ptr<hsa::SPMPacket>
+spm_construct_packet(const rocprofiler_agent_id_t         agent_id,
+                     const std::vector<counters::Metric>& metrics,
+                     double                               sample_freq,
+                     uint64_t                             buffer_size,
+                     uint64_t                             timeout);
+
+rocprofiler_status_t
+spm_can_collect(const rocprofiler_agent_id_t         agent_id,
+                const std::vector<counters::Metric>& metrics);
+
 }  // namespace aql
 }  // namespace rocprofiler

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/aql_packet.cpp

@@ -21,6 +21,8 @@
 // THE SOFTWARE.
 
 #include "lib/rocprofiler-sdk/hsa/aql_packet.hpp"
+#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp"
+
 #include <fmt/core.h>
 #include <cstdlib>
 #include <iostream>
@@ -242,5 +244,136 @@ CodeobjMarkerAQLPacket::CodeobjMarkerAQLPacket(const TraceMemoryPool& _tracepool
     clear();
 }
 
+SPMMemoryPool::SPMMemoryPool(const AgentCache& agent, const AmdExtTable& ext, copy_fn_t copy_fn)
+{
+    allocate_fn     = ext.hsa_amd_memory_pool_allocate_fn;
+    allow_access_fn = ext.hsa_amd_agents_allow_access_fn;
+    free_fn         = ext.hsa_amd_memory_pool_free_fn;
+    fill_fn         = ext.hsa_amd_memory_fill_fn;
+    api_copy_fn     = copy_fn;
+
+    gpu_agent     = agent.get_hsa_agent();
+    cpu_pool_     = agent.cpu_pool();
+    gpu_pool_     = agent.gpu_pool();
+    kernarg_pool_ = agent.kernarg_pool();
+}
+
+void
+SPMMemoryPool::Free(void* ptr, void* data)
+{
+    if(ptr == nullptr) return;
+    auto* pool = reinterpret_cast<SPMMemoryPool*>(data);
+
+    ROCP_FATAL_IF(!pool || !pool->free_fn) << "Unable to deallocate from HSA memory pool";
+    pool->free_fn(ptr);
+}
+
+hsa_status_t
+SPMMemoryPool::Copy(void* dst, const void* src, size_t size, void* data)
+{
+    if(size == 0) return HSA_STATUS_SUCCESS;
+    auto* pool = reinterpret_cast<SPMMemoryPool*>(data);
+    ROCP_FATAL_IF(!pool || !pool->api_copy_fn) << "Unable to copy HSA memory";
+
+    return pool->api_copy_fn(dst, src, size);
+}
+
+hsa_status_t
+SPMMemoryPool::Alloc(void** ptr, size_t size, aqlprofile_buffer_desc_flags_t flags, void* data)
+{
+    hsa_status_t status = HSA_STATUS_ERROR;
+
+    if(size == 0)
+    {
+        if(ptr != nullptr) *ptr = nullptr;
+        return HSA_STATUS_SUCCESS;
+    }
+    if(!data) return HSA_STATUS_ERROR;
+
+    auto& pool = *reinterpret_cast<SPMMemoryPool*>(data);
+    if(!pool.allocate_fn || !pool.free_fn || !pool.allow_access_fn) return HSA_STATUS_ERROR;
+
+    if(flags.host_access)
+        status = pool.allocate_fn(pool.cpu_pool_, size, hsa_amd_memory_pool_executable_flag, ptr);
+    else
+        status =
+            pool.allocate_fn(pool.kernarg_pool_, size, hsa_amd_memory_pool_executable_flag, ptr);
+
+    if(status == HSA_STATUS_SUCCESS)
+        status = pool.allow_access_fn(1, &pool.gpu_agent, nullptr, *ptr);
+    if(status == HSA_STATUS_SUCCESS) status = pool.fill_fn(*ptr, 0u, size / sizeof(uint32_t));
+
+    return status;
+}
+
+SPMPacket::SPMPacket(aqlprofile_agent_handle_t aql_agent, aqlprofile_spm_profile_t profile)
+: agent(aql_agent)
+, sym()
+{
+    if(!sym.valid()) return;
+    auto status = sym.create_packets_fn(&handle, &aql_desc, &packets, profile, 0);
+
+    if(status == HSA_STATUS_ERROR_INVALID_AGENT) return;
+
+    packets.start_packet.header            = VENDOR_BIT | BARRIER_BIT;
+    packets.stop_packet.header             = VENDOR_BIT | BARRIER_BIT;
+    packets.start_packet.completion_signal = hsa_signal_t{.handle = 0};
+    packets.stop_packet.completion_signal  = hsa_signal_t{.handle = 0};
+
+    status = sym.spm_query_fn(aql_desc, AQLPROFILE_SPM_DECODE_QUERY_SEG_SIZE, &spm_desc.seg_size);
+    if(status != HSA_STATUS_SUCCESS) return;
+    status = sym.spm_query_fn(aql_desc, AQLPROFILE_SPM_DECODE_QUERY_NUM_XCC, &spm_desc.buffer_num);
+    if(status != HSA_STATUS_SUCCESS) return;
+
+    is_valid = true;
+    empty    = false;
+}
+
+void
+SPMPacket::populate_before()
+{
+    hsa_barrier_and_packet_t barrier{};
+    barrier.header = HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE;
+    barrier.header |= BARRIER_BIT;
+
+    before_krn_barrier_pkt.push_back(barrier);
+    before_krn_barrier_pkt.push_back(barrier);
+    before_krn_pkt.push_back(packets.start_packet);
+};
+
+void
+SPMPacket::populate_after()
+{
+    after_krn_pkt.push_back(packets.stop_packet);
+};
+
+void
+SPMPacket::kfd_start()
+{
+    ROCP_FATAL_IF(!handle.handle) << "Attempt at starting SPM with unitialized packet!";
+
+    if(running.exchange(true))
+    {
+        ROCP_ERROR << "Double call to KFD start!";
+        return;
+    }
+
+    auto status = sym.spm_start_fn(this->handle, spm::aql_data_callback, this);
+    ROCP_FATAL_IF(status != HSA_STATUS_SUCCESS) << "Unable to acquire KFD thread";
+}
+
+void
+SPMPacket::kfd_stop()
+{
+    if(running.exchange(false))
+        sym.spm_stop_fn(this->handle);
+    else
+        ROCP_WARNING << "Double call to KFD stop!";
+}
+
+SPMPacket::~SPMPacket()
+{
+    if(running.exchange(false) && sym.valid()) sym.spm_stop_fn(this->handle);
+}
 }  // namespace hsa
 }  // namespace rocprofiler