Merge pull request #1206 from lplewa/fixed_ctl

add ctl to fixed provider

Author: KFilipek

benchmark/benchmark_umf.hpp

@@ -44,7 +44,8 @@ struct provider_interface {
         if (state.thread_index() != 0) {
             return;
         }
-        umfCtlExec("umf.provider.by_handle.stats.reset", provider, NULL);
+        umfCtlExec("umf.provider.by_handle.stats.peak_memory.reset", provider,
+                   NULL);
     }
 
     void postBench([[maybe_unused]] ::benchmark::State &state) {

src/ctl/ctl.c

@@ -102,7 +102,7 @@ umf_result_t umfCtlSet(const char *name, void *ctx, void *arg) {
 }
 
 umf_result_t umfCtlExec(const char *name, void *ctx, void *arg) {
-    if (name == NULL || arg == NULL || ctx == NULL) {
+    if (name == NULL || ctx == NULL) {
         return UMF_RESULT_ERROR_INVALID_ARGUMENT;
     }
     return ctl_query(NULL, ctx, CTL_QUERY_PROGRAMMATIC, name,
@@ -140,6 +140,7 @@ static const umf_ctl_node_t *ctl_find_node(const umf_ctl_node_t *nodes,
      * in the main ctl tree.
      */
     while (node_name != NULL) {
+        char *next_node = strtok_r(NULL, CTL_QUERY_NODE_SEPARATOR, &sptr);
         *name_offset = node_name - parse_str;
         if (n != NULL && n->type == CTL_NODE_SUBTREE) {
             // if a subtree occurs, the subtree handler should be called
@@ -168,6 +169,14 @@ static const umf_ctl_node_t *ctl_find_node(const umf_ctl_node_t *nodes,
             if (index_entry && n->type == CTL_NODE_INDEXED) {
                 break;
             } else if (strcmp(n->name, node_name) == 0) {
+                if (n->type == CTL_NODE_LEAF && next_node != NULL) {
+                    // this is not the last node in the query, so it couldn't be leaf
+                    continue;
+                }
+                if (n->type != CTL_NODE_LEAF && next_node == NULL) {
+                    // this is the last node in the query, so it must be a leaf
+                    continue;
+                }
                 break;
             }
         }
@@ -181,7 +190,7 @@ static const umf_ctl_node_t *ctl_find_node(const umf_ctl_node_t *nodes,
         }
 
         nodes = n->children;
-        node_name = strtok_r(NULL, CTL_QUERY_NODE_SEPARATOR, &sptr);
+        node_name = next_node;
     }
 
     umf_ba_global_free(parse_str);

src/provider/provider_ctl_stats_impl.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifdef UMF_PROVIDER_CTL_STATS_IMPL_H
+#error This file should not be included more than once
+#else
+#define UMF_PROVIDER_CTL_STATS_IMPL_H 1
+
+#ifndef CTL_PROVIDER_TYPE
+#error "CTL_PROVIDER_TYPE must be defined"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ctl/ctl.h"
+#include "utils/utils_assert.h"
+
+static int CTL_READ_HANDLER(peak_memory)(void *ctx,
+                                         umf_ctl_query_source_t source,
+                                         void *arg,
+                                         umf_ctl_index_utlist_t *indexes,
+                                         const char *extra_name,
+                                         umf_ctl_query_type_t query_type) {
+    /* suppress unused-parameter errors */
+    (void)source, (void)indexes, (void)extra_name, (void)query_type;
+
+    size_t *arg_out = arg;
+    CTL_PROVIDER_TYPE *provider = (CTL_PROVIDER_TYPE *)ctx;
+    utils_atomic_load_acquire_size_t(&provider->stats.peak_memory, arg_out);
+    return 0;
+}
+
+static int CTL_READ_HANDLER(allocated_memory)(void *ctx,
+                                              umf_ctl_query_source_t source,
+                                              void *arg,
+                                              umf_ctl_index_utlist_t *indexes,
+                                              const char *extra_name,
+                                              umf_ctl_query_type_t query_type) {
+    /* suppress unused-parameter errors */
+    (void)source, (void)indexes, (void)extra_name, (void)query_type;
+
+    size_t *arg_out = arg;
+    CTL_PROVIDER_TYPE *provider = (CTL_PROVIDER_TYPE *)ctx;
+    utils_atomic_load_acquire_size_t(&provider->stats.allocated_memory,
+                                     arg_out);
+    return 0;
+}
+
+static int CTL_RUNNABLE_HANDLER(reset)(void *ctx, umf_ctl_query_source_t source,
+                                       void *arg,
+                                       umf_ctl_index_utlist_t *indexes,
+                                       const char *extra_name,
+                                       umf_ctl_query_type_t query_type) {
+    /* suppress unused-parameter errors */
+    (void)source, (void)indexes, (void)arg, (void)extra_name, (void)query_type;
+
+    CTL_PROVIDER_TYPE *provider = (CTL_PROVIDER_TYPE *)ctx;
+    size_t allocated;
+    size_t current_peak;
+
+    utils_atomic_load_acquire_size_t(&provider->stats.peak_memory,
+                                     &current_peak);
+    do {
+        utils_atomic_load_acquire_size_t(&provider->stats.allocated_memory,
+                                         &allocated);
+    } while (!utils_compare_exchange_size_t(&provider->stats.peak_memory,
+                                            &current_peak, &allocated));
+
+    return 0;
+}
+
+static const umf_ctl_node_t CTL_NODE(peak_memory)[] = {CTL_LEAF_RUNNABLE(reset),
+                                                       CTL_NODE_END};
+
+static const umf_ctl_node_t CTL_NODE(stats)[] = {
+    CTL_LEAF_RO(allocated_memory), CTL_LEAF_RO(peak_memory),
+    CTL_CHILD(peak_memory), CTL_LEAF_RUNNABLE(reset), CTL_NODE_END};
+
+static inline void provider_ctl_stats_alloc(CTL_PROVIDER_TYPE *provider,
+                                            size_t size) {
+    size_t allocated =
+        utils_fetch_and_add_size_t(&provider->stats.allocated_memory, size) +
+        size;
+
+    size_t peak;
+    utils_atomic_load_acquire_size_t(&provider->stats.peak_memory, &peak);
+
+    // If the compare-exchange fails, 'peak' is updated to the current value
+    // of peak_memory. We then re-check whether allocated is still greater than
+    // the updated peak value.
+    while (allocated > peak &&
+           !utils_compare_exchange_size_t(&provider->stats.peak_memory, &peak,
+                                          &allocated)) {
+        ;
+    }
+}
+
+static inline void provider_ctl_stats_free(CTL_PROVIDER_TYPE *provider,
+                                           size_t size) {
+    utils_fetch_and_sub_size_t(&provider->stats.allocated_memory, size);
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif

src/provider/provider_ctl_stats_type.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+*/
+
+#ifndef UMF_PROVIDER_CTL_STATS_TYPE_H
+#define UMF_PROVIDER_CTL_STATS_TYPE_H 1
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ctl_stats_t {
+    size_t allocated_memory;
+    size_t peak_memory;
+} ctl_stats_t;
+
+#ifdef __cplusplus
+}
+#endif
+#endif

src/provider/provider_fixed_memory.c

@@ -20,6 +20,7 @@
 #include "base_alloc_global.h"
 #include "coarse.h"
 #include "libumf.h"
+#include "provider_ctl_stats_type.h"
 #include "utils_common.h"
 #include "utils_concurrency.h"
 #include "utils_log.h"
@@ -30,6 +31,7 @@ typedef struct fixed_memory_provider_t {
     void *base;       // base address of memory
     size_t size;      // size of the memory region
     coarse_t *coarse; // coarse library handle
+    ctl_stats_t stats;
 } fixed_memory_provider_t;
 
 // Fixed Memory provider settings struct
@@ -52,6 +54,17 @@ static __TLS fixed_last_native_error_t TLS_last_native_error;
 #define _UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED                             \
     (UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED - UMF_FIXED_RESULT_SUCCESS)
 
+#define CTL_PROVIDER_TYPE fixed_memory_provider_t
+#include "provider_ctl_stats_impl.h"
+
+struct ctl *fixed_memory_ctl_root;
+static UTIL_ONCE_FLAG ctl_initialized = UTIL_ONCE_FLAG_INIT;
+
+static void initialize_fixed_ctl(void) {
+    fixed_memory_ctl_root = ctl_new();
+    CTL_REGISTER_MODULE(fixed_memory_ctl_root, stats);
+}
+
 static const char *Native_error_str[] = {
     [_UMF_FIXED_RESULT_SUCCESS] = "success",
     [_UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED] = "force purging failed"};
@@ -153,7 +166,14 @@ static umf_result_t fixed_alloc(void *provider, size_t size, size_t alignment,
     fixed_memory_provider_t *fixed_provider =
         (fixed_memory_provider_t *)provider;
 
-    return coarse_alloc(fixed_provider->coarse, size, alignment, resultPtr);
+    umf_result_t ret =
+        coarse_alloc(fixed_provider->coarse, size, alignment, resultPtr);
+
+    if (ret == UMF_RESULT_SUCCESS) {
+        provider_ctl_stats_alloc(fixed_provider, size);
+    }
+
+    return ret;
 }
 
 static void fixed_get_last_native_error(void *provider, const char **ppMessage,
@@ -250,7 +270,22 @@ static umf_result_t fixed_allocation_merge(void *provider, void *lowPtr,
 static umf_result_t fixed_free(void *provider, void *ptr, size_t size) {
     fixed_memory_provider_t *fixed_provider =
         (fixed_memory_provider_t *)provider;
-    return coarse_free(fixed_provider->coarse, ptr, size);
+
+    umf_result_t ret = coarse_free(fixed_provider->coarse, ptr, size);
+
+    if (ret == UMF_RESULT_SUCCESS) {
+        provider_ctl_stats_free(fixed_provider, size);
+    }
+
+    return ret;
+}
+
+static umf_result_t fixed_ctl(void *provider, int operationType,
+                              const char *name, void *arg,
+                              umf_ctl_query_type_t query_type) {
+    utils_init_once(&ctl_initialized, initialize_fixed_ctl);
+    return ctl_query(fixed_memory_ctl_root, provider, operationType, name,
+                     query_type, arg);
 }
 
 static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = {
@@ -271,7 +306,8 @@ static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = {
     .ipc.get_ipc_handle = NULL,
     .ipc.put_ipc_handle = NULL,
     .ipc.open_ipc_handle = NULL,
-    .ipc.close_ipc_handle = NULL};
+    .ipc.close_ipc_handle = NULL,
+    .ctl = fixed_ctl};
 
 umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) {
     return &UMF_FIXED_MEMORY_PROVIDER_OPS;