x

Author: bratpiorka

include/umf/providers/provider_os_memory.h

@@ -46,6 +46,7 @@ typedef enum umf_numa_mode_t {
     /// umf_numa_split_partition_t can be passed in umf_os_memory_provider_params_t structure
     /// to specify other distribution.
     UMF_NUMA_MODE_SPLIT,
+
     /// The memory is allocated on the node of the CPU that triggered the
     /// allocation. If this mode is specified, nodemask must be NULL and
     /// maxnode must be 0.
@@ -58,6 +59,7 @@ typedef struct umf_numa_split_partition_t {
     /// The weight of the partition, representing the proportion of
     /// the allocation that should be assigned to this NUMA node.
     unsigned weight;
+
     /// The NUMA node where the pages assigned to this partition will be bound.
     unsigned target;
 } umf_numa_split_partition_t;

src/libumf.c

@@ -45,6 +45,9 @@ umf_result_t umfInit(void) {
     utils_mutex_lock(&initMutex);
 
     if (umfRefCount == 0) {
+        // debug comment
+        LOG_FATAL("umfInit");
+
         utils_log_init();
         umf_result_t umf_result = umfMemoryTrackerCreate(&TRACKER);
         if (umf_result != UMF_RESULT_SUCCESS) {

src/provider/provider_os_memory.c

@@ -8,11 +8,16 @@
 #include <assert.h>
 #include <errno.h>
 #include <limits.h>
-
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+
+#ifndef _WIN32
+#include <numaif.h>
+#include <sys/syscall.h>
+#endif
+
 #include <umf.h>
 #include <umf/base.h>
 #include <umf/memory_provider.h>
@@ -32,8 +37,8 @@
 #define CTL_PROVIDER_TYPE os_memory_provider_t
 #include "provider_ctl_stats_impl.h"
 
+#define MAX_NUMNODES 1024
 #define NODESET_STR_BUF_LEN 1024
-
 #define TLS_MSG_BUF_LEN 1024
 
 static const char *DEFAULT_NAME = "OS";
@@ -152,8 +157,42 @@ static umf_result_t initialize_nodeset(os_memory_provider_t *os_provider,
         // Hwloc_set_area_membind fails if empty nodeset is passed so
         // if no node is specified, just pass all available nodes.
         // For modes where no node is needed, they will be ignored anyway.
+
+#ifdef _WIN32
         out_nodeset[0] = hwloc_bitmap_dup(
             hwloc_topology_get_complete_nodeset(os_provider->topo));
+#else
+        size_t *nodes = umf_ba_global_alloc(sizeof(size_t) * MAX_NUMNODES);
+        if (!nodes) {
+            umf_ba_global_free(out_nodeset);
+            os_provider->nodeset_len = 0;
+            return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        }
+
+        out_nodeset[0] = hwloc_bitmap_alloc();
+        if (!out_nodeset[0]) {
+            umf_ba_global_free(out_nodeset);
+            umf_ba_global_free(nodes);
+            os_provider->nodeset_len = 0;
+            return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        }
+
+        size_t num = 0;
+        int ret = utils_get_complete_nodeset(nodes, MAX_NUMNODES, &num);
+        if (ret < 0) {
+            hwloc_bitmap_free(out_nodeset[0]);
+            umf_ba_global_free(out_nodeset);
+            umf_ba_global_free(nodes);
+            os_provider->nodeset_len = 0;
+            return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        }
+
+        for (size_t i = 0; i < num; i++) {
+            hwloc_bitmap_set(out_nodeset[0], (int)nodes[i]);
+        }
+        umf_ba_global_free(nodes);
+#endif
+
         if (!out_nodeset[0]) {
             goto err_free_list;
         }
@@ -518,6 +557,12 @@ translate_params(const umf_os_memory_provider_params_t *in_params,
 
     provider->numa_flags =
         getHwlocMembindFlags(in_params->numa_mode, is_dedicated_node_bind);
+
+#ifndef _WIN32
+    provider->umf_numa_mode = in_params->numa_mode;
+    provider->dedicated = is_dedicated_node_bind;
+#endif
+
     provider->mode = in_params->numa_mode;
     provider->part_size = in_params->part_size;
 
@@ -561,6 +606,11 @@ static umf_result_t os_initialize(const void *params, void **provider) {
     snprintf(os_provider->name, sizeof(os_provider->name), "%s",
              in_params->name);
 
+#ifdef _WIN32
+
+    //struct timespec ts_init_start, ts_init_end;
+    //clock_gettime(CLOCK_MONOTONIC, &ts_init_start);
+
     int r = hwloc_topology_init(&os_provider->topo);
     if (r) {
         LOG_ERR("HWLOC topology init failed");
@@ -577,6 +627,13 @@ static umf_result_t os_initialize(const void *params, void **provider) {
         goto err_destroy_hwloc_topology;
     }
 
+    //clock_gettime(CLOCK_MONOTONIC, &ts_init_end);
+    //LOG_FATAL("HWLOC topology initialized in %ld.%09ld seconds",
+    //          ts_init_end.tv_sec - ts_init_start.tv_sec,
+    //          ts_init_end.tv_nsec - ts_init_start.tv_nsec);
+
+#endif // _WIN32
+
     os_provider->fd_offset_map = critnib_new(NULL, NULL);
     if (!os_provider->fd_offset_map) {
         LOG_ERR("creating file descriptor offset map failed");
@@ -625,8 +682,10 @@ static umf_result_t os_initialize(const void *params, void **provider) {
 err_destroy_critnib:
     critnib_delete(os_provider->fd_offset_map);
 err_destroy_hwloc_topology:
+#ifdef _WIN32
     hwloc_topology_destroy(os_provider->topo);
 err_free_os_provider:
+#endif
     umf_ba_global_free(os_provider);
     return ret;
 }
@@ -649,7 +708,9 @@ static umf_result_t os_finalize(void *provider) {
     if (os_provider->nodeset_str_buf) {
         umf_ba_global_free(os_provider->nodeset_str_buf);
     }
+#ifdef _WIN32
     hwloc_topology_destroy(os_provider->topo);
+#endif
     umf_ba_global_free(os_provider);
     return UMF_RESULT_SUCCESS;
 }
@@ -1012,10 +1073,52 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment,
 
         do {
             errno = 0;
+            ret = 0;
+
+#ifdef _WIN32
             ret = hwloc_set_area_membind(os_provider->topo, membind.addr,
                                          membind.bind_size, membind.bitmap,
                                          os_provider->numa_policy,
                                          os_provider->numa_flags);
+#else  // !_WIN32
+
+            // NOTE: could we done this
+
+            // on Linux, use mbind syscall directly instead of hwloc
+            unsigned long nodemask = 0;
+            int maxnode = 8 * sizeof(nodemask); // up to 64 nodes
+            if (membind.bitmap) {
+                for (int i = 0; i < maxnode; ++i) {
+                    if (hwloc_bitmap_isset(membind.bitmap, i)) {
+                        nodemask |= (1UL << i);
+                    }
+                }
+            }
+
+            int mbind_mode = MPOL_DEFAULT;
+            if (os_provider->umf_numa_mode == UMF_NUMA_MODE_INTERLEAVE &&
+                os_provider->dedicated == 0) {
+                mbind_mode = MPOL_INTERLEAVE;
+            } else if (os_provider->umf_numa_mode == UMF_NUMA_MODE_SPLIT) {
+                mbind_mode = MPOL_BIND;
+            } else if (os_provider->umf_numa_mode == UMF_NUMA_MODE_LOCAL) {
+                mbind_mode = MPOL_LOCAL;
+                nodemask = 0;
+            } else if (os_provider->umf_numa_mode == UMF_NUMA_MODE_PREFERRED) {
+                mbind_mode = MPOL_BIND;
+            } else if (os_provider->umf_numa_mode == UMF_NUMA_MODE_BIND ||
+                       os_provider->dedicated) {
+                mbind_mode = MPOL_BIND;
+            }
+
+            unsigned long mbind_flags = 0;
+            if (os_provider->dedicated) {
+                mbind_flags |= MPOL_MF_STRICT;
+            }
+
+            ret = syscall(__NR_mbind, membind.addr, membind.bind_size,
+                          mbind_mode, &nodemask, maxnode, mbind_flags);
+#endif // !_WIN32
 
             if (ret) {
                 os_store_last_native_error(UMF_OS_RESULT_ERROR_BIND_FAILED,

src/provider/provider_os_memory_internal.h

@@ -68,7 +68,15 @@ typedef struct os_memory_provider_t {
     unsigned partitions_len;
     size_t partitions_weight_sum;
 
+#ifndef _WIN32
+    // NOTE: on linux we don't want to use hwloc_topology_t directly because
+    // of its long initialization time
+    umf_numa_mode_t umf_numa_mode;
+    int dedicated;
+
+#else
     hwloc_topology_t topo;
+#endif
 
     char name[64];
 

src/utils/utils_common.h

@@ -188,6 +188,8 @@ size_t utils_max(size_t a, size_t b);
 
 size_t utils_min(size_t a, size_t b);
 
+int utils_get_complete_nodeset(size_t *ids, size_t size, size_t *);
+
 #ifdef __cplusplus
 }
 #endif

src/utils/utils_linux_common.c

@@ -1,15 +1,18 @@
 /*
  *
- * Copyright (C) 2023-2024 Intel Corporation
+ * Copyright (C) 2023-2025 Intel Corporation
  *
  * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  */
 
+#include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
@@ -240,3 +243,25 @@ int utils_create_anonymous_fd(void) {
 
     return fd;
 }
+
+int utils_get_complete_nodeset(size_t *nodes, size_t nodes_size, size_t *num) {
+    DIR *dir = opendir("/sys/devices/system/node/");
+    if (!dir) {
+        return -1;
+    }
+
+    struct dirent *entry;
+    while ((entry = readdir(dir)) != NULL) {
+        if (strncmp(entry->d_name, "node", 4) == 0) {
+            char *endptr;
+            long node_id = strtol(entry->d_name + 4, &endptr, 10);
+            if (*endptr == '\0' && node_id >= 0 && *num < nodes_size) {
+                nodes[*num] = (size_t)node_id;
+                (*num)++;
+            }
+        }
+    }
+
+    closedir(dir);
+    return 0;
+}