|
10 | 10 | #define LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
|
11 | 11 |
|
12 | 12 | #include "src/__support/CPP/bit.h"
|
| 13 | +#include "src/__support/CPP/mutex.h" |
13 | 14 | #include "src/__support/OSUtil/linux/vdso.h"
|
14 | 15 | #include "src/__support/OSUtil/syscall.h"
|
15 | 16 | #include "src/__support/blockstore.h"
|
@@ -40,66 +41,88 @@ class GlobalState {
|
40 | 41 | VGetrandomOpaqueParams params;
|
41 | 42 | };
|
42 | 43 |
|
| 44 | + // A lock-free stack of free opaque states. |
43 | 45 | MPMCStack<void *> free_list{};
|
44 |
| - RawMutex page_queue_mutex{}; |
| 46 | + // A mutex protecting the allocation of new pages. |
| 47 | + RawMutex allocation_mutex{}; |
| 48 | + // A block store of allocated pages. |
45 | 49 | BlockStore<void *, 16> allocations{};
|
| 50 | + |
| 51 | + // Shared global configuration. |
46 | 52 | static CallOnceFlag config_flag;
|
47 | 53 | static Config config;
|
48 | 54 |
|
49 |
| - LIBC_INLINE static size_t guess_cpu_count() { |
50 |
| - char cpu_set[128] = {0}; |
51 |
| - int res = LIBC_NAMESPACE::syscall_impl<int>(SYS_sched_getaffinity, 0, |
52 |
| - sizeof(cpu_set), cpu_set); |
53 |
| - if (res <= 0) |
54 |
| - return 1; |
55 |
| - |
56 |
| - size_t count = 0; |
57 |
| - for (size_t i = 0; i < sizeof(cpu_set) / sizeof(unsigned long); ++i) { |
58 |
| - unsigned long *mask_ptr = reinterpret_cast<unsigned long *>(cpu_set); |
59 |
| - count += LIBC_NAMESPACE::cpp::popcount(mask_ptr[i]); |
60 |
| - } |
| 55 | + // We grow the states by the number of CPUs. This function uses |
| 56 | + // SYS_sched_getaffinity to get the number of CPUs. |
| 57 | + LIBC_INLINE static size_t cpu_count(); |
61 | 58 |
|
62 |
| - return count > 0 ? count : 1; |
63 |
| - } |
| 59 | + // Grow available states. This function can fail if the system is out of |
| 60 | + // memory. |
| 61 | + LIBC_INLINE bool grow(); |
64 | 62 |
|
65 | 63 | public:
|
66 | 64 | LIBC_INLINE constexpr GlobalState() {}
|
67 |
| - |
68 |
| - LIBC_INLINE static Config &get_config() { |
69 |
| - callonce(&config_flag, []() { |
70 |
| - config.getrandom = |
71 |
| - LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{}; |
72 |
| - if (!config.getrandom) |
73 |
| - return; |
74 |
| - |
75 |
| - // Call with special flag to get the desired configuration. |
76 |
| - int res = config.getrandom( |
77 |
| - /*buf=*/nullptr, /*count=*/0, /*flags=*/0, |
78 |
| - /*opaque_states=*/&config.params, |
79 |
| - /*size_of_opaque_states=*/~0); |
80 |
| - if (res != 0) |
81 |
| - return; |
82 |
| - |
83 |
| - config.page_size = LIBC_NAMESPACE::getauxval(AT_PAGESZ); |
84 |
| - if (!config.page_size) |
85 |
| - return; |
86 |
| - |
87 |
| - size_t cpu_count = guess_cpu_count(); |
88 |
| - |
89 |
| - config.states_per_page = |
90 |
| - config.page_size / config.params.size_of_opaque_states; |
91 |
| - |
92 |
| - config.pages_per_alloc = cpu_count / config.states_per_page + |
93 |
| - (cpu_count % config.states_per_page != 0); |
94 |
| - }); |
95 |
| - return config; |
96 |
| - } |
| 65 | + LIBC_INLINE static Config &get_config(); |
| 66 | + LIBC_INLINE ~GlobalState() {} |
97 | 67 | };
|
| 68 | + |
98 | 69 | class LocalState {};
|
99 | 70 |
|
100 | 71 | LIBC_INLINE_VAR GlobalState::Config GlobalState::config{};
|
101 | 72 | LIBC_INLINE_VAR CallOnceFlag GlobalState::config_flag = 0;
|
102 | 73 |
|
| 74 | +LIBC_INLINE size_t GlobalState::cpu_count() { |
| 75 | + char cpu_set[128] = {0}; |
| 76 | + int res = LIBC_NAMESPACE::syscall_impl<int>(SYS_sched_getaffinity, 0, |
| 77 | + sizeof(cpu_set), cpu_set); |
| 78 | + if (res <= 0) |
| 79 | + return 1; |
| 80 | + |
| 81 | + size_t count = 0; |
| 82 | + for (size_t i = 0; i < sizeof(cpu_set) / sizeof(unsigned long); ++i) { |
| 83 | + unsigned long *mask_ptr = reinterpret_cast<unsigned long *>(cpu_set); |
| 84 | + count += LIBC_NAMESPACE::cpp::popcount(mask_ptr[i]); |
| 85 | + } |
| 86 | + |
| 87 | + return count > 0 ? count : 1; |
| 88 | +} |
| 89 | + |
| 90 | +LIBC_INLINE GlobalState::Config &GlobalState::get_config() { |
| 91 | + callonce(&config_flag, []() { |
| 92 | + config.getrandom = |
| 93 | + LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{}; |
| 94 | + if (!config.getrandom) |
| 95 | + return; |
| 96 | + |
| 97 | + // Call with special flag to get the desired configuration. |
| 98 | + int res = config.getrandom( |
| 99 | + /*buf=*/nullptr, /*count=*/0, /*flags=*/0, |
| 100 | + /*opaque_states=*/&config.params, |
| 101 | + /*size_of_opaque_states=*/~0); |
| 102 | + if (res != 0) |
| 103 | + return; |
| 104 | + |
| 105 | + config.page_size = LIBC_NAMESPACE::getauxval(AT_PAGESZ); |
| 106 | + if (!config.page_size) |
| 107 | + return; |
| 108 | + |
| 109 | + size_t count = cpu_count(); |
| 110 | + |
| 111 | + config.states_per_page = |
| 112 | + config.page_size / config.params.size_of_opaque_states; |
| 113 | + |
| 114 | + config.pages_per_alloc = |
| 115 | + count / config.states_per_page + (count % config.states_per_page != 0); |
| 116 | + }); |
| 117 | + return config; |
| 118 | +} |
| 119 | + |
| 120 | +LIBC_INLINE bool GlobalState::grow() { |
| 121 | + // reserve a slot for the new page. |
| 122 | + if (!allocations.push_back(nullptr)) |
| 123 | + return false; |
| 124 | +} |
| 125 | + |
103 | 126 | } // namespace vsdo_rng
|
104 | 127 | } // namespace LIBC_NAMESPACE_DECL
|
105 | 128 |
|
|
0 commit comments