diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt index 6c7014940407d..21717807d4244 100644 --- a/libc/src/__support/OSUtil/linux/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt @@ -53,3 +53,4 @@ add_object_library( libc.src.errno.errno libc.src.sys.auxv.getauxval ) + diff --git a/libc/src/__support/OSUtil/linux/aarch64/vdso.h b/libc/src/__support/OSUtil/linux/aarch64/vdso.h index 3c4c6205071da..ee5777ad67f6d 100644 --- a/libc/src/__support/OSUtil/linux/aarch64/vdso.h +++ b/libc/src/__support/OSUtil/linux/aarch64/vdso.h @@ -23,6 +23,8 @@ LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { return "__kernel_clock_gettime"; case VDSOSym::ClockGetRes: return "__kernel_clock_getres"; + case VDSOSym::GetRandom: + return "__kernel_getrandom"; default: return ""; } diff --git a/libc/src/__support/OSUtil/linux/vdso_sym.h b/libc/src/__support/OSUtil/linux/vdso_sym.h index 968e1536c4d27..2b1cf398369de 100644 --- a/libc/src/__support/OSUtil/linux/vdso_sym.h +++ b/libc/src/__support/OSUtil/linux/vdso_sym.h @@ -19,7 +19,6 @@ struct __kernel_timespec; struct timezone; struct riscv_hwprobe; struct getcpu_cache; -struct cpu_set_t; // NOLINTEND(llvmlibc-implementation-in-namespace) namespace LIBC_NAMESPACE_DECL { @@ -35,7 +34,8 @@ enum class VDSOSym { RTSigReturn, FlushICache, RiscvHwProbe, - VDSOSymCount + GetRandom, + VDSOSymCount, }; template LIBC_INLINE constexpr auto dispatcher() { @@ -58,8 +58,11 @@ template LIBC_INLINE constexpr auto dispatcher() { else if constexpr (sym == VDSOSym::FlushICache) return static_cast(nullptr); else if constexpr (sym == VDSOSym::RiscvHwProbe) - return static_cast(nullptr); + else if constexpr (sym == VDSOSym::GetRandom) + return static_cast( + nullptr); else return static_cast(nullptr); } diff --git a/libc/src/__support/OSUtil/linux/x86_64/vdso.h b/libc/src/__support/OSUtil/linux/x86_64/vdso.h index abe7c33e07cfa..f46fcb038f2e6 100644 --- a/libc/src/__support/OSUtil/linux/x86_64/vdso.h +++ b/libc/src/__support/OSUtil/linux/x86_64/vdso.h @@ -29,6 +29,8 @@ LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { return "__vdso_time"; case VDSOSym::ClockGetRes: return "__vdso_clock_getres"; + case VDSOSym::GetRandom: + return "__vdso_getrandom"; default: return ""; } diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp index dad4f75f092ed..12f227a6cd975 100644 --- a/libc/src/__support/threads/thread.cpp +++ b/libc/src/__support/threads/thread.cpp @@ -117,7 +117,7 @@ class ThreadAtExitCallbackMgr { int add_callback(AtExitCallback *callback, void *obj) { cpp::lock_guard lock(mtx); - return callback_list.push_back({callback, obj}); + return callback_list.push_back({callback, obj}) ? 0 : -1; } void call() { @@ -161,6 +161,10 @@ void call_atexit_callbacks(ThreadAttributes *attrib) { } } +bool add_atexit_callback(void (*callback)(void *), void *obj) { + return atexit_callback_mgr.add_callback(callback, obj) == 0; +} + } // namespace internal cpp::optional new_tss_key(TSSDtor *dtor) { diff --git a/libc/src/__support/threads/thread.h b/libc/src/__support/threads/thread.h index f2b1f6bbb253d..dc1854d088060 100644 --- a/libc/src/__support/threads/thread.h +++ b/libc/src/__support/threads/thread.h @@ -246,6 +246,9 @@ namespace internal { // returned by this function. ThreadAtExitCallbackMgr *get_thread_atexit_callback_mgr(); +// Add internal atexit callbacks. +bool add_atexit_callback(void (*callback)(void *), void *obj); + // Call the currently registered thread specific atexit callbacks. Useful for // implementing the thread_exit function. void call_atexit_callbacks(ThreadAttributes *attrib); diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt index 1d3c00a5e0ddb..9dff17f746a99 100644 --- a/libc/src/stdlib/linux/CMakeLists.txt +++ b/libc/src/stdlib/linux/CMakeLists.txt @@ -9,3 +9,21 @@ add_entrypoint_object( libc.src.signal.raise libc.src.stdlib._Exit ) + +add_object_library( + cprng + HDRS + cprng.h + SRCS + cprng.cpp + DEPENDS + libc.src.__support.common + libc.src.__support.OSUtil.linux.vdso + libc.src.__support.threads.callonce + libc.src.__support.threads.linux.raw_mutex + libc.src.__support.CPP.optional + libc.src.__support.CPP.new + libc.src.__support.threads.thread + libc.src.sys.auxv.getauxval + libc.src.stdlib.atexit +) diff --git a/libc/src/stdlib/linux/cprng.cpp b/libc/src/stdlib/linux/cprng.cpp new file mode 100644 index 0000000000000..a02eb17d871ec --- /dev/null +++ b/libc/src/stdlib/linux/cprng.cpp @@ -0,0 +1,350 @@ +//===- Linux implementation of secure random buffer generation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "src/stdlib/linux/cprng.h" +#include "src/__support/CPP/atomic.h" +#include "src/__support/CPP/mutex.h" +#include "src/__support/CPP/new.h" +#include "src/__support/OSUtil/linux/syscall.h" +#include "src/__support/OSUtil/linux/vdso.h" +#include "src/__support/block.h" +#include "src/__support/libc_assert.h" +#include "src/__support/threads/callonce.h" +#include "src/__support/threads/linux/raw_mutex.h" +#include "src/__support/threads/thread.h" +#include "src/stdlib/atexit.h" +#include "src/sys/auxv/getauxval.h" + +namespace LIBC_NAMESPACE_DECL { +namespace cprng { +namespace { + +using namespace vdso; +// A block of random state together with enough space to hold all its freelist. +// ┌───────────┬────────────┐ +// │ │ │ +// │ │ Pages ├──────────────┐ +// │ │ │ ▼ +// │ Prev ├────────────┤ ┌──────────────────┐ +// │ │ │ │ │ +// │ │ State0 ├───►│ Opaque Area │ +// │ │ │ │ │ +// │ ├────────────┤ ├──────────────────┤ +// ├───────────┤ │ │ │ +// │ │ State1 ├───►│ Opaque Area │ +// │ │ │ │ │ +// │ ├────────────┤ ├──────────────────┤ +// │ Next │ │ │ │ +// │ │ State2 ├───►│ Opaque Area │ +// │ │ │ │ │ +// │ ├────────────┤ ├──────────────────┤ +// │ │ ...... │ │ .............. │ +// └───────────┴────────────┘ └──────────────────┘ +// State blocks are doubly linked so that we can iterate bidrectionally as a +// freelist. We'will use a sentinel to simplify the implementation. +struct StateBlock { + StateBlock *prev; + StateBlock *next; + void *appendix[0]; + void *&pages() { return appendix[0]; } + void *&freelist(size_t index) { return appendix[index + 1]; } +}; + +// Parameters from Linux UAPI. Available only for 6.11+. We manually provide it +// to support a mean of compiling libc on old kernels. +struct vgetrandom_opaque_params { + unsigned size_of_opaque_state = 0; + unsigned mmap_prot = 0; + unsigned mmap_flags = 0; + unsigned reserved[13]; +}; + +// Global configuration for state allocation. +// - System Page Size +// - Number of pages per block (allocation) +// - Number of states per page +// - Parameters for state allocation +// An opaque state cannot go across page boundaries. +// We use the number of processors available to the current process to estimate +// the anticipated number of states. +class GlobalConfig { +public: + const size_t page_size = 0; + const size_t pages_per_block = 0; + const size_t states_per_page = 0; + const vgetrandom_opaque_params params = {}; + +private: + static size_t guess_cpu_count() { + unsigned char cpuset[128]{}; + if (syscall_impl(0, sizeof(cpuset), cpuset) <= 0) + return 1u; + size_t count = 0; + for (auto byte : cpuset) + count += cpp::popcount(byte); + return count > 1u ? count : 1u; + } + +private: + constexpr GlobalConfig(size_t page_size, size_t pages_per_block, + size_t states_per_page, + vgetrandom_opaque_params params) + : page_size(page_size), pages_per_block(pages_per_block), + states_per_page(states_per_page), params(params) {} + +public: + static cpp::optional get() { + size_t page_size = 0; + size_t states_per_page = 0; + size_t pages_per_block = 0; + vgetrandom_opaque_params params = {}; + + // check symbol availability + TypedSymbol vgetrandom; + if (!vgetrandom) + return cpp::nullopt; + + // get valid page size + long page_size_res = getauxval(AT_PAGESZ); + if (page_size_res <= 0) + return cpp::nullopt; + page_size = static_cast(page_size_res); + + // get parameters for state allocation + if (vgetrandom(nullptr, 0, 0, ¶ms, ~0U)) + return cpp::nullopt; + + // Compute the number of states per page. On a valid human-constructable + // computer as year 2024, the following operations shall not overflow given + // the above operations are correctly returned. + size_t guessed_bytes = guess_cpu_count() * params.size_of_opaque_state; + size_t aligned_bytes = align_up(guessed_bytes, page_size); + states_per_page = page_size / params.size_of_opaque_state; + pages_per_block = aligned_bytes / page_size; + return GlobalConfig(page_size, pages_per_block, states_per_page, params); + } +}; + +// A monotonic state pool. +class MonotonicStatePool { + RawMutex mutex; + GlobalConfig config; + StateBlock sentinel; + StateBlock *free_cursor; + // invariant: at sentinel, free_count is always locally full such that we + // don't need to specially check the condition for sentinel when recycling. + size_t free_count; + + constexpr MonotonicStatePool(GlobalConfig config) + : mutex(), config(config), sentinel{&sentinel, &sentinel, {}}, + free_cursor(&sentinel), + free_count(config.pages_per_block * config.states_per_page) {} + MonotonicStatePool(const MonotonicStatePool &) = delete; + MonotonicStatePool(MonotonicStatePool &&) = delete; + + bool is_empty() const { return free_cursor == &sentinel; } + bool is_locally_full() const { + return free_count == config.pages_per_block * config.states_per_page; + } + size_t state_block_raw_size() const { + return sizeof(StateBlock) + + (config.states_per_page * config.pages_per_block + 1) * + sizeof(void *); + } + bool allocate_new_block() { + LIBC_ASSERT(is_empty()); + // allocate a new block + AllocChecker ac{}; + size_t raw_size = state_block_raw_size(); + // NOLINTNEXTLINE(llvmlibc-callee-namespace) + auto *block = static_cast(operator new(raw_size, ac)); + if (!ac) + return false; + // allocate associated pages + auto pages = syscall_impl( + SYS_mmap, nullptr, config.pages_per_block * config.page_size, + config.params.mmap_prot, config.params.mmap_flags, -1, 0); + + if (pages <= 0) { + // NOLINTNEXTLINE(llvmlibc-callee-namespace) + ::operator delete(block, raw_size); + return false; + } + // populate the block + block->pages() = reinterpret_cast(pages); + size_t state_idx = 0; + for (size_t p = 0; p < config.pages_per_block; ++p) { + char *page = reinterpret_cast(pages) + p * config.page_size; + for (size_t s = 0; s < config.states_per_page; ++s) { + block->freelist(state_idx++) = + page + s * config.params.size_of_opaque_state; + } + } + // link the block to the sentinel + block->next = sentinel.next; + block->prev = &sentinel; + block->next->prev = block; + block->prev->next = block; + // update the cursor + free_cursor = block; + free_count = state_idx; + // release the ownership of allocation + return true; + } + +public: + void *get() { + cpp::lock_guard guard{this->mutex}; + // if freelist is empty, allocate a new block + if (is_empty() && !allocate_new_block()) + return nullptr; + LIBC_ASSERT(free_count != 0); + void *page = free_cursor->freelist(--free_count); + // move cursor to the previous block if free_count is exhausted + if (free_count == 0) { + free_cursor = free_cursor->prev; + free_count = config.states_per_page * config.pages_per_block; + } + return page; + } + size_t state_size() const { return config.params.size_of_opaque_state; } + void recycle(void *state) { + cpp::lock_guard guard{this->mutex}; + if (is_locally_full()) { + free_cursor = free_cursor->next; + free_count = 0; + } + free_cursor->freelist(free_count++) = static_cast(state); + } + ~MonotonicStatePool() { + StateBlock *block = sentinel.next; + size_t raw_size = state_block_raw_size(); + while (block != &sentinel) { + [[maybe_unused]] + auto res = syscall_impl(SYS_munmap, block->pages(), + config.pages_per_block * config.page_size); + LIBC_ASSERT(res == 0); + StateBlock *next = block->next; + // NOLINTNEXTLINE(llvmlibc-callee-namespace) + ::operator delete(block, raw_size); + block = next; + } + } + + static MonotonicStatePool *instance(); +}; + +alignas(MonotonicStatePool) static char pool[sizeof(MonotonicStatePool)]; +static bool is_valid = false; +static CallOnceFlag once_flag = callonce_impl::NOT_CALLED; +MonotonicStatePool *MonotonicStatePool::instance() { + callonce(&once_flag, []() { + cpp::optional config = GlobalConfig::get(); + if (!config) + return; + new (pool) MonotonicStatePool(*config); + is_valid = !atexit([]() { + reinterpret_cast(pool)->~MonotonicStatePool(); + }); + }); + if (!is_valid) + return nullptr; + return reinterpret_cast(pool); +} + +// We do not guarantee the correctness of calling the cprng functions in signal +// frames. However, we do want to make sure that an mistaken (maliciously +// induced) call to the cprng will never cause a state to be used twice due to +// reentrancy. +class ThreadLocalState { + void *local_state; + bool in_use; + size_t remaining_trials; + + void destroy() { + in_use = true; + cpp::atomic_signal_fence(cpp::MemoryOrder::SEQ_CST); + if (local_state) + MonotonicStatePool::instance()->recycle(local_state); + } + +public: + constexpr ThreadLocalState() + : local_state(nullptr), in_use(false), remaining_trials(256) {} + void *acquire() { + if (in_use) + return nullptr; + in_use = true; + cpp::atomic_signal_fence(cpp::MemoryOrder::SEQ_CST); + if (local_state) + return local_state; + + MonotonicStatePool *pool = MonotonicStatePool::instance(); + if (pool && remaining_trials) { + remaining_trials--; + local_state = pool->get(); + if (local_state) { + if (!internal::add_atexit_callback( + [](void *opaque) { + auto *state = static_cast(opaque); + state->destroy(); + }, + this)) { + pool->recycle(local_state); + local_state = nullptr; + release(); + } + } + } + return local_state; + } + void release() { + in_use = false; + cpp::atomic_signal_fence(cpp::MemoryOrder::SEQ_CST); + } +}; +thread_local ThreadLocalState local_state{}; + +template +size_t fill_buffer_impl(G generator, char *buffer, size_t length) { + size_t filled = 0; + while (filled < length) { + int n = generator(&buffer[filled], length - filled); + if (n < 0) { + if (n == -EAGAIN || n == -EINTR) + continue; + break; + } + filled += static_cast(n); + } + return filled; +} + +} // namespace +size_t fill_buffer(char *buffer, size_t length) { + void *state = local_state.acquire(); + if (state) { + // Given state is valid, state_size shall be valid and the vDSO symbol shall + // be defined. + size_t state_size = MonotonicStatePool::instance()->state_size(); + auto impl = [state, state_size](char *cursor, size_t len) { + TypedSymbol vgetrandom; + return vgetrandom(cursor, len, 0, state, state_size); + }; + local_state.release(); + return fill_buffer_impl(impl, buffer, length); + } else { + auto impl = [](char *cursor, size_t len) { + // use syscall to avoid errno handling + return syscall_impl(SYS_getrandom, cursor, len, 0); + }; + local_state.release(); + return fill_buffer_impl(impl, buffer, length); + } +} +} // namespace cprng +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdlib/linux/cprng.h b/libc/src/stdlib/linux/cprng.h new file mode 100644 index 0000000000000..c67c8381f577a --- /dev/null +++ b/libc/src/stdlib/linux/cprng.h @@ -0,0 +1,68 @@ +//===-- Utilities for getting secure randomness -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_LINUX_CPRNG_H +#define LLVM_LIBC_SRC_STDLIB_LINUX_CPRNG_H + +#include "src/__support/CPP/optional.h" +#include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" + +#define __need_size_t +#include + +// CPRNG provides LLVM-libc with a cryptographically secure random number. +// arc4random is available in BSD systems and later introduced in glibc. +// However, arc4random family faces performance issues on pre 6.11 Linux +// as each entropy generation would demand a round trip to and from the kernel. +// We still use getrandom syscall as fallback for older kernels. For modern +// kernels, we use the vDSO interface to get random bytes. +// Such vDSO interface differs from the getrandom syscall in that it demands +// the userspace to maintain an opaque state. System library (e.g. libc) is in +// charge of allocating a large bunch of such states and lend them to each +// thread demanding random bytes. +// Our approach should be providing similar guarantees as arc4random family on +// BSD and bionic. It is not async-signal-safe, but we have means to prevent +// information leakage due to reentrancy in signal frames. The vDSO getrandom +// implementation itself provides security across forks (by wiping pages). +namespace LIBC_NAMESPACE_DECL { +namespace cprng { +size_t fill_buffer(char *buffer, size_t length); + +template LIBC_INLINE cpp::optional generate() { + static_assert(cpp::is_arithmetic_v, "T must be an arithmetic type."); + union { + T result; + char bytes[sizeof(T)]; + }; + if (fill_buffer(bytes, sizeof(bytes)) == sizeof(bytes)) + return result; + return cpp::nullopt; +} + +// based on https://jacquesheunis.com/post/bounded-random/ +LIBC_INLINE cpp::optional generate_bounded_u32(uint32_t bound) { + auto lifted_bound = static_cast(bound); + cpp::optional seed = generate(); + if (!seed) + return cpp::nullopt; + uint64_t r0 = *seed & 0xFFFFFFFFu; + uint64_t r1 = (*seed >> 32) & 0xFFFFFFFFu; + uint64_t prod0 = static_cast(r0) * lifted_bound; + uint64_t prod0_hi = (prod0 >> 32) & 0xFFFFFFFFu; + uint64_t prod0_lo = prod0 & 0xFFFFFFFFu; + uint64_t prod1 = static_cast(r1) * lifted_bound; + uint64_t prod1_hi = (prod1 >> 32) & 0xFFFFFFFFu; + uint64_t sum = prod0_lo + prod1_hi; + uint64_t sum_hi = (sum >> 32) & 0xFFFFFFFFu; + return static_cast(prod0_hi + sum_hi); +} + +} // namespace cprng +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC_STDLIB_LINUX_CPRNG_H diff --git a/libc/test/integration/src/stdlib/CMakeLists.txt b/libc/test/integration/src/stdlib/CMakeLists.txt index 1efdf607defe9..26eba5c1071f5 100644 --- a/libc/test/integration/src/stdlib/CMakeLists.txt +++ b/libc/test/integration/src/stdlib/CMakeLists.txt @@ -13,3 +13,7 @@ add_integration_test( FRANCE=Paris GERMANY=Berlin ) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) + add_subdirectory(${LIBC_TARGET_OS}) +endif() diff --git a/libc/test/integration/src/stdlib/linux/CMakeLists.txt b/libc/test/integration/src/stdlib/linux/CMakeLists.txt new file mode 100644 index 0000000000000..66ec786d3b501 --- /dev/null +++ b/libc/test/integration/src/stdlib/linux/CMakeLists.txt @@ -0,0 +1,11 @@ +add_integration_test( + cprng_test + SUITE + stdlib-integration-tests + SRCS + cprng_test.cpp + DEPENDS + libc.src.stdlib.linux.cprng + libc.src.pthread.pthread_create + libc.src.pthread.pthread_join +) diff --git a/libc/test/integration/src/stdlib/linux/cprng_test.cpp b/libc/test/integration/src/stdlib/linux/cprng_test.cpp new file mode 100644 index 0000000000000..7b84dbad2685b --- /dev/null +++ b/libc/test/integration/src/stdlib/linux/cprng_test.cpp @@ -0,0 +1,50 @@ +//===-- Unittests for CPRNG -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "src/pthread/pthread_create.h" +#include "src/pthread/pthread_join.h" +#include "src/stdlib/linux/cprng.h" +#include "test/IntegrationTest/test.h" + +namespace LIBC_NAMESPACE_DECL { +namespace cprng { +void smoke_test() { + auto result = generate(); + ASSERT_TRUE(result.has_value()); +} +void bounded_test() { + for (uint32_t bound = 1; bound < 5000; ++bound) { + auto result = generate_bounded_u32(bound); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(*result < bound); + } +} +void threaded_bounded_test() { + pthread_t threads[10]; + for (auto &thread : threads) { + ASSERT_EQ(LIBC_NAMESPACE::pthread_create( + &thread, nullptr, + [](void *) -> void * { + bounded_test(); + return nullptr; + }, + nullptr), + 0); + } + for (auto &thread : threads) { + ASSERT_EQ(LIBC_NAMESPACE::pthread_join(thread, nullptr), 0); + } +} +} // namespace cprng +} // namespace LIBC_NAMESPACE_DECL + +TEST_MAIN() { + LIBC_NAMESPACE::cprng::smoke_test(); + LIBC_NAMESPACE::cprng::bounded_test(); + LIBC_NAMESPACE::cprng::threaded_bounded_test(); + return 0; +}