From f7ae58cc5704270d9faa9133a4dc805336e56740 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Fri, 26 Sep 2025 14:06:56 +0000 Subject: [PATCH 1/4] [libunwind] Call `__arm_za_disable` before entering EH This is done by defining ` __arm_za_disable` as a weak symbol with the assumption that if libunwind is being linked against SME-aware code, the ABI routines will be provided (by libgcc or compiler-rt). --- libunwind/src/UnwindLevel1.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libunwind/src/UnwindLevel1.c b/libunwind/src/UnwindLevel1.c index b0cd60dfb9141..69c274c504ffc 100644 --- a/libunwind/src/UnwindLevel1.c +++ b/libunwind/src/UnwindLevel1.c @@ -202,6 +202,10 @@ unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *except } extern int __unw_step_stage2(unw_cursor_t *); +#if defined(__aarch64__) +extern void __attribute__((weak)) __arm_za_disable(void); +#endif + #if defined(_LIBUNWIND_USE_GCS) // Enable the GCS target feature to permit gcspop instructions to be used. __attribute__((target("+gcs"))) @@ -214,6 +218,29 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_obj=%p)", (void *)exception_object); +#if defined(__aarch64__) + // The platform must ensure that all the following conditions are true on + // entry to EH: + // + // - PSTATE.SM is 0. + // - PSTATE.ZA is 0. + // - TPIDR2_EL0 is null. + // + // The first point is ensured by routines for throwing exceptions having a + // non-streaming interface. TPIDR2_EL0 is set to null and ZA disabled by + // calling __arm_za_disable. + // + // See: + // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#exceptions + if (__arm_za_disable) { + // FIXME: Is SME is available and `__arm_za_disable` is not, this should + // abort. + __arm_za_disable(); + } else { + _LIBUNWIND_DEBUG_LOG("failed to call __arm_za_disable in %s", __FUNCTION__); + } +#endif + // uc is initialized by __unw_getcontext in the parent frame. The first stack // frame walked is unwind_phase2. unsigned framesWalked = 1; From 3984605aacff1a5daa9fd6761237df211f0d395e Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Fri, 26 Sep 2025 15:48:51 +0000 Subject: [PATCH 2/4] Fixups --- libunwind/src/UnwindLevel1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libunwind/src/UnwindLevel1.c b/libunwind/src/UnwindLevel1.c index 69c274c504ffc..ac733453beeb1 100644 --- a/libunwind/src/UnwindLevel1.c +++ b/libunwind/src/UnwindLevel1.c @@ -233,10 +233,10 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, // See: // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#exceptions if (__arm_za_disable) { - // FIXME: Is SME is available and `__arm_za_disable` is not, this should - // abort. __arm_za_disable(); } else { + // FIXME: If SME is available and `__arm_za_disable` is not, this should + // abort. _LIBUNWIND_DEBUG_LOG("failed to call __arm_za_disable in %s", __FUNCTION__); } #endif From 8d54ded7a89a3fbe0c7bd47709dfb4ee1fa22c55 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Wed, 29 Oct 2025 17:50:25 +0000 Subject: [PATCH 3/4] Rework and add test --- libunwind/src/Registers.hpp | 32 +++++- libunwind/src/UnwindLevel1.c | 27 ----- libunwind/test/aarch64_za_unwind.pass.cpp | 117 ++++++++++++++++++++++ 3 files changed, 148 insertions(+), 28 deletions(-) create mode 100644 libunwind/test/aarch64_za_unwind.pass.cpp diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp index 5a5b57835379a..0a9065144be08 100644 --- a/libunwind/src/Registers.hpp +++ b/libunwind/src/Registers.hpp @@ -1829,6 +1829,10 @@ inline const char *Registers_ppc64::getRegisterName(int regNum) { class _LIBUNWIND_HIDDEN Registers_arm64; extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *); +#if !defined(__APPLE__) +extern "C" void __attribute__((weak)) __arm_za_disable(); +#endif + #if defined(_LIBUNWIND_USE_GCS) extern "C" void *__libunwind_shstk_get_jump_target() { return reinterpret_cast(&__libunwind_Registers_arm64_jumpto); @@ -1855,7 +1859,7 @@ class _LIBUNWIND_HIDDEN Registers_arm64 { v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); static const char *getRegisterName(int num); - void jumpto() { __libunwind_Registers_arm64_jumpto(this); } + void jumpto(); static constexpr int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64; } @@ -1971,6 +1975,32 @@ Registers_arm64::operator=(const Registers_arm64 &other) { return *this; } +void Registers_arm64::jumpto() { +#if !defined(__APPLE__) + // The platform must ensure that all the following conditions are true on + // entry to EH: + // + // - PSTATE.SM is 0. + // - PSTATE.ZA is 0. + // - TPIDR2_EL0 is null. + // + // The first point is ensured by routines for throwing exceptions having a + // non-streaming interface. TPIDR2_EL0 is set to null and ZA disabled by + // calling __arm_za_disable. + // + // See: + // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#exceptions + if (__arm_za_disable) { + __arm_za_disable(); + } else { + // FIXME: If SME is available and `__arm_za_disable` is not, this should + // abort. + _LIBUNWIND_DEBUG_LOG("failed to call __arm_za_disable in %s", __FUNCTION__); + } +#endif + __libunwind_Registers_arm64_jumpto(this); +} + inline Registers_arm64::Registers_arm64() { memset(static_cast(this), 0, sizeof(*this)); } diff --git a/libunwind/src/UnwindLevel1.c b/libunwind/src/UnwindLevel1.c index ac733453beeb1..b0cd60dfb9141 100644 --- a/libunwind/src/UnwindLevel1.c +++ b/libunwind/src/UnwindLevel1.c @@ -202,10 +202,6 @@ unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *except } extern int __unw_step_stage2(unw_cursor_t *); -#if defined(__aarch64__) -extern void __attribute__((weak)) __arm_za_disable(void); -#endif - #if defined(_LIBUNWIND_USE_GCS) // Enable the GCS target feature to permit gcspop instructions to be used. __attribute__((target("+gcs"))) @@ -218,29 +214,6 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_obj=%p)", (void *)exception_object); -#if defined(__aarch64__) - // The platform must ensure that all the following conditions are true on - // entry to EH: - // - // - PSTATE.SM is 0. - // - PSTATE.ZA is 0. - // - TPIDR2_EL0 is null. - // - // The first point is ensured by routines for throwing exceptions having a - // non-streaming interface. TPIDR2_EL0 is set to null and ZA disabled by - // calling __arm_za_disable. - // - // See: - // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#exceptions - if (__arm_za_disable) { - __arm_za_disable(); - } else { - // FIXME: If SME is available and `__arm_za_disable` is not, this should - // abort. - _LIBUNWIND_DEBUG_LOG("failed to call __arm_za_disable in %s", __FUNCTION__); - } -#endif - // uc is initialized by __unw_getcontext in the parent frame. The first stack // frame walked is unwind_phase2. unsigned framesWalked = 1; diff --git a/libunwind/test/aarch64_za_unwind.pass.cpp b/libunwind/test/aarch64_za_unwind.pass.cpp new file mode 100644 index 0000000000000..2985bb8d298de --- /dev/null +++ b/libunwind/test/aarch64_za_unwind.pass.cpp @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// REQUIRES: linux && target={{aarch64-.+}} + +#include +#include +#include +#include +#include +#include + +// Basic test of unwinding with SME lazy saves. This tests libunwind disables ZA +// (and commits a lazy save of ZA) before resuming from unwinding. + +// Note: This test requires SME (and is setup to pass on targets without SME). + +static bool checkHasSME() { + constexpr int hwcap2_sme = (1 << 23); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + return (hwcap2 & hwcap2_sme) != 0; +} + +struct TPIDR2Block { + void *za_save_buffer; + uint64_t num_save_slices; +}; + +__attribute__((noinline)) void private_za() { + // Note: Lazy save active on entry to function. + unw_context_t context; + unw_cursor_t cursor; + + unw_getcontext(&context); + unw_init_local(&cursor, &context); + unw_step(&cursor); + unw_resume(&cursor); +} + +bool isZAOn() { + register uint64_t svcr asm("x20"); + asm(".inst 0xd53b4254" : "=r"(svcr)); + return (svcr & 0b10) != 0; +} + +__attribute__((noinline)) void za_function_with_lazy_save() { + register uint64_t tmp asm("x8"); + + // SMSTART ZA (should zero ZA) + asm(".inst 0xd503457f"); + + // RDSVL x8, #1 (read streaming vector length) + asm(".inst 0x04bf5828" : "=r"(tmp)); + + // Allocate and fill ZA save buffer with 0xAA. + size_t buffer_size = tmp * tmp; + uint8_t *za_save_buffer = (uint8_t *)alloca(buffer_size); + memset(za_save_buffer, 0xAA, buffer_size); + + TPIDR2Block block = {za_save_buffer, tmp}; + tmp = reinterpret_cast(&block); + + // MRS TPIDR2_EL0, x8 (setup lazy save of ZA) + asm(".inst 0xd51bd0a8" ::"r"(tmp)); + + // ZA should be on before unwinding. + if (!isZAOn()) { + fprintf(stderr, __FILE__ ": fail (ZA not on before call)\n"); + abort(); + } else { + fprintf(stderr, __FILE__ ": pass (ZA on before call)\n"); + } + + private_za(); + + // ZA should be off after unwinding. + if (isZAOn()) { + fprintf(stderr, __FILE__ ": fail (ZA on after unwinding)\n"); + abort(); + } else { + fprintf(stderr, __FILE__ ": pass (ZA off after unwinding)\n"); + } + + // MRS x8, TPIDR2_EL0 (read TPIDR2_EL0) + asm(".inst 0xd53bd0a8" : "=r"(tmp)); + // ZA should have been saved (TPIDR2_EL0 zero). + if (tmp != 0) { + fprintf(stderr, __FILE__ ": fail (TPIDR2_EL0 non-null after unwinding)\n"); + abort(); + } else { + fprintf(stderr, __FILE__ ": pass (TPIDR2_EL0 null after unwinding)\n"); + } + + // ZA (all zero) should have been saved to the buffer. + for (unsigned i = 0; i < buffer_size; ++i) { + if (za_save_buffer[i] != 0) { + fprintf(stderr, + __FILE__ ": fail (za_save_buffer non-zero after unwinding)\n"); + abort(); + } + } + fprintf(stderr, __FILE__ ": pass (za_save_buffer zero'd after unwinding)\n"); +} + +int main(int, char **) { + if (!checkHasSME()) { + fprintf(stderr, __FILE__ ": pass (no SME support)\n"); + return 0; // Pass (SME is required for this test to run). + } + za_function_with_lazy_save(); + return 0; +} From 74fe5289307f25670b144cee49aa07d0ecb016d2 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Thu, 30 Oct 2025 11:25:31 +0000 Subject: [PATCH 4/4] Fixups --- libunwind/src/Registers.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp index 0a9065144be08..332e073688f15 100644 --- a/libunwind/src/Registers.hpp +++ b/libunwind/src/Registers.hpp @@ -1829,7 +1829,9 @@ inline const char *Registers_ppc64::getRegisterName(int regNum) { class _LIBUNWIND_HIDDEN Registers_arm64; extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *); -#if !defined(__APPLE__) +#if defined(__APPLE__) +extern "C" void __attribute__((weak_import)) __arm_za_disable(); +#else extern "C" void __attribute__((weak)) __arm_za_disable(); #endif @@ -1858,8 +1860,8 @@ class _LIBUNWIND_HIDDEN Registers_arm64 { bool validVectorRegister(int num) const; v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); + void jumpto(); static const char *getRegisterName(int num); - void jumpto(); static constexpr int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64; } @@ -1975,8 +1977,7 @@ Registers_arm64::operator=(const Registers_arm64 &other) { return *this; } -void Registers_arm64::jumpto() { -#if !defined(__APPLE__) +inline void Registers_arm64::jumpto() { // The platform must ensure that all the following conditions are true on // entry to EH: // @@ -1997,7 +1998,6 @@ void Registers_arm64::jumpto() { // abort. _LIBUNWIND_DEBUG_LOG("failed to call __arm_za_disable in %s", __FUNCTION__); } -#endif __libunwind_Registers_arm64_jumpto(this); }