From f04208de159de107b138bb75e2133e8a47739466 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 24 Jul 2025 14:59:47 -0700 Subject: [PATCH 01/13] xtensa: fatal: use named macros for PS register bits... ...instead of magic numbers when printing exception information. Signed-off-by: Daniel Leung --- arch/xtensa/core/vector_handlers.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/xtensa/core/vector_handlers.c b/arch/xtensa/core/vector_handlers.c index d15d779933a03..9cb23e1e33b7a 100644 --- a/arch/xtensa/core/vector_handlers.c +++ b/arch/xtensa/core/vector_handlers.c @@ -259,10 +259,13 @@ static void print_fatal_exception(void *print_stack, int cause, EXCEPTION_DUMP(" ** PS %p", (void *)bsa->ps); EXCEPTION_DUMP(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", - get_bits(0, 4, ps), get_bits(4, 1, ps), - get_bits(5, 1, ps), get_bits(6, 2, ps), - get_bits(18, 1, ps), - get_bits(8, 4, ps), get_bits(16, 2, ps)); + get_bits(XCHAL_PS_INTLEVEL_SHIFT, XCHAL_PS_INTLEVEL_BITS, ps), + get_bits(XCHAL_PS_EXCM_SHIFT, XCHAL_PS_EXCM_BITS, ps), + get_bits(XCHAL_PS_UM_SHIFT, XCHAL_PS_UM_BITS, ps), + get_bits(XCHAL_PS_RING_SHIFT, XCHAL_PS_RING_BITS, ps), + get_bits(XCHAL_PS_WOE_SHIFT, XCHAL_PS_WOE_BITS, ps), + get_bits(XCHAL_PS_OWB_SHIFT, XCHAL_PS_OWB_BITS, ps), + get_bits(XCHAL_PS_CALLINC_SHIFT, XCHAL_PS_CALLINC_BITS, ps)); } static ALWAYS_INLINE void usage_stop(void) From 55274d21a1d74ed3f23d574551b141a6b6b90b89 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 12 Aug 2025 15:09:59 -0700 Subject: [PATCH 02/13] xtensa: mmu: properly restore PTE attributes via reset_region() The software bits inside PTE are used to store original PTE attributes and ring value, and those bits are used to restore PTE to previous state. This modifies reset_region() to properly restore attributes and ring value when resetting memory regions to the same as when the system boots. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 160 ++++++++++++++++++-------- arch/xtensa/include/xtensa_mmu_priv.h | 60 ++++++++-- 2 files changed, 162 insertions(+), 58 deletions(-) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index 605dbe2453723..0f001866c9419 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -22,6 +22,14 @@ */ #define OPTION_NO_TLB_IPI BIT(0) +/* Restore the PTE attributes if they have been + * stored in the SW bits part in the PTE. + */ +#define OPTION_RESTORE_ATTRS BIT(1) + +/* Save the PTE attributes and ring in the SW bits part in the PTE. */ +#define OPTION_SAVE_ATTRS BIT(2) + /* Level 1 contains page table entries * necessary to map the page table itself. */ @@ -157,6 +165,8 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { }, }; +static inline uint32_t restore_pte(uint32_t pte); + /** * @brief Check if the page table entry is illegal. * @@ -174,18 +184,19 @@ static inline bool is_pte_illegal(uint32_t pte) return (attr == 12) || (attr == 14); } -/* - * @brief Initialize all page table entries to be illegal. +/** + * @brief Initialize all page table entries to the same value (@a val). * - * @param[in] Pointer to page table. - * @param[in] Number of page table entries in the page table. + * @param[in] ptable Pointer to page table. + * @param[in] num_entries Number of page table entries in the page table. + * @param[in] val Initialize all PTEs with this value. */ -static void init_page_table(uint32_t *ptable, size_t num_entries) +static void init_page_table(uint32_t *ptable, size_t num_entries, uint32_t val) { int i; for (i = 0; i < num_entries; i++) { - ptable[i] = XTENSA_MMU_PTE_ILLEGAL; + ptable[i] = val; } } @@ -203,18 +214,20 @@ static inline uint32_t *alloc_l2_table(void) } static void map_memory_range(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, const uint32_t options) { uint32_t page, *table; bool shared = !!(attrs & XTENSA_MMU_MAP_SHARED); - uint32_t sw_attrs = (attrs & XTENSA_MMU_PTE_ATTR_ORIGINAL) == XTENSA_MMU_PTE_ATTR_ORIGINAL ? - attrs : 0; + bool do_save_attrs = (options & OPTION_SAVE_ATTRS) == OPTION_SAVE_ATTRS; + uint32_t ring, sw_attrs, sw_ring, pte_sw; + + ring = shared ? XTENSA_MMU_SHARED_RING : XTENSA_MMU_KERNEL_RING; + sw_attrs = do_save_attrs ? attrs : XTENSA_MMU_PTE_ATTR_ILLEGAL; + sw_ring = do_save_attrs ? ring : XTENSA_MMU_KERNEL_RING; + pte_sw = XTENSA_MMU_PTE_SW(sw_ring, sw_attrs); for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { - uint32_t pte = XTENSA_MMU_PTE(page, - shared ? XTENSA_MMU_SHARED_RING : - XTENSA_MMU_KERNEL_RING, - sw_attrs, attrs); + uint32_t pte = XTENSA_MMU_PTE(page, ring, pte_sw, attrs); uint32_t l2_pos = XTENSA_MMU_L2_POS(page); uint32_t l1_pos = XTENSA_MMU_L1_POS(page); @@ -224,11 +237,12 @@ static void map_memory_range(const uint32_t start, const uint32_t end, __ASSERT(table != NULL, "There is no l2 page table available to " "map 0x%08x\n", page); - init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES, + XTENSA_MMU_PTE_L2_ILLEGAL); xtensa_kernel_ptables[l1_pos] = XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING, - sw_attrs, XTENSA_MMU_PAGE_TABLE_ATTR); + 0, XTENSA_MMU_PAGE_TABLE_ATTR); } table = (uint32_t *)(xtensa_kernel_ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); @@ -237,26 +251,28 @@ static void map_memory_range(const uint32_t start, const uint32_t end, } static void map_memory(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, const uint32_t options) { #ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP uint32_t uc_attrs = attrs & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK; uint32_t c_attrs = attrs | XTENSA_MMU_CACHED_WB; if (sys_cache_is_ptr_uncached((void *)start)) { - map_memory_range(start, end, uc_attrs); + map_memory_range(start, end, uc_attrs, options); map_memory_range(POINTER_TO_UINT(sys_cache_cached_ptr_get((void *)start)), - POINTER_TO_UINT(sys_cache_cached_ptr_get((void *)end)), c_attrs); + POINTER_TO_UINT(sys_cache_cached_ptr_get((void *)end)), + c_attrs, options); } else if (sys_cache_is_ptr_cached((void *)start)) { - map_memory_range(start, end, c_attrs); + map_memory_range(start, end, c_attrs, options); map_memory_range(POINTER_TO_UINT(sys_cache_uncached_ptr_get((void *)start)), - POINTER_TO_UINT(sys_cache_uncached_ptr_get((void *)end)), uc_attrs); + POINTER_TO_UINT(sys_cache_uncached_ptr_get((void *)end)), + uc_attrs, options); } else #endif { - map_memory_range(start, end, attrs); + map_memory_range(start, end, attrs, options); } } @@ -270,19 +286,20 @@ static void xtensa_init_page_tables(void) } already_inited = true; - init_page_table(xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES); + init_page_table(xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES, + XTENSA_MMU_PTE_L1_ILLEGAL); atomic_set_bit(l1_page_table_track, 0); for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; - map_memory(range->start, range->end, range->attrs | XTENSA_MMU_PTE_ATTR_ORIGINAL); + map_memory(range->start, range->end, range->attrs, OPTION_SAVE_ATTRS); } for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) { const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry]; - map_memory(range->start, range->end, range->attrs | XTENSA_MMU_PTE_ATTR_ORIGINAL); + map_memory(range->start, range->end, range->attrs, OPTION_SAVE_ATTRS); } /* Finally, the direct-mapped pages used in the page tables @@ -292,10 +309,10 @@ static void xtensa_init_page_tables(void) */ map_memory_range((uint32_t) &l1_page_table[0], (uint32_t) &l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES], - XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W); + XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W, OPTION_SAVE_ATTRS); map_memory_range((uint32_t) &l2_page_tables[0], (uint32_t) &l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES], - XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W); + XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W, OPTION_SAVE_ATTRS); sys_cache_data_flush_all(); } @@ -375,7 +392,7 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, return false; } - init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES, XTENSA_MMU_PTE_L2_ILLEGAL); l1_table[l1_pos] = XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING, 0, XTENSA_MMU_PAGE_TABLE_ATTR); @@ -386,7 +403,9 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); table[l2_pos] = XTENSA_MMU_PTE(phys, is_user ? XTENSA_MMU_USER_RING : XTENSA_MMU_KERNEL_RING, - 0, flags); + XTENSA_MMU_PTE_SW(XTENSA_MMU_KERNEL_RING, + XTENSA_MMU_PTE_ATTR_ILLEGAL), + flags); sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); xtensa_tlb_autorefill_invalidate(); @@ -549,17 +568,24 @@ static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); - l2_table[l2_pos] = XTENSA_MMU_PTE_ILLEGAL; + /* Restore the PTE to previous ring and attributes. */ + l2_table[l2_pos] = restore_pte(l2_table[l2_pos]); sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) { if (!is_pte_illegal(l2_table[l2_pos])) { + /* If any PTE is mapped (== not illegal), we need to + * keep this L2 table. + */ goto end; } } - l1_table[l1_pos] = XTENSA_MMU_PTE_ILLEGAL; + /* All L2 PTE are illegal (== nothing mapped), we can safely remove + * the L2 table mapping in L1 table and return the L2 table to the pool. + */ + l1_table[l1_pos] = XTENSA_MMU_PTE_L1_ILLEGAL; sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); @@ -719,6 +745,37 @@ void xtensa_mmu_tlb_shootdown(void) arch_irq_unlock(key); } +/** + * @brief Restore PTE ring and attributes from those stashed in SW bits. + * + * @param[in] pte Page table entry to be restored. + * + * @note This does not check if the SW bits contain ring and attributes to be + * restored. + * + * @return PTE with restored ring and attributes. Illegal entry if original is + * illegal. + */ +static inline uint32_t restore_pte(uint32_t pte) +{ + uint32_t restored_pte; + + uint32_t original_sw = XTENSA_MMU_PTE_SW_GET(pte); + uint32_t original_attr = XTENSA_MMU_PTE_SW_ATTR_GET(original_sw); + + if (original_attr != XTENSA_MMU_PTE_ATTR_ILLEGAL) { + uint8_t original_ring = XTENSA_MMU_PTE_SW_RING_GET(original_sw); + + restored_pte = pte; + restored_pte = XTENSA_MMU_PTE_ATTR_SET(restored_pte, original_attr); + restored_pte = XTENSA_MMU_PTE_RING_SET(restored_pte, original_ring); + } else { + restored_pte = XTENSA_MMU_PTE_L2_ILLEGAL; + } + + return restored_pte; +} + #ifdef CONFIG_USERSPACE static inline uint32_t *thread_page_tables_get(const struct k_thread *thread) @@ -757,7 +814,7 @@ static uint32_t *dup_table(void) if (is_pte_illegal(xtensa_kernel_ptables[i]) || (i == XTENSA_MMU_L1_POS(XTENSA_MMU_PTEVADDR))) { - dst_table[i] = XTENSA_MMU_PTE_ILLEGAL; + dst_table[i] = XTENSA_MMU_PTE_L1_ILLEGAL; continue; } @@ -768,18 +825,7 @@ static uint32_t *dup_table(void) } for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) { - uint32_t original_attr = XTENSA_MMU_PTE_SW_GET(src_l2_table[j]); - - l2_table[j] = src_l2_table[j]; - if (original_attr != 0x0) { - uint8_t ring; - - ring = XTENSA_MMU_PTE_RING_GET(l2_table[j]); - l2_table[j] = XTENSA_MMU_PTE_ATTR_SET(l2_table[j], original_attr); - l2_table[j] = XTENSA_MMU_PTE_RING_SET(l2_table[j], - ring == XTENSA_MMU_SHARED_RING ? - XTENSA_MMU_SHARED_RING : XTENSA_MMU_KERNEL_RING); - } + l2_table[j] = restore_pte(src_l2_table[j]); } /* The page table is using kernel ASID because we don't @@ -848,10 +894,11 @@ int arch_mem_domain_init(struct k_mem_domain *domain) } static void region_map_update(uint32_t *ptables, uintptr_t start, - size_t size, uint32_t ring, uint32_t flags) + size_t size, uint32_t ring, uint32_t flags, uint32_t option) { for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { uint32_t *l2_table, pte; + uint32_t new_ring, new_attrs; uint32_t page = start + offset; uint32_t l1_pos = XTENSA_MMU_L1_POS(page); uint32_t l2_pos = XTENSA_MMU_L2_POS(page); @@ -862,8 +909,20 @@ static void region_map_update(uint32_t *ptables, uintptr_t start, sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); - pte = XTENSA_MMU_PTE_RING_SET(l2_table[l2_pos], ring); - pte = XTENSA_MMU_PTE_ATTR_SET(pte, flags); + pte = l2_table[l2_pos]; + + if ((option & OPTION_RESTORE_ATTRS) == OPTION_RESTORE_ATTRS) { + uint32_t original_sw = XTENSA_MMU_PTE_SW_GET(pte); + + new_attrs = XTENSA_MMU_PTE_SW_ATTR_GET(original_sw); + new_ring = XTENSA_MMU_PTE_SW_RING_GET(original_sw); + } else { + new_attrs = flags; + new_ring = ring; + } + + pte = XTENSA_MMU_PTE_RING_SET(pte, new_ring); + pte = XTENSA_MMU_PTE_ATTR_SET(pte, new_attrs); l2_table[l2_pos] = pte; @@ -895,10 +954,10 @@ static void update_region(uint32_t *ptables, uintptr_t start, size_t size, new_flags_uc = (flags & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK); new_flags = new_flags_uc | XTENSA_MMU_CACHED_WB; - region_map_update(ptables, va, size, ring, new_flags); - region_map_update(ptables, va_uc, size, ring, new_flags_uc); + region_map_update(ptables, va, size, ring, new_flags, option); + region_map_update(ptables, va_uc, size, ring, new_flags_uc, option); #else - region_map_update(ptables, start, size, ring, flags); + region_map_update(ptables, start, size, ring, flags, option); #endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ #if CONFIG_MP_MAX_NUM_CPUS > 1 @@ -914,7 +973,8 @@ static void update_region(uint32_t *ptables, uintptr_t start, size_t size, static inline void reset_region(uint32_t *ptables, uintptr_t start, size_t size, uint32_t option) { update_region(ptables, start, size, - XTENSA_MMU_KERNEL_RING, XTENSA_MMU_PERM_W, option); + XTENSA_MMU_KERNEL_RING, XTENSA_MMU_PERM_W, + option | OPTION_RESTORE_ATTRS); } void xtensa_user_stack_perms(struct k_thread *thread) diff --git a/arch/xtensa/include/xtensa_mmu_priv.h b/arch/xtensa/include/xtensa_mmu_priv.h index 7cd51f1329361..cf8e4c0aca780 100644 --- a/arch/xtensa/include/xtensa_mmu_priv.h +++ b/arch/xtensa/include/xtensa_mmu_priv.h @@ -52,17 +52,38 @@ #define XTENSA_MMU_PTE_RING_SHIFT 4U /** Number of bits to shift for SW reserved ared in PTE */ -#define XTENSA_MMU_PTE_SW_SHIFT 6U +#define XTENSA_MMU_PTE_SW_SHIFT 6U /** Mask for SW bits in PTE */ -#define XTENSA_MMU_PTE_SW_MASK 0x00000FC0U +#define XTENSA_MMU_PTE_SW_MASK 0x00000FC0U /** - * Internal bit just used to indicate that the attr field must - * be set in the SW bits too. It is used later when duplicating the - * kernel page tables. + * Number of bits to shift for backup attributes in PTE SW field. + * + * This is relative to the SW field, not the PTE entry. + */ +#define XTENSA_MMU_PTE_SW_ATTR_SHIFT 0U + +/** + * Mask for backup attributes in PTE SW field. + * + * This is relative to the SW field, not the PTE entry. + */ +#define XTENSA_MMU_PTE_SW_ATTR_MASK 0x0000000FU + +/** + * Number of bits to shift for backup ring value in PTE SW field. + * + * This is relative to the SW field, not the PTE entry. */ -#define XTENSA_MMU_PTE_ATTR_ORIGINAL BIT(31) +#define XTENSA_MMU_PTE_SW_RING_SHIFT 4U + +/** + * Mask for backup ring value in PTE SW field. + * + * This is relative to the SW field, not the PTE entry. + */ +#define XTENSA_MMU_PTE_SW_RING_MASK 0x00000030U /** Construct a page table entry (PTE) */ #define XTENSA_MMU_PTE(paddr, ring, sw, attr) \ @@ -87,6 +108,19 @@ #define XTENSA_MMU_PTE_SW_GET(pte) \ (((pte) & XTENSA_MMU_PTE_SW_MASK) >> XTENSA_MMU_PTE_SW_SHIFT) +/** Construct a PTE SW field to be used for backing up PTE ring and attributes. */ +#define XTENSA_MMU_PTE_SW(ring, attr) \ + ((((ring) << XTENSA_MMU_PTE_SW_RING_SHIFT) & XTENSA_MMU_PTE_SW_RING_MASK) | \ + (((attr) << XTENSA_MMU_PTE_SW_ATTR_SHIFT) & XTENSA_MMU_PTE_SW_ATTR_MASK)) + +/** Get the backed up attributes from the PTE SW field. */ +#define XTENSA_MMU_PTE_SW_ATTR_GET(sw) \ + (((sw) & XTENSA_MMU_PTE_SW_ATTR_MASK) >> XTENSA_MMU_PTE_SW_ATTR_SHIFT) + +/** Get the backed up ring value from the PTE SW field. */ +#define XTENSA_MMU_PTE_SW_RING_GET(sw) \ + (((sw) & XTENSA_MMU_PTE_SW_RING_MASK) >> XTENSA_MMU_PTE_SW_RING_SHIFT) + /** Set the ring in a PTE */ #define XTENSA_MMU_PTE_RING_SET(pte, ring) \ (((pte) & ~XTENSA_MMU_PTE_RING_MASK) | \ @@ -148,8 +182,18 @@ /** Number of auto-refill ways */ #define XTENSA_MMU_NUM_TLB_AUTOREFILL_WAYS 4 -/** Indicate PTE is illegal. */ -#define XTENSA_MMU_PTE_ILLEGAL (BIT(3) | BIT(2)) +/** Attribute indicating PTE is illegal. */ +#define XTENSA_MMU_PTE_ATTR_ILLEGAL (BIT(3) | BIT(2)) + +/** Illegal PTE entry for Level 1 page tables */ +#define XTENSA_MMU_PTE_L1_ILLEGAL XTENSA_MMU_PTE_ATTR_ILLEGAL + +/** Illegal PTE entry for Level 2 page tables */ +#define XTENSA_MMU_PTE_L2_ILLEGAL \ + XTENSA_MMU_PTE(0, XTENSA_MMU_KERNEL_RING, \ + XTENSA_MMU_PTE_SW(XTENSA_MMU_KERNEL_RING, \ + XTENSA_MMU_PTE_ATTR_ILLEGAL), \ + XTENSA_MMU_PTE_ATTR_ILLEGAL) /** * PITLB HIT bit. From 7a94c847e476d36423b635db8b04e9f262826c04 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 30 Jul 2025 09:21:54 -0700 Subject: [PATCH 03/13] xtensa: save EXCCAUSE/EXCVADDR in BSA during exception This saves the EXCCAUSE and EXCVADDR into BSA during exception. These will then be used during exception handling. The reason for doing this instead of reading from both registers during exception handing is that another exception will override the value in these register (e.g. DTLB miss). When returning to the previous exception handler, these register no longer contains the original exception cause and address. We need to save it so that we are actually handling the orignal exception. Coredump also now uses the EXCCAUSE and EXCVADDR from BSA instead of reading the registers. Signed-off-by: Daniel Leung --- arch/xtensa/core/coredump.c | 5 ++--- arch/xtensa/core/gen_zsr.py | 2 +- arch/xtensa/core/offsets/offsets.c | 1 + arch/xtensa/core/vector_handlers.c | 27 ++++++++++------------- arch/xtensa/include/xtensa_asm2_context.h | 3 ++- arch/xtensa/include/xtensa_asm2_s.h | 20 +++++++++++------ 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/arch/xtensa/core/coredump.c b/arch/xtensa/core/coredump.c index 761cf9a05ac13..14942586ad6fb 100644 --- a/arch/xtensa/core/coredump.c +++ b/arch/xtensa/core/coredump.c @@ -135,8 +135,6 @@ void arch_coredump_info_dump(const struct arch_esf *esf) xtensa_coredump_fault_sp = (uint32_t)esf; #endif - __asm__ volatile("rsr.exccause %0" : "=r"(arch_blk.r.exccause)); - _xtensa_irq_stack_frame_raw_t *frame = (void *)esf; _xtensa_irq_bsa_t *bsa = frame->ptr_to_bsa; uintptr_t num_high_regs; @@ -150,7 +148,7 @@ void arch_coredump_info_dump(const struct arch_esf *esf) regs_blk_remaining = (int)num_high_regs / 4; arch_blk.r.pc = bsa->pc; - __asm__ volatile("rsr.excvaddr %0" : "=r"(arch_blk.r.excvaddr)); + arch_blk.r.excvaddr = bsa->excvaddr; arch_blk.r.ps = bsa->ps; #if XCHAL_HAVE_S32C1I arch_blk.r.scompare1 = bsa->scompare1; @@ -160,6 +158,7 @@ void arch_coredump_info_dump(const struct arch_esf *esf) arch_blk.r.a1 = (uint32_t)((char *)bsa) + sizeof(*bsa); arch_blk.r.a2 = bsa->a2; arch_blk.r.a3 = bsa->a3; + arch_blk.r.exccause = bsa->exccause; if (regs_blk_remaining > 0) { regs_blk_remaining--; diff --git a/arch/xtensa/core/gen_zsr.py b/arch/xtensa/core/gen_zsr.py index 8cb8a713019e3..a50e205e84cd0 100755 --- a/arch/xtensa/core/gen_zsr.py +++ b/arch/xtensa/core/gen_zsr.py @@ -31,7 +31,7 @@ def parse_args(): NEEDED = ["A0SAVE", "CPU"] if args.mmu: - NEEDED += ["DBLEXC", "DEPC_SAVE", "EXCCAUSE_SAVE"] + NEEDED += ["DBLEXC", "DEPC_SAVE"] if args.flush_reg: NEEDED += ["FLUSH"] diff --git a/arch/xtensa/core/offsets/offsets.c b/arch/xtensa/core/offsets/offsets.c index 9edf341b7a7b4..3a1d04636576c 100644 --- a/arch/xtensa/core/offsets/offsets.c +++ b/arch/xtensa/core/offsets/offsets.c @@ -22,6 +22,7 @@ GEN_OFFSET_SYM(_xtensa_irq_bsa_t, a2); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, a3); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, exccause); +GEN_OFFSET_SYM(_xtensa_irq_bsa_t, excvaddr); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, pc); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, ps); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, sar); diff --git a/arch/xtensa/core/vector_handlers.c b/arch/xtensa/core/vector_handlers.c index 9cb23e1e33b7a..149e0543e5e14 100644 --- a/arch/xtensa/core/vector_handlers.c +++ b/arch/xtensa/core/vector_handlers.c @@ -220,38 +220,35 @@ static inline unsigned int get_bits(int offset, int num_bits, unsigned int val) return val & mask; } -static void print_fatal_exception(void *print_stack, int cause, - bool is_dblexc, uint32_t depc) +static void print_fatal_exception(void *print_stack, bool is_dblexc, uint32_t depc) { void *pc; - uint32_t ps, vaddr; + uint32_t ps; _xtensa_irq_bsa_t *bsa = (void *)*(int **)print_stack; - __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); - if (is_dblexc) { EXCEPTION_DUMP(" ** FATAL EXCEPTION (DOUBLE)"); } else { EXCEPTION_DUMP(" ** FATAL EXCEPTION"); } - EXCEPTION_DUMP(" ** CPU %d EXCCAUSE %d (%s)", - arch_curr_cpu()->id, cause, - xtensa_exccause(cause)); + EXCEPTION_DUMP(" ** CPU %d EXCCAUSE %u (%s)", + arch_curr_cpu()->id, (uint32_t)bsa->exccause, + xtensa_exccause(bsa->exccause)); /* Don't print information if the BSA area is invalid as any elements * obtained via de-referencing the pointer are probably also invalid. * Or worse, cause another access violation. */ if (xtensa_is_outside_stack_bounds((uintptr_t)bsa, sizeof(*bsa), UINT32_MAX)) { - EXCEPTION_DUMP(" ** VADDR %p Invalid SP %p", (void *)vaddr, print_stack); + EXCEPTION_DUMP(" ** VADDR %p Invalid SP %p", (void *)bsa->excvaddr, print_stack); return; } ps = bsa->ps; pc = (void *)bsa->pc; - EXCEPTION_DUMP(" ** PC %p VADDR %p", pc, (void *)vaddr); + EXCEPTION_DUMP(" ** PC %p VADDR %p", pc, (void *)bsa->excvaddr); if (is_dblexc) { EXCEPTION_DUMP(" ** DEPC %p", (void *)depc); @@ -547,13 +544,12 @@ void *xtensa_excint1_c(void *esf) #ifdef CONFIG_XTENSA_MMU depc = XTENSA_RSR(ZSR_DEPC_SAVE_STR); - cause = XTENSA_RSR(ZSR_EXCCAUSE_SAVE_STR); is_dblexc = (depc != 0U); -#else /* CONFIG_XTENSA_MMU */ - __asm__ volatile("rsr.exccause %0" : "=r"(cause)); #endif /* CONFIG_XTENSA_MMU */ + cause = bsa->exccause; + switch (cause) { case EXCCAUSE_LEVEL1_INTERRUPT: #ifdef CONFIG_XTENSA_MMU @@ -664,7 +660,6 @@ void *xtensa_excint1_c(void *esf) if (cause == EXCCAUSE_ILLEGAL) { if (pc == (void *)&xtensa_arch_except_epc) { cause = 63; - __asm__ volatile("wsr.exccause %0" : : "r"(cause)); reason = bsa->a2; } else if (pc == (void *)&xtensa_arch_kernel_oops_epc) { cause = 64; /* kernel oops */ @@ -677,11 +672,13 @@ void *xtensa_excint1_c(void *esf) */ print_stack = (void *)bsa->a3; } + + bsa->exccause = cause; } skip_checks: if (reason != K_ERR_KERNEL_OOPS) { - print_fatal_exception(print_stack, cause, is_dblexc, depc); + print_fatal_exception(print_stack, is_dblexc, depc); } #ifdef CONFIG_XTENSA_EXCEPTION_ENTER_GDB extern void z_gdb_isr(struct arch_esf *esf); diff --git a/arch/xtensa/include/xtensa_asm2_context.h b/arch/xtensa/include/xtensa_asm2_context.h index 56fc84b56acbd..6b669e3804687 100644 --- a/arch/xtensa/include/xtensa_asm2_context.h +++ b/arch/xtensa/include/xtensa_asm2_context.h @@ -116,7 +116,7 @@ #endif /* Must have fields regardless of features. */ -#define _BSA_PADDING_COMMON (sizeof(uintptr_t) * 12U) +#define _BSA_PADDING_COMMON (sizeof(uintptr_t) * 13U) /* Raw size by adding up all the above. */ #define _BSA_PADDING_BASE_SIZE \ @@ -194,6 +194,7 @@ struct xtensa_irq_base_save_area { #endif uintptr_t exccause; + uintptr_t excvaddr; #if XCHAL_HAVE_LOOPS uintptr_t lcount; diff --git a/arch/xtensa/include/xtensa_asm2_s.h b/arch/xtensa/include/xtensa_asm2_s.h index af345ab308360..d82375696a685 100644 --- a/arch/xtensa/include/xtensa_asm2_s.h +++ b/arch/xtensa/include/xtensa_asm2_s.h @@ -663,7 +663,7 @@ _Level\LVL\()Vector: * For double exception, DEPC in saved in earlier vector * code. */ - wsr a0, ZSR_EXCCAUSE_SAVE + wsr a0, ZSR_A0SAVE esync @@ -690,15 +690,11 @@ _Level\LVL\()Vector: * jump to an infinite loop, or quit the simulator, or invoke * debugger. */ - rsr a0, ZSR_EXCCAUSE_SAVE + rsr a0, ZSR_A0SAVE j _TripleFault _not_triple_fault: - rsr.exccause a0 - - xsr a0, ZSR_EXCCAUSE_SAVE - - esync + rsr a0, ZSR_A0SAVE .endif #endif @@ -707,6 +703,16 @@ _Level\LVL\()Vector: s32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET s32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET + /* Save registers needed for handling the exception as + * these registers can be overwritten during nested + * exceptions. + */ + rsr.exccause a0 + s32i a0, a1, ___xtensa_irq_bsa_t_exccause_OFFSET + + rsr.excvaddr a0 + s32i a0, a1, ___xtensa_irq_bsa_t_excvaddr_OFFSET + /* Level "1" is the exception handler, which uses a different * calling convention. No special register holds the * interrupted PS, instead we just assume that the CPU has From 9ed581de0758ee740b0c78bb21a70c30b9711611 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 29 Jul 2025 14:27:44 -0700 Subject: [PATCH 04/13] xtensa: userspace: handle load/store ring exception When a page can be accessed by both kernel and user threads, the autofill DTLB may contain an entry for kernel thread. This will result in load/store ring exception when it is accessed by user thread later. In this case, we need to invalidate all associated TLBs related to kernel access so hardware can reload the page table the correct permission for user thread. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 48 +++++++++++++++++++++++++++ arch/xtensa/core/vector_handlers.c | 7 ++++ arch/xtensa/include/xtensa_internal.h | 16 +++++++++ arch/xtensa/include/xtensa_mmu_priv.h | 9 +++++ 4 files changed, 80 insertions(+) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index 0f001866c9419..ed9563b7413e6 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -1172,6 +1173,53 @@ int arch_buffer_validate(const void *addr, size_t size, int write) return mem_buffer_validate(addr, size, write, XTENSA_MMU_USER_RING); } +bool xtensa_exc_load_store_ring_error_check(void *bsa_p) +{ + uintptr_t ring, vaddr; + _xtensa_irq_bsa_t *bsa = (_xtensa_irq_bsa_t *)bsa_p; + + ring = (bsa->ps & XCHAL_PS_RING_MASK) >> XCHAL_PS_RING_SHIFT; + + if (ring != XTENSA_MMU_USER_RING) { + return true; + } + + vaddr = bsa->excvaddr; + + if (arch_buffer_validate((void *)vaddr, sizeof(uint32_t), false) != 0) { + /* User thread DO NOT have access to this memory according to + * page table. so this is a true access violation. + */ + return true; + } + + /* User thread has access to this memory according to + * page table. so this is not a true access violation. + * + * Now we need to find all associated auto-refilled DTLBs + * and invalidate them. So that hardware can reload + * from page table with correct permission for user + * thread. + */ + while (true) { + uint32_t dtlb_entry = xtensa_dtlb_probe((void *)vaddr); + + if ((dtlb_entry & XTENSA_MMU_PDTLB_HIT) != XTENSA_MMU_PDTLB_HIT) { + /* No more DTLB entry found. */ + return false; + } + + if ((dtlb_entry & XTENSA_MMU_PDTLB_WAY_MASK) >= + XTENSA_MMU_NUM_TLB_AUTOREFILL_WAYS) { + return false; + } + + xtensa_dtlb_entry_invalidate_sync(dtlb_entry); + } + + return false; +} + #ifdef CONFIG_XTENSA_MMU_FLUSH_AUTOREFILL_DTLBS_ON_SWAP /* This is only used when swapping page tables and auto-refill DTLBs * needing to be invalidated. Otherwise, SWAP_PAGE_TABLE assembly diff --git a/arch/xtensa/core/vector_handlers.c b/arch/xtensa/core/vector_handlers.c index 149e0543e5e14..0cc5a6860840f 100644 --- a/arch/xtensa/core/vector_handlers.c +++ b/arch/xtensa/core/vector_handlers.c @@ -630,6 +630,13 @@ void *xtensa_excint1_c(void *esf) xtensa_lazy_hifi_load(thread->arch.hifi_regs); break; #endif /* CONFIG_XTENSA_LAZY_HIFI_SHARING */ +#if defined(CONFIG_XTENSA_MMU) && defined(CONFIG_USERSPACE) + case EXCCAUSE_LOAD_STORE_RING: + if (!xtensa_exc_load_store_ring_error_check(bsa)) { + break; + } + __fallthrough; +#endif /* CONFIG_XTENSA_MMU && CONFIG_USERSPACE */ default: reason = K_ERR_CPU_EXCEPTION; diff --git a/arch/xtensa/include/xtensa_internal.h b/arch/xtensa/include/xtensa_internal.h index 982a2711b2351..220c61df4ebc8 100644 --- a/arch/xtensa/include/xtensa_internal.h +++ b/arch/xtensa/include/xtensa_internal.h @@ -74,6 +74,22 @@ void xtensa_userspace_enter(k_thread_entry_t user_entry, */ bool xtensa_mem_kernel_has_access(const void *addr, size_t size, int write); +/** + * @brief Check if it is a true load/store ring exception. + * + * When a page can be accessed by both kernel and user threads, the autofill DTLB + * may contain an entry for kernel thread. This will result in load/store ring + * exception when it is accessed by user thread later. In this case, this will + * invalidate all associated TLBs related to kernel access so hardware can reload + * the page table the correct permission for user thread. + * + * @param bsa_p Pointer to BSA struct. + * + * @retval True This is a true access violation. + * @retval False Access violation is due to incorrectly cached auto-refilled TLB. + */ +bool xtensa_exc_load_store_ring_error_check(void *bsa_p); + /** * @} */ diff --git a/arch/xtensa/include/xtensa_mmu_priv.h b/arch/xtensa/include/xtensa_mmu_priv.h index cf8e4c0aca780..eab1df8c27423 100644 --- a/arch/xtensa/include/xtensa_mmu_priv.h +++ b/arch/xtensa/include/xtensa_mmu_priv.h @@ -213,6 +213,15 @@ */ #define XTENSA_MMU_PDTLB_HIT BIT(4) +/** + * PDTLB WAY mask. + * + * For more information see + * Xtensa Instruction Set Architecture (ISA) Reference Manual + * 4.6.5.7 Formats for Probing MMU Option TLB Entries + */ +#define XTENSA_MMU_PDTLB_WAY_MASK 0xFU + /** * Virtual address where the page table is mapped */ From 40da30f821999d427650f560582d8a7daeaf4c53 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 29 Jul 2025 14:52:43 -0700 Subject: [PATCH 05/13] xtensa: userspace: handle DTLB multihit exception This adds a function to handle DTLB multihit exception when userspace is enabled, as this exception may be raised due to the same page being able to accessed by both kernel and user threads. The auto-refill DTLBs may contain entries for same page, one for kernel and one for user under some situations. We need to invalidate those existing DTLB entries so that hardware can reload from the page table. This is an alternative to the kconfig invalidating DTLBs on every swap: CONFIG_XTENSA_MMU_FLUSH_AUTOREFILL_DTLBS_ON_SWAP. Although this does not have extra processing per context switching, exception handling itself has a high cost. So care needs to be taken on whether to enable that kconfig. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 10 ++++++++++ arch/xtensa/core/vector_handlers.c | 4 ++++ arch/xtensa/include/xtensa_internal.h | 8 ++++++++ 3 files changed, 22 insertions(+) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index ed9563b7413e6..de07d6e89bcf0 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -1173,6 +1173,16 @@ int arch_buffer_validate(const void *addr, size_t size, int write) return mem_buffer_validate(addr, size, write, XTENSA_MMU_USER_RING); } +void xtensa_exc_dtlb_multihit_handle(void) +{ + /* For some unknown reasons, using xtensa_dtlb_probe() would result in + * QEMU raising privileged instruction exception. So for now, just + * invalidate all auto-refilled DTLBs. + */ + + xtensa_dtlb_autorefill_invalidate(); +} + bool xtensa_exc_load_store_ring_error_check(void *bsa_p) { uintptr_t ring, vaddr; diff --git a/arch/xtensa/core/vector_handlers.c b/arch/xtensa/core/vector_handlers.c index 0cc5a6860840f..c51d5e83a588e 100644 --- a/arch/xtensa/core/vector_handlers.c +++ b/arch/xtensa/core/vector_handlers.c @@ -3,6 +3,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ +#include "xtensa/corebits.h" #include #include #include @@ -631,6 +632,9 @@ void *xtensa_excint1_c(void *esf) break; #endif /* CONFIG_XTENSA_LAZY_HIFI_SHARING */ #if defined(CONFIG_XTENSA_MMU) && defined(CONFIG_USERSPACE) + case EXCCAUSE_DTLB_MULTIHIT: + xtensa_exc_dtlb_multihit_handle(); + break; case EXCCAUSE_LOAD_STORE_RING: if (!xtensa_exc_load_store_ring_error_check(bsa)) { break; diff --git a/arch/xtensa/include/xtensa_internal.h b/arch/xtensa/include/xtensa_internal.h index 220c61df4ebc8..c56e1068afe30 100644 --- a/arch/xtensa/include/xtensa_internal.h +++ b/arch/xtensa/include/xtensa_internal.h @@ -74,6 +74,14 @@ void xtensa_userspace_enter(k_thread_entry_t user_entry, */ bool xtensa_mem_kernel_has_access(const void *addr, size_t size, int write); +/** + * @brief Handle DTLB multihit exception. + * + * Handle DTLB multihit exception by invalidating all auto-refilled DTLBs of + * a particular memory page. + */ +void xtensa_exc_dtlb_multihit_handle(void); + /** * @brief Check if it is a true load/store ring exception. * From 7928fef15e7b77a56bdd69a7f19a9a9a6b8ee505 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 4 Aug 2025 16:39:07 -0700 Subject: [PATCH 06/13] xtensa: move saved FPU and HIFI registers to end of BSA This moves the block of FPU and HIFI registers in the Base Save Area (BSA), used during interrupts and exceptions, to the end of the block. This is done to minimize code usage in the vector text section. During interrupt entrance, the code stores a small set of registers first before jumping to the long handler. When the offset to these registers from the beginning of BSA is small enough, the compiler will emit S32I.N instead of S32I. This saves us one byte per store (2 vs 3 bytes). This is mainly done to avoid overflowing the vector text area. Signed-off-by: Daniel Leung --- arch/xtensa/include/xtensa_asm2_context.h | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/xtensa/include/xtensa_asm2_context.h b/arch/xtensa/include/xtensa_asm2_context.h index 6b669e3804687..8e1bfc1099892 100644 --- a/arch/xtensa/include/xtensa_asm2_context.h +++ b/arch/xtensa/include/xtensa_asm2_context.h @@ -152,6 +152,31 @@ * are saved after the BSA. */ struct xtensa_irq_base_save_area { +#if XCHAL_HAVE_THREADPTR + uintptr_t threadptr; +#endif + +#if XCHAL_HAVE_S32C1I + uintptr_t scompare1; +#endif + + uintptr_t exccause; + uintptr_t excvaddr; + +#if XCHAL_HAVE_LOOPS + uintptr_t lcount; + uintptr_t lend; + uintptr_t lbeg; +#endif + + uintptr_t sar; + uintptr_t ps; + uintptr_t pc; + uintptr_t a0; + uintptr_t scratch; + uintptr_t a2; + uintptr_t a3; + #if XCHAL_HAVE_FP && defined(CONFIG_CPU_HAS_FPU) && defined(CONFIG_FPU_SHARING) uintptr_t fcr; uintptr_t fsr; @@ -185,31 +210,6 @@ struct xtensa_irq_base_save_area { uint8_t hifi[XCHAL_CP1_SA_SIZE + XCHAL_CP1_SA_ALIGN]; #endif -#if XCHAL_HAVE_THREADPTR - uintptr_t threadptr; -#endif - -#if XCHAL_HAVE_S32C1I - uintptr_t scompare1; -#endif - - uintptr_t exccause; - uintptr_t excvaddr; - -#if XCHAL_HAVE_LOOPS - uintptr_t lcount; - uintptr_t lend; - uintptr_t lbeg; -#endif - - uintptr_t sar; - uintptr_t ps; - uintptr_t pc; - uintptr_t a0; - uintptr_t scratch; - uintptr_t a2; - uintptr_t a3; - uintptr_t padding[_BSA_PADDING_NEEDED / sizeof(uintptr_t)]; uintptr_t caller_a0; From 082c5dc7e02631da21b815f16038791344a8c95f Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 8 Oct 2025 13:14:37 -0700 Subject: [PATCH 07/13] xtensa: rename xtensa_asm2_s.h to xtensa_asm2.inc.S xtensa_asm2_s.h is not exactly a C header. Rename it to xtensa_asm2.inc.S to clearly state that it is has assembly code in it as its main purpose is to declare assembly macros. This is being done to keep checkpatch.pl from treating it as C file and complaining about non-C syntax. Signed-off-by: Daniel Leung --- arch/xtensa/core/userspace.S | 2 +- arch/xtensa/core/xtensa_asm2_util.S | 2 +- arch/xtensa/include/{xtensa_asm2_s.h => xtensa_asm2.inc.S} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename arch/xtensa/include/{xtensa_asm2_s.h => xtensa_asm2.inc.S} (100%) diff --git a/arch/xtensa/core/userspace.S b/arch/xtensa/core/userspace.S index 56d53d59b0b56..be1f959ac0582 100644 --- a/arch/xtensa/core/userspace.S +++ b/arch/xtensa/core/userspace.S @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include +#include #include #include #include diff --git a/arch/xtensa/core/xtensa_asm2_util.S b/arch/xtensa/core/xtensa_asm2_util.S index ffa75bbc3a6a0..24893db692dc5 100644 --- a/arch/xtensa/core/xtensa_asm2_util.S +++ b/arch/xtensa/core/xtensa_asm2_util.S @@ -3,7 +3,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -#include +#include #include #include diff --git a/arch/xtensa/include/xtensa_asm2_s.h b/arch/xtensa/include/xtensa_asm2.inc.S similarity index 100% rename from arch/xtensa/include/xtensa_asm2_s.h rename to arch/xtensa/include/xtensa_asm2.inc.S From 580d6ec3a560776d7f9feb5987b4ed428325fbca Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 4 Aug 2025 16:46:13 -0700 Subject: [PATCH 08/13] xtensa: userspace: save PS to thread struct later This is simply done to conserve code space in the vector text areas. These vector text areas are very small and we should only put code that is absolutely necessary for interrupt and exception entrance. The saving of PS into the thread struct can be deferred a bit. So do that. Signed-off-by: Daniel Leung --- arch/xtensa/include/xtensa_asm2.inc.S | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/arch/xtensa/include/xtensa_asm2.inc.S b/arch/xtensa/include/xtensa_asm2.inc.S index d82375696a685..9d8ce2bf7b8a1 100644 --- a/arch/xtensa/include/xtensa_asm2.inc.S +++ b/arch/xtensa/include/xtensa_asm2.inc.S @@ -431,6 +431,29 @@ _xstack_returned_\@: */ s32i a2, a1, ___xtensa_irq_bsa_t_scratch_OFFSET +#ifdef CONFIG_USERSPACE + /* When restoring context via xtensa_switch and + * returning from non-nested interrupts, we will be + * using the stashed PS value in the thread struct + * instead of the one in the thread stack. Both of + * these scenarios will have nested value of 0. + * So when nested value is zero, we store the PS + * value into thread struct. + */ + rsr.ZSR_CPU a0 + l32i a2, a0, ___cpu_t_nested_OFFSET + bnez a2, _excint_skip_ps_save_to_thread + + l32i a2, a0, ___cpu_t_current_OFFSET + s32i a3, a2, _thread_offset_to_return_ps + +_excint_skip_ps_save_to_thread: + /* DEF_EXCINT saved PS into A3 so we need to restore + * A3 here before proceeding. + */ + l32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET +#endif + ODD_REG_SAVE a0, a1 #if XCHAL_HAVE_FP && defined(CONFIG_CPU_HAS_FPU) && defined(CONFIG_FPU_SHARING) @@ -740,22 +763,10 @@ _not_triple_fault: .endif #ifdef CONFIG_USERSPACE - /* When restoring context via xtensa_switch and - * returning from non-nested interrupts, we will be - * using the stashed PS value in the thread struct - * instead of the one in the thread stack. Both of - * these scenarios will have nested value of 0. - * So when nested value is zero, we store the PS - * value into thread struct. + /* Stash the PS into A3 so EXCINT_HANDLER can read this + * and save it into thread struct if needed. */ - rsr.ZSR_CPU a3 - l32i a2, a3, ___cpu_t_nested_OFFSET - bnez a2, _excint_skip_ps_save_to_thread_\LVL - - l32i a2, a3, ___cpu_t_current_OFFSET - s32i a0, a2, _thread_offset_to_return_ps - -_excint_skip_ps_save_to_thread_\LVL: + mov a3, a0 #endif rsr.epc\LVL a0 From 02ff0c6870811bf779534c7da53f89d473600466 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Fri, 8 Aug 2025 14:03:20 -0700 Subject: [PATCH 09/13] xtensa: mmu: fix __arch_mem_map assert message The assert error messages when l2_page_table_map() fails are not correct. It returns false when it cannot allocate new L2 table, and it is not able the address having already mapped. So correct the error message. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index de07d6e89bcf0..a64e88740a13d 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -448,12 +448,12 @@ static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr, paddr, flags, is_user); - __ASSERT(ret, "Virtual address (%p) already mapped", va); + __ASSERT(ret, "Cannot map virtual address (%p)", va); if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc, flags_uc, is_user); - __ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc); + __ASSERT(ret, "Cannot map virtual address (%p)", vaddr_uc); } #ifndef CONFIG_USERSPACE @@ -470,14 +470,14 @@ static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr, flags, is_user); - __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + __ASSERT(ret, "Cannot map virtual address (%p) for domain %p", vaddr, domain); if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { ret = l2_page_table_map(domain->ptables, (void *)vaddr_uc, paddr_uc, flags_uc, is_user); - __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + __ASSERT(ret, "Cannot map virtual address (%p) for domain %p", vaddr_uc, domain); } } From b9443f84ffd270c55cb73eeb6ef58e212e912333 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Fri, 8 Aug 2025 14:14:04 -0700 Subject: [PATCH 10/13] xtensa: mmu: cosmetic changes to page table variable names In functions which manipulate both L1 and L2 tables, make the variable names obvious by prefixing them with l1_ or l2_. This is mainly done to avoid confusion when reading through those functions. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 85 +++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index a64e88740a13d..205754980613e 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -62,14 +62,14 @@ BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, * Each memory domain contains its own l1 page table. The kernel l1 page table is * located at the index 0. */ -static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES] +static uint32_t l1_page_tables[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); /* * That is an alias for the page tables set used by the kernel. */ -uint32_t *xtensa_kernel_ptables = (uint32_t *)l1_page_table[0]; +uint32_t *xtensa_kernel_ptables = (uint32_t *)l1_page_tables[0]; /* * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one @@ -84,7 +84,7 @@ static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_T * * @note: The first bit is set because it is used for the kernel page tables. */ -static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES); +static ATOMIC_DEFINE(l1_page_tables_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES); /* * This additional variable tracks which l2 tables are in use. This is kept separated from @@ -217,7 +217,7 @@ static inline uint32_t *alloc_l2_table(void) static void map_memory_range(const uint32_t start, const uint32_t end, const uint32_t attrs, const uint32_t options) { - uint32_t page, *table; + uint32_t page; bool shared = !!(attrs & XTENSA_MMU_MAP_SHARED); bool do_save_attrs = (options & OPTION_SAVE_ATTRS) == OPTION_SAVE_ATTRS; uint32_t ring, sw_attrs, sw_ring, pte_sw; @@ -228,26 +228,27 @@ static void map_memory_range(const uint32_t start, const uint32_t end, pte_sw = XTENSA_MMU_PTE_SW(sw_ring, sw_attrs); for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { + uint32_t *l2_table; uint32_t pte = XTENSA_MMU_PTE(page, ring, pte_sw, attrs); uint32_t l2_pos = XTENSA_MMU_L2_POS(page); uint32_t l1_pos = XTENSA_MMU_L1_POS(page); if (is_pte_illegal(xtensa_kernel_ptables[l1_pos])) { - table = alloc_l2_table(); + l2_table = alloc_l2_table(); - __ASSERT(table != NULL, "There is no l2 page table available to " - "map 0x%08x\n", page); + __ASSERT(l2_table != NULL, + "There is no l2 page table available to map 0x%08x\n", page); - init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES, + init_page_table(l2_table, XTENSA_L2_PAGE_TABLE_ENTRIES, XTENSA_MMU_PTE_L2_ILLEGAL); xtensa_kernel_ptables[l1_pos] = - XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING, + XTENSA_MMU_PTE((uint32_t)l2_table, XTENSA_MMU_KERNEL_RING, 0, XTENSA_MMU_PAGE_TABLE_ATTR); } - table = (uint32_t *)(xtensa_kernel_ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); - table[l2_pos] = pte; + l2_table = (uint32_t *)(xtensa_kernel_ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); + l2_table[l2_pos] = pte; } } @@ -289,7 +290,7 @@ static void xtensa_init_page_tables(void) init_page_table(xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES, XTENSA_MMU_PTE_L1_ILLEGAL); - atomic_set_bit(l1_page_table_track, 0); + atomic_set_bit(l1_page_tables_track, 0); for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; @@ -308,8 +309,8 @@ static void xtensa_init_page_tables(void) * must be writable, obviously). They shouldn't be left at * the default. */ - map_memory_range((uint32_t) &l1_page_table[0], - (uint32_t) &l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES], + map_memory_range((uint32_t) &l1_page_tables[0], + (uint32_t) &l1_page_tables[CONFIG_XTENSA_MMU_NUM_L1_TABLES], XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W, OPTION_SAVE_ATTRS); map_memory_range((uint32_t) &l2_page_tables[0], (uint32_t) &l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES], @@ -382,33 +383,33 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, { uint32_t l1_pos = XTENSA_MMU_L1_POS((uint32_t)vaddr); uint32_t l2_pos = XTENSA_MMU_L2_POS((uint32_t)vaddr); - uint32_t *table; + uint32_t *l2_table; sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); if (is_pte_illegal(l1_table[l1_pos])) { - table = alloc_l2_table(); + l2_table = alloc_l2_table(); - if (table == NULL) { + if (l2_table == NULL) { return false; } - init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES, XTENSA_MMU_PTE_L2_ILLEGAL); + init_page_table(l2_table, XTENSA_L2_PAGE_TABLE_ENTRIES, XTENSA_MMU_PTE_L2_ILLEGAL); - l1_table[l1_pos] = XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING, + l1_table[l1_pos] = XTENSA_MMU_PTE((uint32_t)l2_table, XTENSA_MMU_KERNEL_RING, 0, XTENSA_MMU_PAGE_TABLE_ATTR); sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); } - table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); - table[l2_pos] = XTENSA_MMU_PTE(phys, is_user ? XTENSA_MMU_USER_RING : - XTENSA_MMU_KERNEL_RING, - XTENSA_MMU_PTE_SW(XTENSA_MMU_KERNEL_RING, - XTENSA_MMU_PTE_ATTR_ILLEGAL), - flags); + l2_table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); + l2_table[l2_pos] = XTENSA_MMU_PTE(phys, is_user ? XTENSA_MMU_USER_RING : + XTENSA_MMU_KERNEL_RING, + XTENSA_MMU_PTE_SW(XTENSA_MMU_KERNEL_RING, + XTENSA_MMU_PTE_ATTR_ILLEGAL), + flags); - sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); xtensa_tlb_autorefill_invalidate(); return true; @@ -551,7 +552,7 @@ static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) uint32_t l1_pos = XTENSA_MMU_L1_POS((uint32_t)vaddr); uint32_t l2_pos = XTENSA_MMU_L2_POS((uint32_t)vaddr); uint32_t *l2_table; - uint32_t table_pos; + uint32_t table_trk_pos; bool exec; sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); @@ -589,8 +590,8 @@ static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) l1_table[l1_pos] = XTENSA_MMU_PTE_L1_ILLEGAL; sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); - table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); - atomic_clear_bit(l2_page_tables_track, table_pos); + table_trk_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); + atomic_clear_bit(l2_page_tables_track, table_trk_pos); end: /* Need to invalidate L2 page table as it is no longer valid. */ @@ -696,7 +697,7 @@ void xtensa_mmu_tlb_shootdown(void) /* We don't have information on which page tables have changed, * so we just invalidate the cache for all L1 page tables. */ - sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table)); + sys_cache_data_invd_range((void *)l1_page_tables, sizeof(l1_page_tables)); sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); } @@ -793,8 +794,8 @@ static inline uint32_t *alloc_l1_table(void) uint16_t idx; for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) { - if (!atomic_test_and_set_bit(l1_page_table_track, idx)) { - return (uint32_t *)&l1_page_table[idx]; + if (!atomic_test_and_set_bit(l1_page_tables_track, idx)) { + return (uint32_t *)&l1_page_tables[idx]; } } @@ -804,9 +805,9 @@ static inline uint32_t *alloc_l1_table(void) static uint32_t *dup_table(void) { uint16_t i, j; - uint32_t *dst_table = alloc_l1_table(); + uint32_t *l1_table = alloc_l1_table(); - if (!dst_table) { + if (!l1_table) { return NULL; } @@ -815,7 +816,7 @@ static uint32_t *dup_table(void) if (is_pte_illegal(xtensa_kernel_ptables[i]) || (i == XTENSA_MMU_L1_POS(XTENSA_MMU_PTEVADDR))) { - dst_table[i] = XTENSA_MMU_PTE_L1_ILLEGAL; + l1_table[i] = XTENSA_MMU_PTE_L1_ILLEGAL; continue; } @@ -832,15 +833,15 @@ static uint32_t *dup_table(void) /* The page table is using kernel ASID because we don't * user thread manipulate it. */ - dst_table[i] = XTENSA_MMU_PTE((uint32_t)l2_table, XTENSA_MMU_KERNEL_RING, - 0, XTENSA_MMU_PAGE_TABLE_ATTR); + l1_table[i] = XTENSA_MMU_PTE((uint32_t)l2_table, XTENSA_MMU_KERNEL_RING, + 0, XTENSA_MMU_PAGE_TABLE_ATTR); sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE); } - sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE); + sys_cache_data_flush_range((void *)l1_table, XTENSA_L1_PAGE_TABLE_SIZE); - return dst_table; + return l1_table; err: /* TODO: Cleanup failed allocation*/ @@ -894,7 +895,7 @@ int arch_mem_domain_init(struct k_mem_domain *domain) return ret; } -static void region_map_update(uint32_t *ptables, uintptr_t start, +static void region_map_update(uint32_t *l1_table, uintptr_t start, size_t size, uint32_t ring, uint32_t flags, uint32_t option) { for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { @@ -904,9 +905,9 @@ static void region_map_update(uint32_t *ptables, uintptr_t start, uint32_t l1_pos = XTENSA_MMU_L1_POS(page); uint32_t l2_pos = XTENSA_MMU_L2_POS(page); /* Make sure we grab a fresh copy of L1 page table */ - sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0])); + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); - l2_table = (uint32_t *)(ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); + l2_table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK); sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); From 81eb9fe5f772a7a01f591ee8c6205b23c1877556 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 11 Aug 2025 15:22:38 -0700 Subject: [PATCH 11/13] xtensa: mmu/ptables: rename flags to attrs under arch_mem_map() arch_mem_map() takes in some flags to describe the to-be mapped memory regions' permissions and cache status. When the flags are translated, they become attributes in PTEs. So for functions being called by arch_mem_map() and beyond, rename flags to attrs to better describe its purpose. Signed-off-by: Daniel Leung --- arch/xtensa/core/ptables.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index 205754980613e..3680fc7cf2a65 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -379,7 +379,7 @@ __weak void arch_reserved_pages_update(void) #endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, - uint32_t flags, bool is_user) + uint32_t attrs, bool is_user) { uint32_t l1_pos = XTENSA_MMU_L1_POS((uint32_t)vaddr); uint32_t l2_pos = XTENSA_MMU_L2_POS((uint32_t)vaddr); @@ -407,7 +407,7 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, XTENSA_MMU_KERNEL_RING, XTENSA_MMU_PTE_SW(XTENSA_MMU_KERNEL_RING, XTENSA_MMU_PTE_ATTR_ILLEGAL), - flags); + attrs); sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); xtensa_tlb_autorefill_invalidate(); @@ -415,12 +415,12 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, return true; } -static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user) +static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t new_attrs, bool is_user) { bool ret; void *vaddr, *vaddr_uc; uintptr_t paddr, paddr_uc; - uint32_t flags, flags_uc; + uint32_t attrs, attrs_uc; if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { if (sys_cache_is_ptr_cached(va)) { @@ -439,21 +439,21 @@ static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, paddr_uc = pa; } - flags_uc = (xtensa_flags & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK); - flags = flags_uc | XTENSA_MMU_CACHED_WB; + attrs_uc = (new_attrs & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK); + attrs = attrs_uc | XTENSA_MMU_CACHED_WB; } else { vaddr = va; paddr = pa; - flags = xtensa_flags; + attrs = new_attrs; } ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr, paddr, - flags, is_user); + attrs, is_user); __ASSERT(ret, "Cannot map virtual address (%p)", va); if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc, - flags_uc, is_user); + attrs_uc, is_user); __ASSERT(ret, "Cannot map virtual address (%p)", vaddr_uc); } @@ -470,14 +470,14 @@ static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, domain = CONTAINER_OF(node, struct arch_mem_domain, node); ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr, - flags, is_user); + attrs, is_user); __ASSERT(ret, "Cannot map virtual address (%p) for domain %p", vaddr, domain); if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { ret = l2_page_table_map(domain->ptables, (void *)vaddr_uc, paddr_uc, - flags_uc, is_user); + attrs_uc, is_user); __ASSERT(ret, "Cannot map virtual address (%p) for domain %p", vaddr_uc, domain); } @@ -492,7 +492,7 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) uint32_t va = (uint32_t)virt; uint32_t pa = (uint32_t)phys; uint32_t rem_size = (uint32_t)size; - uint32_t xtensa_flags = 0; + uint32_t attrs = 0; k_spinlock_key_t key; bool is_user; @@ -505,10 +505,10 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) switch (flags & K_MEM_CACHE_MASK) { case K_MEM_CACHE_WB: - xtensa_flags |= XTENSA_MMU_CACHED_WB; + attrs |= XTENSA_MMU_CACHED_WB; break; case K_MEM_CACHE_WT: - xtensa_flags |= XTENSA_MMU_CACHED_WT; + attrs |= XTENSA_MMU_CACHED_WT; break; case K_MEM_CACHE_NONE: __fallthrough; @@ -517,10 +517,10 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) } if ((flags & K_MEM_PERM_RW) == K_MEM_PERM_RW) { - xtensa_flags |= XTENSA_MMU_PERM_W; + attrs |= XTENSA_MMU_PERM_W; } if ((flags & K_MEM_PERM_EXEC) == K_MEM_PERM_EXEC) { - xtensa_flags |= XTENSA_MMU_PERM_X; + attrs |= XTENSA_MMU_PERM_X; } is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER; @@ -528,7 +528,7 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) key = k_spin_lock(&xtensa_mmu_lock); while (rem_size > 0) { - __arch_mem_map((void *)va, pa, xtensa_flags, is_user); + __arch_mem_map((void *)va, pa, attrs, is_user); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); From 3512d6fbde0d5f615d84abd5e1f0980463a000b0 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 24 Jul 2025 15:36:22 -0700 Subject: [PATCH 12/13] tests: mem_protect/userspace: fix privilege stack ptr for Xtensa The priv_stack_ptr for reading and writing tests was not set correctly on Xtensa, and was always 0. Test would pass since the NULL page should always generate access faults on most Xtensa platforms. However, those tests are supposed to test reading and writing to the privilege stack. So fix that by populating the priv_stack_ptr correctly. Signed-off-by: Daniel Leung --- tests/kernel/mem_protect/userspace/src/main.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kernel/mem_protect/userspace/src/main.c b/tests/kernel/mem_protect/userspace/src/main.c index 6211f5ebc5995..1ea47793a153a 100644 --- a/tests/kernel/mem_protect/userspace/src/main.c +++ b/tests/kernel/mem_protect/userspace/src/main.c @@ -1191,6 +1191,12 @@ void *userspace_setup(void) priv_stack_ptr = (char *)((uintptr_t)ztest_thread_stack + Z_RISCV_STACK_GUARD_SIZE); #endif +#elif defined(CONFIG_XTENSA) + struct xtensa_thread_stack_header *hdr; + void *vhdr = ((struct xtensa_thread_stack_header *)ztest_thread_stack); + + hdr = vhdr; + priv_stack_ptr = (((char *)&hdr->privilege_stack) + (sizeof(hdr->privilege_stack) - 1)); #endif k_thread_access_grant(k_current_get(), &test_thread, &test_stack, From d8b86979055597888cba1df221686fa1e889aa21 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 24 Jul 2025 10:18:40 -0700 Subject: [PATCH 13/13] tests: mem_protect/userspace: add thread switching tests This adds a few tests to test switching between threads where they are under different memory domains. This is mainly to test if permissions are set correctly during context switch. By default the new tests are disabled and must be explicitly enabled. This is due to some architectures requiring some config changes (for example, x86 needing more reserved memory domains, and ARM64 needing more translation tables). To avoid causing mass CI failures, only enable platforms that have been tested, for now. Signed-off-by: Daniel Leung --- tests/kernel/mem_protect/userspace/Kconfig | 15 ++ .../boards/intel_adsp_ace30_ptl.conf | 2 + .../boards/intel_adsp_ace30_ptl_sim.conf | 2 + .../userspace/boards/qemu_cortex_a53.conf | 2 + .../qemu_cortex_a53_qemu_cortex_a53_smp.conf | 2 + .../userspace/boards/qemu_x86.conf | 2 + .../userspace/boards/qemu_x86_64.conf | 2 + .../userspace/boards/qemu_x86_atom_nopae.conf | 2 + .../userspace/boards/qemu_x86_lakemont.conf | 2 + .../boards/qemu_xtensa_dc233c_mmu.conf | 1 + tests/kernel/mem_protect/userspace/src/main.c | 2 +- .../mem_protect/userspace/src/switching.c | 232 ++++++++++++++++++ 12 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 tests/kernel/mem_protect/userspace/Kconfig create mode 100644 tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl_sim.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53_qemu_cortex_a53_smp.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_x86.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_x86_64.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_x86_atom_nopae.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_x86_lakemont.conf create mode 100644 tests/kernel/mem_protect/userspace/boards/qemu_xtensa_dc233c_mmu.conf create mode 100644 tests/kernel/mem_protect/userspace/src/switching.c diff --git a/tests/kernel/mem_protect/userspace/Kconfig b/tests/kernel/mem_protect/userspace/Kconfig new file mode 100644 index 0000000000000..21f6ffed0245b --- /dev/null +++ b/tests/kernel/mem_protect/userspace/Kconfig @@ -0,0 +1,15 @@ +# Copyright (c) 2025 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +mainmenu "Userspace test" + +source "Kconfig.zephyr" + +config USERSPACE_SWITCHING_TESTS + bool "Run thread switching tests" + select SCHED_CPU_MASK if MP_MAX_NUM_CPUS > 1 + help + Run userspace_domain_switching tests. + + Enable this via board overlay. diff --git a/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl.conf b/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl.conf new file mode 100644 index 0000000000000..c0c9f02279360 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl.conf @@ -0,0 +1,2 @@ +CONFIG_MAX_THREAD_BYTES=3 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl_sim.conf b/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl_sim.conf new file mode 100644 index 0000000000000..c0c9f02279360 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/intel_adsp_ace30_ptl_sim.conf @@ -0,0 +1,2 @@ +CONFIG_MAX_THREAD_BYTES=3 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53.conf b/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53.conf new file mode 100644 index 0000000000000..e74376d117959 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53.conf @@ -0,0 +1,2 @@ +CONFIG_MAX_XLAT_TABLES=24 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53_qemu_cortex_a53_smp.conf b/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53_qemu_cortex_a53_smp.conf new file mode 100644 index 0000000000000..e74376d117959 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_cortex_a53_qemu_cortex_a53_smp.conf @@ -0,0 +1,2 @@ +CONFIG_MAX_XLAT_TABLES=24 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_x86.conf b/tests/kernel/mem_protect/userspace/boards/qemu_x86.conf new file mode 100644 index 0000000000000..a2d1cb23f9a90 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_x86.conf @@ -0,0 +1,2 @@ +CONFIG_X86_MAX_ADDITIONAL_MEM_DOMAINS=4 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_x86_64.conf b/tests/kernel/mem_protect/userspace/boards/qemu_x86_64.conf new file mode 100644 index 0000000000000..a2d1cb23f9a90 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_x86_64.conf @@ -0,0 +1,2 @@ +CONFIG_X86_MAX_ADDITIONAL_MEM_DOMAINS=4 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_x86_atom_nopae.conf b/tests/kernel/mem_protect/userspace/boards/qemu_x86_atom_nopae.conf new file mode 100644 index 0000000000000..a2d1cb23f9a90 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_x86_atom_nopae.conf @@ -0,0 +1,2 @@ +CONFIG_X86_MAX_ADDITIONAL_MEM_DOMAINS=4 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_x86_lakemont.conf b/tests/kernel/mem_protect/userspace/boards/qemu_x86_lakemont.conf new file mode 100644 index 0000000000000..a2d1cb23f9a90 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_x86_lakemont.conf @@ -0,0 +1,2 @@ +CONFIG_X86_MAX_ADDITIONAL_MEM_DOMAINS=4 +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/boards/qemu_xtensa_dc233c_mmu.conf b/tests/kernel/mem_protect/userspace/boards/qemu_xtensa_dc233c_mmu.conf new file mode 100644 index 0000000000000..260345c5a32a8 --- /dev/null +++ b/tests/kernel/mem_protect/userspace/boards/qemu_xtensa_dc233c_mmu.conf @@ -0,0 +1 @@ +CONFIG_USERSPACE_SWITCHING_TESTS=y diff --git a/tests/kernel/mem_protect/userspace/src/main.c b/tests/kernel/mem_protect/userspace/src/main.c index 1ea47793a153a..032c2e7ed5ed2 100644 --- a/tests/kernel/mem_protect/userspace/src/main.c +++ b/tests/kernel/mem_protect/userspace/src/main.c @@ -63,7 +63,7 @@ K_APP_BMEM(alt_part) volatile bool alt_bool; static struct k_thread test_thread; static K_THREAD_STACK_DEFINE(test_stack, STACKSIZE); -static void clear_fault(void) +void clear_fault(void) { expect_fault = false; compiler_barrier(); diff --git a/tests/kernel/mem_protect/userspace/src/switching.c b/tests/kernel/mem_protect/userspace/src/switching.c new file mode 100644 index 0000000000000..3b312de10fa7f --- /dev/null +++ b/tests/kernel/mem_protect/userspace/src/switching.c @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2025 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include /* for z_libc_partition */ + +#define NUM_THREADS 3 +#define TIMES_SWITCHING 10 +#define STACKSIZE (256 + CONFIG_TEST_EXTRA_STACK_SIZE) + +extern void clear_fault(void); + +#ifdef CONFIG_USERSPACE_SWITCHING_TESTS +/* + * Even numbered threads use domain_a. + * Odd numbered threads use domain_b. + */ + +struct k_mem_domain domain_a; +K_APPMEM_PARTITION_DEFINE(partition_a); +K_APP_BMEM(partition_a) volatile unsigned int part_a_loops[NUM_THREADS]; + +struct k_mem_domain domain_b; +K_APPMEM_PARTITION_DEFINE(partition_b); +K_APP_BMEM(partition_b) volatile unsigned int part_b_loops[NUM_THREADS]; + +static struct k_thread threads[NUM_THREADS]; +static K_THREAD_STACK_ARRAY_DEFINE(threads_stacks, NUM_THREADS, STACKSIZE); + +static K_SEM_DEFINE(sem_switching, 1, 1); + +static void switch_thread_fn(void *arg1, void *arg2, void *arg3) +{ + volatile unsigned int *loop_ptr; + const uintptr_t thread_id = (uintptr_t)arg1; + + if ((thread_id % 2) == 0) { + loop_ptr = &part_a_loops[thread_id]; + } else { + loop_ptr = &part_b_loops[thread_id]; + } + + for (int i = 0; i < TIMES_SWITCHING; i++) { +#ifdef CONFIG_DEBUG + TC_PRINT("Thread %lu (%u)\n", thread_id, *loop_ptr); +#endif + + *loop_ptr += 1; + compiler_barrier(); + + /* Make sure this can still use kernel objects. */ + k_sem_take(&sem_switching, K_FOREVER); + k_sem_give(&sem_switching); + + k_yield(); + } +} + +#endif /* CONFIG_USERSPACE_SWITCHING_TESTS */ + +static void run_switching(int num_kernel_threads) +{ +#ifdef CONFIG_USERSPACE_SWITCHING_TESTS + unsigned int i; + int remaining_kernel_threads = num_kernel_threads; + + /* Not expecting any errors. */ + clear_fault(); + + for (i = 0; i < NUM_THREADS; i++) { + uint32_t perms; + bool is_kernel_thread = remaining_kernel_threads > 0; + + if (is_kernel_thread) { + perms = K_INHERIT_PERMS; + + remaining_kernel_threads--; + } else { + perms = K_INHERIT_PERMS | K_USER; + } + + /* Clear loop counters. */ + part_a_loops[i] = 0; + part_b_loops[i] = 0; + + /* Must delay start of threads to apply memory domains to them. */ + k_thread_create(&threads[i], threads_stacks[i], STACKSIZE, switch_thread_fn, + (void *)(uintptr_t)i, NULL, NULL, -1, perms, K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + /* + * Make sure all created threads run on the same CPU + * so that memory domain switching is being tested. + */ + (void)k_thread_cpu_pin(&threads[i], 0); +#endif /* CONFIG_SCHED_CPU_MASK */ + + k_thread_access_grant(&threads[i], &sem_switching); + + /* + * Kernel threads by default has access to all memory. + * So no need to put them into memory domains. + */ + if (!is_kernel_thread) { + /* Remember EVEN -> A, ODD -> B. */ + if ((i % 2) == 0) { + k_mem_domain_add_thread(&domain_a, &threads[i]); + } else { + k_mem_domain_add_thread(&domain_b, &threads[i]); + } + } + } + + /* Start the thread loops. */ + for (i = 0; i < NUM_THREADS; i++) { + k_thread_start(&threads[i]); + } + + /* Wait for all threads to finish. */ + for (i = 0; i < NUM_THREADS; i++) { + k_thread_join(&threads[i], K_FOREVER); + } + + /* Check to make sure all threads have looped enough times. */ + for (i = 0; i < NUM_THREADS; i++) { + int loops; + + /* + * Each thread should never have access to the loop counters on + * the other partition. Accessing them should generate faults. + * Though we check just in case. + */ + if ((i % 2) == 0) { + loops = part_a_loops[i]; + + zassert_equal(part_b_loops[i], 0, "part_b_loops[%i] should be zero but not", + i); + } else { + loops = part_b_loops[i]; + + zassert_equal(part_a_loops[i], 0, "part_a_loops[%i] should be zero but not", + i); + } + + zassert_equal(loops, TIMES_SWITCHING, + "thread %u has not done enough loops (%u != %u)", i, loops, + TIMES_SWITCHING); + } +#else /* CONFIG_USERSPACE_SWITCHING_TESTS */ + ztest_test_skip(); +#endif /* CONFIG_USERSPACE_SWITCHING_TESTS */ +} + +ZTEST(userspace_domain_switching, test_kernel_only_switching) +{ + /* + * Run with all kernel threads. + * + * This should work as kernel threads by default have access to + * all memory, without having to attach them to memory domains. + * This serves as a baseline check. + */ + run_switching(NUM_THREADS); +} + +ZTEST(userspace_domain_switching, test_user_only_switching) +{ + /* Run with all user threads. */ + run_switching(0); +} + +ZTEST(userspace_domain_switching, test_kernel_user_mix_switching) +{ + /* Run with one kernel thread while others are all user threads. */ + run_switching(1); +} + +void *switching_setup(void) +{ +#ifdef CONFIG_USERSPACE_SWITCHING_TESTS + static bool already_inited; + + if (already_inited) { + return NULL; + } + + struct k_mem_partition *parts_a[] = { +#if Z_LIBC_PARTITION_EXISTS + &z_libc_partition, +#endif + &ztest_mem_partition, &partition_a + }; + + struct k_mem_partition *parts_b[] = { +#if Z_LIBC_PARTITION_EXISTS + &z_libc_partition, +#endif + &ztest_mem_partition, &partition_b + }; + + zassert_equal(k_mem_domain_init(&domain_a, ARRAY_SIZE(parts_a), parts_a), 0, + "failed to initialize memory domain A"); + + zassert_equal(k_mem_domain_init(&domain_b, ARRAY_SIZE(parts_b), parts_b), 0, + "failed to initialize memory domain B"); + + already_inited = true; +#endif /* CONFIG_USERSPACE_SWITCHING_TESTS */ + + return NULL; +} + +void switching_before(void *fixture) +{ +#ifdef CONFIG_USERSPACE_SWITCHING_TESTS + int i; + + for (i = 0; i < NUM_THREADS; i++) { + k_thread_access_grant(k_current_get(), &threads[i]); + } +#endif /* CONFIG_USERSPACE_SWITCHING_TESTS */ +} + +ZTEST_SUITE(userspace_domain_switching, NULL, switching_setup, switching_before, NULL, NULL);