diff --git a/arch/Kconfig b/arch/Kconfig index 355a6247af383..774840dda0f92 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -55,6 +55,8 @@ config ARM64 select ARCH_HAS_DIRECTED_IPIS select ARCH_HAS_DEMAND_PAGING select ARCH_HAS_DEMAND_MAPPING + select ARCH_SUPPORTS_EVICTION_TRACKING + select EVICTION_TRACKING if DEMAND_PAGING help ARM64 (AArch64) architecture @@ -694,6 +696,12 @@ config ARCH_SUPPORTS_ROM_START config ARCH_SUPPORTS_EMPTY_IRQ_SPURIOUS bool +config ARCH_SUPPORTS_EVICTION_TRACKING + bool + help + Architecture code supports page tracking for eviction algorithms + when demand paging is enabled. + config ARCH_HAS_EXTRA_EXCEPTION_INFO bool diff --git a/doc/kernel/memory_management/demand_paging.rst b/doc/kernel/memory_management/demand_paging.rst index db068397a1423..76e962bd8e8b0 100644 --- a/doc/kernel/memory_management/demand_paging.rst +++ b/doc/kernel/memory_management/demand_paging.rst @@ -156,8 +156,12 @@ Two eviction algorithms are currently available: to the NRU code but also considerably more efficient. This is recommended for production use. -To implement a new eviction algorithm, the five functions mentioned -above must be implemented. +To implement a new eviction algorithm, :c:func:`k_mem_paging_eviction_init()` +and :c:func:`k_mem_paging_eviction_select()` must be implemented. +If :kconfig:option:`CONFIG_EVICTION_TRACKING` is enabled for an algorithm, +these additional functions must also be implemented, +:c:func:`k_mem_paging_eviction_add()`, :c:func:`k_mem_paging_eviction_remove()`, +:c:func:`k_mem_paging_eviction_accessed()`. Backing Store ************* diff --git a/include/zephyr/kernel/mm/demand_paging.h b/include/zephyr/kernel/mm/demand_paging.h index 120ee3299a349..10a2e7918acec 100644 --- a/include/zephyr/kernel/mm/demand_paging.h +++ b/include/zephyr/kernel/mm/demand_paging.h @@ -217,6 +217,8 @@ __syscall void k_mem_paging_histogram_backing_store_page_out_get( * @{ */ +#if defined(CONFIG_EVICTION_TRACKING) || defined(__DOXYGEN__) + /** * Submit a page frame for eviction candidate tracking * @@ -261,6 +263,25 @@ void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf); */ void k_mem_paging_eviction_accessed(uintptr_t phys); +#else /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ + +static inline void k_mem_paging_eviction_add(struct k_mem_page_frame *pf) +{ + ARG_UNUSED(pf); +} + +static inline void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf) +{ + ARG_UNUSED(pf); +} + +static inline void k_mem_paging_eviction_accessed(uintptr_t phys) +{ + ARG_UNUSED(phys); +} + +#endif /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ + /** * Select a page frame for eviction * diff --git a/kernel/mmu.c b/kernel/mmu.c index b03ff978786a4..788f30ff730ef 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -556,7 +556,7 @@ static int map_anon_page(void *addr, uint32_t flags) } frame_mapped_set(pf, addr); #ifdef CONFIG_DEMAND_PAGING - if (!lock) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!lock)) { k_mem_paging_eviction_add(pf); } #endif @@ -784,7 +784,8 @@ void k_mem_unmap_phys_guard(void *addr, size_t size, bool is_anon) arch_mem_unmap(pos, CONFIG_MMU_PAGE_SIZE); #ifdef CONFIG_DEMAND_PAGING - if (!k_mem_page_frame_is_pinned(pf)) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && + (!k_mem_page_frame_is_pinned(pf))) { k_mem_paging_eviction_remove(pf); } #endif @@ -1041,7 +1042,8 @@ static void mark_linker_section_pinned(void *start_addr, void *end_addr, } else { k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED); #ifdef CONFIG_DEMAND_PAGING - if (k_mem_page_frame_is_evictable(pf)) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && + k_mem_page_frame_is_evictable(pf)) { k_mem_paging_eviction_add(pf); } #endif @@ -1147,10 +1149,13 @@ void z_mem_manage_init(void) #endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */ k_mem_paging_backing_store_init(); k_mem_paging_eviction_init(); - /* start tracking evictable page installed above if any */ - K_MEM_PAGE_FRAME_FOREACH(phys, pf) { - if (k_mem_page_frame_is_evictable(pf)) { - k_mem_paging_eviction_add(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + /* start tracking evictable page installed above if any */ + K_MEM_PAGE_FRAME_FOREACH(phys, pf) { + if (k_mem_page_frame_is_evictable(pf)) { + k_mem_paging_eviction_add(pf); + } } } #endif /* CONFIG_DEMAND_PAGING */ @@ -1347,7 +1352,10 @@ static int page_frame_prepare_locked(struct k_mem_page_frame *pf, bool *dirty_pt return -ENOMEM; } arch_mem_page_out(k_mem_page_frame_to_virt(pf), *location_ptr); - k_mem_paging_eviction_remove(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_remove(pf); + } } else { /* Shouldn't happen unless this function is mis-used */ __ASSERT(!dirty, "un-mapped page determined to be dirty"); @@ -1683,7 +1691,9 @@ static bool do_page_fault(void *addr, bool pin) pf = k_mem_phys_to_page_frame(phys); if (!k_mem_page_frame_is_pinned(pf)) { - k_mem_paging_eviction_remove(pf); + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_remove(pf); + } k_mem_page_frame_set(pf, K_MEM_PAGE_FRAME_PINNED); } } @@ -1738,7 +1748,7 @@ static bool do_page_fault(void *addr, bool pin) arch_mem_page_in(addr, k_mem_page_frame_to_phys(pf)); k_mem_paging_backing_store_page_finalize(pf, page_in_location); - if (!pin) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!pin)) { k_mem_paging_eviction_add(pf); } out: @@ -1807,7 +1817,10 @@ static void do_mem_unpin(void *addr) pf = k_mem_phys_to_page_frame(phys); if (k_mem_page_frame_is_pinned(pf)) { k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED); - k_mem_paging_eviction_add(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_add(pf); + } } } k_spin_unlock(&z_mm_lock, key); diff --git a/subsys/demand_paging/eviction/Kconfig b/subsys/demand_paging/eviction/Kconfig index 3d4044b1fff35..d920f814a0e93 100644 --- a/subsys/demand_paging/eviction/Kconfig +++ b/subsys/demand_paging/eviction/Kconfig @@ -11,6 +11,7 @@ choice EVICTION_CHOICE config EVICTION_CUSTOM bool "Custom eviction algorithm" + imply EVICTION_TRACKING help This option is chosen when the eviction algorithm will be implemented by the application, instead of using one included in Zephyr. @@ -30,6 +31,7 @@ config EVICTION_NRU config EVICTION_LRU bool "Least Recently Used (LRU) page eviction algorithm" + select EVICTION_TRACKING help This implements a Least Recently Used page eviction algorithm. Usage is tracked based on MMU protection making pages unaccessible @@ -49,3 +51,11 @@ config EVICTION_NRU_PERIOD pages that are capable of being paged out. At eviction time, if a page still has the accessed property, it will be considered as recently used. endif # EVICTION_NRU + +config EVICTION_TRACKING + bool + depends on ARCH_SUPPORTS_EVICTION_TRACKING + help + Selected by eviction algorithms which needs page tracking and need to + implement the following functions: k_mem_paging_eviction_add(), + k_mem_paging_eviction_remove() and k_mem_paging_eviction_accessed(). diff --git a/subsys/demand_paging/eviction/nru.c b/subsys/demand_paging/eviction/nru.c index 2fd92ccb884a2..246d2d12300ce 100644 --- a/subsys/demand_paging/eviction/nru.c +++ b/subsys/demand_paging/eviction/nru.c @@ -111,8 +111,11 @@ void k_mem_paging_eviction_init(void) K_MSEC(CONFIG_EVICTION_NRU_PERIOD)); } +#ifdef CONFIG_EVICTION_TRACKING /* - * unused interfaces + * Empty functions defined here so that architectures unconditionally + * implement eviction tracking can still use this algorithm for + * testing. */ void k_mem_paging_eviction_add(struct k_mem_page_frame *pf) @@ -129,3 +132,5 @@ void k_mem_paging_eviction_accessed(uintptr_t phys) { ARG_UNUSED(phys); } + +#endif /* CONFIG_EVICTION_TRACKING */