demand_paging: eviction: add kconfig CONFIG_EVICTION_TRACKING

This adds a new kconfig for eviction algorithm which needs page
tracking. When enabled, k_mem_paging_eviction_add()/_remove()
and k_mem_paging_eviction_accessed() must be implemented.
If an algorithm does not do page tracking, there is no need to
implement these functions, and no need for the kernel MMU code
to call into empty functions. This should save a few function
calls and some CPU cycles.

Note that arm64 unconditionally calls those functions so
forces CONFIG_EVICTION_TRACKING to be enabled there.

Signed-off-by: Daniel Leung <[email protected]>

Author: dcpleung

arch/Kconfig

@@ -55,6 +55,8 @@ config ARM64
 	select ARCH_HAS_DIRECTED_IPIS
 	select ARCH_HAS_DEMAND_PAGING
 	select ARCH_HAS_DEMAND_MAPPING
+	select ARCH_SUPPORTS_EVICTION_TRACKING
+	select EVICTION_TRACKING if DEMAND_PAGING
 	help
 	  ARM64 (AArch64) architecture
 
@@ -694,6 +696,12 @@ config ARCH_SUPPORTS_ROM_START
 config ARCH_SUPPORTS_EMPTY_IRQ_SPURIOUS
 	bool
 
+config ARCH_SUPPORTS_EVICTION_TRACKING
+	bool
+	help
+	  Architecture code supports page tracking for eviction algorithms
+	  when demand paging is enabled.
+
 config ARCH_HAS_EXTRA_EXCEPTION_INFO
 	bool
 

doc/kernel/memory_management/demand_paging.rst

@@ -156,8 +156,12 @@ Two eviction algorithms are currently available:
   to the NRU code but also considerably more efficient. This is recommended for
   production use.
 
-To implement a new eviction algorithm, the five functions mentioned
-above must be implemented.
+To implement a new eviction algorithm, :c:func:`k_mem_paging_eviction_init()`
+and :c:func:`k_mem_paging_eviction_select()` must be implemented.
+If :kconfig:option:`CONFIG_EVICTION_TRACKING` is enabled for an algorithm,
+these additional functions must also be implemented,
+:c:func:`k_mem_paging_eviction_add()`, :c:func:`k_mem_paging_eviction_remove()`,
+:c:func:`k_mem_paging_eviction_accessed()`.
 
 Backing Store
 *************

include/zephyr/kernel/mm/demand_paging.h

@@ -217,6 +217,8 @@ __syscall void k_mem_paging_histogram_backing_store_page_out_get(
  * @{
  */
 
+#if defined(CONFIG_EVICTION_TRACKING) || defined(__DOXYGEN__)
+
 /**
  * Submit a page frame for eviction candidate tracking
  *
@@ -261,6 +263,25 @@ void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf);
  */
 void k_mem_paging_eviction_accessed(uintptr_t phys);
 
+#else /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */
+
+static inline void k_mem_paging_eviction_add(struct k_mem_page_frame *pf)
+{
+	ARG_UNUSED(pf);
+}
+
+static inline void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf)
+{
+	ARG_UNUSED(pf);
+}
+
+static inline void k_mem_paging_eviction_accessed(uintptr_t phys)
+{
+	ARG_UNUSED(phys);
+}
+
+#endif /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */
+
 /**
  * Select a page frame for eviction
  *

kernel/mmu.c

@@ -556,7 +556,7 @@ static int map_anon_page(void *addr, uint32_t flags)
 	}
 	frame_mapped_set(pf, addr);
 #ifdef CONFIG_DEMAND_PAGING
-	if (!lock) {
+	if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!lock)) {
 		k_mem_paging_eviction_add(pf);
 	}
 #endif
@@ -784,7 +784,8 @@ void k_mem_unmap_phys_guard(void *addr, size_t size, bool is_anon)
 
 			arch_mem_unmap(pos, CONFIG_MMU_PAGE_SIZE);
 #ifdef CONFIG_DEMAND_PAGING
-			if (!k_mem_page_frame_is_pinned(pf)) {
+			if (IS_ENABLED(CONFIG_EVICTION_TRACKING) &&
+			    (!k_mem_page_frame_is_pinned(pf))) {
 				k_mem_paging_eviction_remove(pf);
 			}
 #endif
@@ -1041,7 +1042,8 @@ static void mark_linker_section_pinned(void *start_addr, void *end_addr,
 		} else {
 			k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED);
 #ifdef CONFIG_DEMAND_PAGING
-			if (k_mem_page_frame_is_evictable(pf)) {
+			if (IS_ENABLED(CONFIG_EVICTION_TRACKING) &&
+			    k_mem_page_frame_is_evictable(pf)) {
 				k_mem_paging_eviction_add(pf);
 			}
 #endif
@@ -1147,10 +1149,13 @@ void z_mem_manage_init(void)
 #endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */
 	k_mem_paging_backing_store_init();
 	k_mem_paging_eviction_init();
-	/* start tracking evictable page installed above if any */
-	K_MEM_PAGE_FRAME_FOREACH(phys, pf) {
-		if (k_mem_page_frame_is_evictable(pf)) {
-			k_mem_paging_eviction_add(pf);
+
+	if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) {
+		/* start tracking evictable page installed above if any */
+		K_MEM_PAGE_FRAME_FOREACH(phys, pf) {
+			if (k_mem_page_frame_is_evictable(pf)) {
+				k_mem_paging_eviction_add(pf);
+			}
 		}
 	}
 #endif /* CONFIG_DEMAND_PAGING */
@@ -1347,7 +1352,10 @@ static int page_frame_prepare_locked(struct k_mem_page_frame *pf, bool *dirty_pt
 			return -ENOMEM;
 		}
 		arch_mem_page_out(k_mem_page_frame_to_virt(pf), *location_ptr);
-		k_mem_paging_eviction_remove(pf);
+
+		if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) {
+			k_mem_paging_eviction_remove(pf);
+		}
 	} else {
 		/* Shouldn't happen unless this function is mis-used */
 		__ASSERT(!dirty, "un-mapped page determined to be dirty");
@@ -1683,7 +1691,9 @@ static bool do_page_fault(void *addr, bool pin)
 
 			pf = k_mem_phys_to_page_frame(phys);
 			if (!k_mem_page_frame_is_pinned(pf)) {
-				k_mem_paging_eviction_remove(pf);
+				if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) {
+					k_mem_paging_eviction_remove(pf);
+				}
 				k_mem_page_frame_set(pf, K_MEM_PAGE_FRAME_PINNED);
 			}
 		}
@@ -1738,7 +1748,7 @@ static bool do_page_fault(void *addr, bool pin)
 
 	arch_mem_page_in(addr, k_mem_page_frame_to_phys(pf));
 	k_mem_paging_backing_store_page_finalize(pf, page_in_location);
-	if (!pin) {
+	if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!pin)) {
 		k_mem_paging_eviction_add(pf);
 	}
 out:
@@ -1807,7 +1817,10 @@ static void do_mem_unpin(void *addr)
 		pf = k_mem_phys_to_page_frame(phys);
 		if (k_mem_page_frame_is_pinned(pf)) {
 			k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED);
-			k_mem_paging_eviction_add(pf);
+
+			if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) {
+				k_mem_paging_eviction_add(pf);
+			}
 		}
 	}
 	k_spin_unlock(&z_mm_lock, key);

subsys/demand_paging/eviction/Kconfig

@@ -11,6 +11,7 @@ choice EVICTION_CHOICE
 
 config EVICTION_CUSTOM
 	bool "Custom eviction algorithm"
+	imply EVICTION_TRACKING
 	help
 	  This option is chosen when the eviction algorithm will be implemented
 	  by the application, instead of using one included in Zephyr.
@@ -30,6 +31,7 @@ config EVICTION_NRU
 
 config EVICTION_LRU
 	bool "Least Recently Used (LRU) page eviction algorithm"
+	select EVICTION_TRACKING
 	help
 	  This implements a Least Recently Used page eviction algorithm.
 	  Usage is tracked based on MMU protection making pages unaccessible
@@ -49,3 +51,11 @@ config EVICTION_NRU_PERIOD
 	  pages that are capable of being paged out. At eviction time, if a page
 	  still has the accessed property, it will be considered as recently used.
 endif # EVICTION_NRU
+
+config EVICTION_TRACKING
+	bool
+	depends on ARCH_SUPPORTS_EVICTION_TRACKING
+	help
+	  Selected by eviction algorithms which needs page tracking and need to
+	  implement the following functions: k_mem_paging_eviction_add(),
+	  k_mem_paging_eviction_remove() and k_mem_paging_eviction_accessed().

subsys/demand_paging/eviction/nru.c

@@ -111,8 +111,11 @@ void k_mem_paging_eviction_init(void)
 		      K_MSEC(CONFIG_EVICTION_NRU_PERIOD));
 }
 
+#ifdef CONFIG_EVICTION_TRACKING
 /*
- * unused interfaces
+ * Empty functions defined here so that architectures unconditionally
+ * implement eviction tracking can still use this algorithm for
+ * testing.
  */
 
 void k_mem_paging_eviction_add(struct k_mem_page_frame *pf)
@@ -129,3 +132,5 @@ void k_mem_paging_eviction_accessed(uintptr_t phys)
 {
 	ARG_UNUSED(phys);
 }
+
+#endif /* CONFIG_EVICTION_TRACKING */