common: reduce stack usage by eliminating complex memmove_ inline function

Signed-off-by: Tomasz Gromadzki <[email protected]>

Author: grom72

ChangeLog

@@ -2,6 +2,7 @@
 
 	This release:
 	- Significantly reduces the libpmem's stack usage.
+	- Make memmove_mov_* function non-inline to prevent huge stack consumption.
 
 Tue Aug 8 2023 Oksana Sałyk <[email protected]>
 

src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2017-2020, Intel Corporation */
+/* Copyright 2017-2023, Intel Corporation */
 
 #include <immintrin.h>
 #include <stddef.h>
@@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
 	flush64b(dest + 0 * 64);
 }
 
-static force_inline void
+static void
 memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
 		flush_fn flush, flush64b_fn flush64b)
 {
@@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
 		memmove_small_avx512f(dest, src, len, flush);
 }
 
-static force_inline void
+static void
 memmove_mov_avx512f_bw(char *dest, const char *src, size_t len,
 		flush_fn flush, flush64b_fn flush64b)
 {

src/stats/stack-usage-debug.txt

@@ -8,16 +8,10 @@
 51344 memmove_movnt_avx512f_clflush : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
 51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
 51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
-50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+25344 memmove_mov_avx512f_fw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+25344 memmove_mov_avx512f_fw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+25344 memmove_mov_avx512f_bw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+25344 memmove_mov_avx512f_bw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
 24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
 24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
 24144 memmove_movnt_avx_noflush_nobarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
@@ -952,6 +946,16 @@
 112 obj_open_common : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
 112 obj_norep_memmove : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
 112 obj_norep_memcpy : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
+112 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
+112 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
 112 memblock_run_bitmap : src/debug/libpmemobj/memblock.su:memblock.c dynamic,bounded
 112 list_remove_single : src/debug/libpmemobj/list.su:list.c dynamic,bounded
 112 lane_boot : src/debug/libpmemobj/lane.su:lane.c static

src/stats/stack-usage-nondebug.txt

@@ -10,8 +10,6 @@
 8224 pmem2_get_type_from_stat : src/nondebug/libpmemobj/pmem2_utils_linux.su:pmem2_utils_linux.c static
 8224 pmem2_get_type_from_stat : src/nondebug/libpmem2/pmem2_utils_linux.su:pmem2_utils_linux.c static
 8224 pmem2_get_type_from_stat : src/nondebug/common/pmem2_utils_linux.su:pmem2_utils_linux.c static
-7424 memmove_mov_avx512f_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-7424 memmove_mov_avx512f_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
 4352 pool_params_parse.isra.2.constprop : src/nondebug/libpmempool/pool.su:pool.c dynamic,bounded
 4272 ndctl_match_fsdax : src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c static
 4272 ndctl_match_fsdax : src/nondebug/libpmempool/region_namespace_ndctl.su:region_namespace_ndctl.c static
@@ -420,6 +418,10 @@
 128 memset_mov_avx_clflush : src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c static
 128 memset_mov_avx_clflushopt : src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c static
 128 memset_mov_avx_clflushopt : src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c static
+128 memmove_mov_avx512f_fw : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+128 memmove_mov_avx512f_fw : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+128 memmove_mov_avx512f_bw : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+128 memmove_mov_avx512f_bw : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
 128 memblock_header_legacy_reinit : src/nondebug/libpmemobj/memblock.su:memblock.c static
 128 memblock_from_offset_opt : src/nondebug/libpmemobj/memblock.su:memblock.c static
 128 list_insert_new_user : src/nondebug/libpmemobj/list.su:list.c dynamic,bounded
@@ -746,14 +748,6 @@
 64 memmove_mov_avx_clflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static
 64 memmove_mov_avx_clflushopt : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static
 64 memmove_mov_avx_clflushopt : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static
-64 memmove_mov_avx512f_noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
-64 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
 64 heap_set_arena_auto : src/nondebug/libpmemobj/heap.su:heap.c static
 64 heap_create_alloc_class_buckets : src/nondebug/libpmemobj/heap.su:heap.c static
 64 heap_arena_new : src/nondebug/libpmemobj/heap.su:heap.c static
@@ -1528,6 +1522,16 @@
 16 obj_get_nlanes : src/nondebug/libpmemobj/obj.su:obj.c static
 16 obj_fini : src/nondebug/libpmemobj/obj.su:obj.c static
 16 obj_ctl_init_and_load : src/nondebug/libpmemobj/obj.su:obj.c static
+16 memmove_mov_avx512f_noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
+16 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
 16 memblock_rebuild_state : src/nondebug/libpmemobj/memblock.su:memblock.c static
 16 memblock_header_legacy_get_size : src/nondebug/libpmemobj/memblock.su:memblock.c static
 16 memblock_header_legacy_get_flags : src/nondebug/libpmemobj/memblock.su:memblock.c static
@@ -1850,6 +1854,12 @@
 8 pmem_emit_log : src/nondebug/libpmem/pmem.su:pmem.c static
 8 pmem_drain : src/nondebug/libpmem/pmem.su:pmem.c static
 8 pmem_deep_drain : src/nondebug/libpmem/pmem.su:pmem.c static
+8 pmem_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 pmem_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
+8 pmem_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 pmem_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
+8 pmem_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 pmem_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
 8 pmem2_vm_reservation_get_size : src/nondebug/libpmem2/vm_reservation.su:vm_reservation.c static
 8 pmem2_vm_reservation_get_address : src/nondebug/libpmem2/vm_reservation.su:vm_reservation.c static
 8 pmem2_region_get_next_badblock : src/nondebug/libpmempool/badblocks_ndctl.su:badblocks_ndctl.c static
@@ -2252,6 +2262,10 @@
 8 obj_norep_memcpy : src/nondebug/libpmemobj/obj.su:obj.c static
 8 obj_norep_drain : src/nondebug/libpmemobj/obj.su:obj.c static
 8 obj_drain_empty : src/nondebug/libpmemobj/obj.su:obj.c static
+8 noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h static
+8 noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h static
+8 noflush64b : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h static
+8 noflush64b : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h static
 8 memory_barrier : src/nondebug/libpmem/init.su:init.c static
 8 memory_barrier : src/nondebug/libpmem2/init.su:init.c static
 8 membuf_ptr_user_data : src/nondebug/libpmempool/membuf.su:membuf.c static
@@ -2302,12 +2316,22 @@
 8 hash64 : src/nondebug/libpmemobj/rand.su:rand.c static
 8 hash64 : src/nondebug/common/rand.su:rand.c static
 8 flush_empty : src/nondebug/libpmem/pmem.su:pmem.c static
+8 flush_empty_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:pmem2_arch.h static
+8 flush_empty_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:pmem2_arch.h static
 8 flush_clwb : src/nondebug/libpmem/init.su:init.c static
 8 flush_clwb : src/nondebug/libpmem2/init.su:init.c static
+8 flush_clwb_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 flush_clwb_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
 8 flush_clflush : src/nondebug/libpmem/init.su:init.c static
 8 flush_clflush : src/nondebug/libpmem2/init.su:init.c static
 8 flush_clflushopt : src/nondebug/libpmem/init.su:init.c static
 8 flush_clflushopt : src/nondebug/libpmem2/init.su:init.c static
+8 flush_clflushopt_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 flush_clflushopt_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
+8 flush_clflush_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 flush_clflush_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
+8 flush64b_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static
+8 flush64b_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static
 8 fence_empty : src/nondebug/libpmem/pmem.su:pmem.c static
 8 debug_ctl_register : src/nondebug/libpmemobj/ctl_debug.su:ctl_debug.c static
 8 ctl__worker_read : src/nondebug/libpmemobj/tx.su:tx.c static