88#include <linux/ftrace.h>
99#include <linux/uaccess.h>
1010#include <linux/memory.h>
11+ #include <linux/stop_machine.h>
1112#include <asm/cacheflush.h>
1213#include <asm/patch.h>
1314
@@ -75,8 +76,7 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
7576 make_call_t0 (hook_pos , target , call );
7677
7778 /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */
78- if (patch_text_nosync
79- ((void * )hook_pos , enable ? call : nops , MCOUNT_INSN_SIZE ))
79+ if (patch_insn_write ((void * )hook_pos , enable ? call : nops , MCOUNT_INSN_SIZE ))
8080 return - EPERM ;
8181
8282 return 0 ;
@@ -88,7 +88,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
8888
8989 make_call_t0 (rec -> ip , addr , call );
9090
91- if (patch_text_nosync ((void * )rec -> ip , call , MCOUNT_INSN_SIZE ))
91+ if (patch_insn_write ((void * )rec -> ip , call , MCOUNT_INSN_SIZE ))
9292 return - EPERM ;
9393
9494 return 0 ;
@@ -99,7 +99,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
9999{
100100 unsigned int nops [2 ] = {NOP4 , NOP4 };
101101
102- if (patch_text_nosync ((void * )rec -> ip , nops , MCOUNT_INSN_SIZE ))
102+ if (patch_insn_write ((void * )rec -> ip , nops , MCOUNT_INSN_SIZE ))
103103 return - EPERM ;
104104
105105 return 0 ;
@@ -120,6 +120,9 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
120120 out = ftrace_make_nop (mod , rec , MCOUNT_ADDR );
121121 mutex_unlock (& text_mutex );
122122
123+ if (!mod )
124+ local_flush_icache_range (rec -> ip , rec -> ip + MCOUNT_INSN_SIZE );
125+
123126 return out ;
124127}
125128
@@ -134,6 +137,42 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
134137
135138 return ret ;
136139}
140+
141+ struct ftrace_modify_param {
142+ int command ;
143+ atomic_t cpu_count ;
144+ };
145+
146+ static int __ftrace_modify_code (void * data )
147+ {
148+ struct ftrace_modify_param * param = data ;
149+
150+ if (atomic_inc_return (& param -> cpu_count ) == num_online_cpus ()) {
151+ ftrace_modify_all_code (param -> command );
152+ /*
153+ * Make sure the patching store is effective *before* we
154+ * increment the counter which releases all waiting CPUs
155+ * by using the release variant of atomic increment. The
156+ * release pairs with the call to local_flush_icache_all()
157+ * on the waiting CPU.
158+ */
159+ atomic_inc_return_release (& param -> cpu_count );
160+ } else {
161+ while (atomic_read (& param -> cpu_count ) <= num_online_cpus ())
162+ cpu_relax ();
163+ }
164+
165+ local_flush_icache_all ();
166+
167+ return 0 ;
168+ }
169+
170+ void arch_ftrace_update_code (int command )
171+ {
172+ struct ftrace_modify_param param = { command , ATOMIC_INIT (0 ) };
173+
174+ stop_machine (__ftrace_modify_code , & param , cpu_online_mask );
175+ }
137176#endif
138177
139178#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
0 commit comments