Skip to content

Commit 19e5eb1

Browse files
chenhuacaichenhuacai
committed
LoongArch: Add alternative runtime patching mechanism
Introduce the "alternative" mechanism from ARM64 and x86 for LoongArch to apply runtime patching. The main purpose of this patch is to provide a framework. In future we can use this mechanism (i.e., the ALTERNATIVE and ALTERNATIVE_2 macros) to optimize hotspot functions according to cpu features. Signed-off-by: Jun Yi <[email protected]> Signed-off-by: Huacai Chen <[email protected]>
1 parent 61a6fcc commit 19e5eb1

File tree

9 files changed

+507
-1
lines changed

9 files changed

+507
-1
lines changed

‎arch/loongarch/include/asm/alternative-asm.h

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef _ASM_ALTERNATIVE_ASM_H
3+
#define _ASM_ALTERNATIVE_ASM_H
4+
5+
#ifdef __ASSEMBLY__
6+
7+
#include <asm/asm.h>
8+
9+
/*
10+
* Issue one struct alt_instr descriptor entry (need to put it into
11+
* the section .altinstructions, see below). This entry contains
12+
* enough information for the alternatives patching code to patch an
13+
* instruction. See apply_alternatives().
14+
*/
15+
.macro altinstruction_entry orig alt feature orig_len alt_len
16+
.long \orig - .
17+
.long \alt - .
18+
.short \feature
19+
.byte \orig_len
20+
.byte \alt_len
21+
.endm
22+
23+
/*
24+
* Define an alternative between two instructions. If @feature is
25+
* present, early code in apply_alternatives() replaces @oldinstr with
26+
* @newinstr. ".fill" directive takes care of proper instruction padding
27+
* in case @newinstr is longer than @oldinstr.
28+
*/
29+
.macro ALTERNATIVE oldinstr, newinstr, feature
30+
140 :
31+
\oldinstr
32+
141 :
33+
.fill - (((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)) / 4, 4, 0x03400000
34+
142 :
35+
36+
.pushsection .altinstructions, "a"
37+
altinstruction_entry 140b, 143f, \feature, 142b-140b, 144f-143f
38+
.popsection
39+
40+
.subsection 1
41+
143 :
42+
\newinstr
43+
144 :
44+
.previous
45+
.endm
46+
47+
#define old_len (141b-140b)
48+
#define new_len1 (144f-143f)
49+
#define new_len2 (145f-144f)
50+
51+
#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
52+
53+
/*
54+
* Same as ALTERNATIVE macro above but for two alternatives. If CPU
55+
* has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
56+
* @feature2, it replaces @oldinstr with @feature2.
57+
*/
58+
.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
59+
140 :
60+
\oldinstr
61+
141 :
62+
.fill - ((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \
63+
(alt_max_short(new_len1, new_len2) - (old_len)) / 4, 4, 0x03400000
64+
142 :
65+
66+
.pushsection .altinstructions, "a"
67+
altinstruction_entry 140b, 143f, \feature1, 142b-140b, 144f-143f, 142b-141b
68+
altinstruction_entry 140b, 144f, \feature2, 142b-140b, 145f-144f, 142b-141b
69+
.popsection
70+
71+
.subsection 1
72+
143 :
73+
\newinstr1
74+
144 :
75+
\newinstr2
76+
145 :
77+
.previous
78+
.endm
79+
80+
#endif /* __ASSEMBLY__ */
81+
82+
#endif /* _ASM_ALTERNATIVE_ASM_H */

‎arch/loongarch/include/asm/alternative.h

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef _ASM_ALTERNATIVE_H
3+
#define _ASM_ALTERNATIVE_H
4+
5+
#ifndef __ASSEMBLY__
6+
7+
#include <linux/types.h>
8+
#include <linux/stddef.h>
9+
#include <linux/stringify.h>
10+
#include <asm/asm.h>
11+
12+
struct alt_instr {
13+
s32 instr_offset; /* offset to original instruction */
14+
s32 replace_offset; /* offset to replacement instruction */
15+
u16 feature; /* feature bit set for replacement */
16+
u8 instrlen; /* length of original instruction */
17+
u8 replacementlen; /* length of new instruction */
18+
} __packed;
19+
20+
/*
21+
* Debug flag that can be tested to see whether alternative
22+
* instructions were patched in already:
23+
*/
24+
extern int alternatives_patched;
25+
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
26+
27+
extern void alternative_instructions(void);
28+
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
29+
30+
#define b_replacement(num) "664"#num
31+
#define e_replacement(num) "665"#num
32+
33+
#define alt_end_marker "663"
34+
#define alt_slen "662b-661b"
35+
#define alt_total_slen alt_end_marker"b-661b"
36+
#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
37+
38+
#define __OLDINSTR(oldinstr, num) \
39+
"661:\n\t" oldinstr "\n662:\n" \
40+
".fill -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
41+
"((" alt_rlen(num) ")-(" alt_slen ")) / 4, 4, 0x03400000\n"
42+
43+
#define OLDINSTR(oldinstr, num) \
44+
__OLDINSTR(oldinstr, num) \
45+
alt_end_marker ":\n"
46+
47+
#define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))"
48+
49+
/*
50+
* Pad the second replacement alternative with additional NOPs if it is
51+
* additionally longer than the first replacement alternative.
52+
*/
53+
#define OLDINSTR_2(oldinstr, num1, num2) \
54+
"661:\n\t" oldinstr "\n662:\n" \
55+
".fill -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
56+
"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) / 4, " \
57+
"4, 0x03400000\n" \
58+
alt_end_marker ":\n"
59+
60+
#define ALTINSTR_ENTRY(feature, num) \
61+
" .long 661b - .\n" /* label */ \
62+
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
63+
" .short " __stringify(feature) "\n" /* feature bit */ \
64+
" .byte " alt_total_slen "\n" /* source len */ \
65+
" .byte " alt_rlen(num) "\n" /* replacement len */
66+
67+
#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
68+
b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
69+
70+
/* alternative assembly primitive: */
71+
#define ALTERNATIVE(oldinstr, newinstr, feature) \
72+
OLDINSTR(oldinstr, 1) \
73+
".pushsection .altinstructions,\"a\"\n" \
74+
ALTINSTR_ENTRY(feature, 1) \
75+
".popsection\n" \
76+
".subsection 1\n" \
77+
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
78+
".previous\n"
79+
80+
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
81+
OLDINSTR_2(oldinstr, 1, 2) \
82+
".pushsection .altinstructions,\"a\"\n" \
83+
ALTINSTR_ENTRY(feature1, 1) \
84+
ALTINSTR_ENTRY(feature2, 2) \
85+
".popsection\n" \
86+
".subsection 1\n" \
87+
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
88+
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
89+
".previous\n"
90+
91+
/*
92+
* Alternative instructions for different CPU types or capabilities.
93+
*
94+
* This allows to use optimized instructions even on generic binary
95+
* kernels.
96+
*
97+
* length of oldinstr must be longer or equal the length of newinstr
98+
* It can be padded with nops as needed.
99+
*
100+
* For non barrier like inlines please define new variants
101+
* without volatile and memory clobber.
102+
*/
103+
#define alternative(oldinstr, newinstr, feature) \
104+
(asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory"))
105+
106+
#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
107+
(asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory"))
108+
109+
#endif /* __ASSEMBLY__ */
110+
111+
#endif /* _ASM_ALTERNATIVE_H */

‎arch/loongarch/include/asm/bugs.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* This is included by init/main.c to check for architecture-dependent bugs.
4+
*
5+
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6+
*/
7+
#ifndef _ASM_BUGS_H
8+
#define _ASM_BUGS_H
9+
10+
#include <asm/cpu.h>
11+
#include <asm/cpu-info.h>
12+
13+
extern void check_bugs(void);
14+
15+
#endif /* _ASM_BUGS_H */

‎arch/loongarch/include/asm/inst.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <linux/types.h>
99
#include <asm/asm.h>
1010

11+
#define INSN_NOP 0x03400000
1112
#define INSN_BREAK 0x002a0000
1213

1314
#define ADDR_IMMMASK_LU52ID 0xFFF0000000000000
@@ -28,13 +29,16 @@ enum reg0i26_op {
2829
enum reg1i20_op {
2930
lu12iw_op = 0x0a,
3031
lu32id_op = 0x0b,
32+
pcaddi_op = 0x0c,
3133
pcaddu12i_op = 0x0e,
3234
pcaddu18i_op = 0x0f,
3335
};
3436

3537
enum reg1i21_op {
3638
beqz_op = 0x10,
3739
bnez_op = 0x11,
40+
bceqz_op = 0x12, /* bits[9:8] = 0x00 */
41+
bcnez_op = 0x12, /* bits[9:8] = 0x01 */
3842
};
3943

4044
enum reg2_op {
@@ -315,6 +319,12 @@ static inline bool is_imm_negative(unsigned long val, unsigned int bit)
315319
return val & (1UL << (bit - 1));
316320
}
317321

322+
static inline bool is_pc_ins(union loongarch_instruction *ip)
323+
{
324+
return ip->reg1i20_format.opcode >= pcaddi_op &&
325+
ip->reg1i20_format.opcode <= pcaddu18i_op;
326+
}
327+
318328
static inline bool is_branch_ins(union loongarch_instruction *ip)
319329
{
320330
return ip->reg1i21_format.opcode >= beqz_op &&
@@ -353,6 +363,14 @@ static inline bool unsigned_imm_check(unsigned long val, unsigned int bit)
353363
return val < (1UL << bit);
354364
}
355365

366+
static inline unsigned long sign_extend(unsigned long val, unsigned int idx)
367+
{
368+
if (!is_imm_negative(val, idx + 1))
369+
return ((1UL << idx) - 1) & val;
370+
else
371+
return ~((1UL << idx) - 1) | val;
372+
}
373+
356374
#define DEF_EMIT_REG0I26_FORMAT(NAME, OP) \
357375
static inline void emit_##NAME(union loongarch_instruction *insn, \
358376
int offset) \

‎arch/loongarch/kernel/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ extra-y := vmlinux.lds
88
obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \
99
traps.o irq.o idle.o process.o dma.o mem.o io.o reset.o switch.o \
1010
elf.o syscall.o signal.o time.o topology.o inst.o ptrace.o vdso.o \
11-
unaligned.o
11+
alternative.o unaligned.o
1212

1313
obj-$(CONFIG_ACPI) += acpi.o
1414
obj-$(CONFIG_EFI) += efi.o

0 commit comments

Comments
 (0)