Skip to content

Commit d90f0bc

Browse files
author
Alexei Starovoitov
committed
Merge branch 's390-bpf-use-kernel-s-expoline-thunks'
Ilya Leoshkevich says: ==================== This series simplifies the s390 JIT by replacing the generation of expolines (Spectre mitigation) with using the ones from the kernel text. This is possible thanks to the V!=R s390 kernel rework. Patch 1 is a small prerequisite for arch/s390 that I would like to get in via the BPF tree. It has Heiko's Acked-by. Patches 2 and 3 are the implementation. ==================== Link: https://patch.msgid.link/[email protected] Signed-off-by: Alexei Starovoitov <[email protected]>
2 parents 4e2e684 + 7f332f9 commit d90f0bc

File tree

2 files changed

+59
-66
lines changed

2 files changed

+59
-66
lines changed

‎arch/s390/include/asm/nospec-branch.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ static inline bool nospec_uses_trampoline(void)
2626
return __is_defined(CC_USING_EXPOLINE) && !nospec_disable;
2727
}
2828

29-
#ifdef CONFIG_EXPOLINE_EXTERN
30-
3129
void __s390_indirect_jump_r1(void);
3230
void __s390_indirect_jump_r2(void);
3331
void __s390_indirect_jump_r3(void);
@@ -44,8 +42,6 @@ void __s390_indirect_jump_r13(void);
4442
void __s390_indirect_jump_r14(void);
4543
void __s390_indirect_jump_r15(void);
4644

47-
#endif
48-
4945
#endif /* __ASSEMBLY__ */
5046

5147
#endif /* _ASM_S390_EXPOLINE_H */

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 59 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ struct bpf_jit {
4848
int lit64; /* Current position in 64-bit literal pool */
4949
int base_ip; /* Base address for literal pool */
5050
int exit_ip; /* Address of exit */
51-
int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
52-
int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
5351
int tail_call_start; /* Tail call start offset */
5452
int excnt; /* Number of exception table entries */
5553
int prologue_plt_ret; /* Return address for prologue hotpatch PLT */
@@ -127,6 +125,18 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
127125
jit->seen_regs |= (1 << r1);
128126
}
129127

128+
static s32 off_to_pcrel(struct bpf_jit *jit, u32 off)
129+
{
130+
return off - jit->prg;
131+
}
132+
133+
static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr)
134+
{
135+
if (jit->prg_buf)
136+
return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg);
137+
return 0;
138+
}
139+
130140
#define REG_SET_SEEN(b1) \
131141
({ \
132142
reg_set_seen(jit, b1); \
@@ -201,7 +211,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
201211

202212
#define EMIT4_PCREL_RIC(op, mask, target) \
203213
({ \
204-
int __rel = ((target) - jit->prg) / 2; \
214+
int __rel = off_to_pcrel(jit, target) / 2; \
205215
_EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
206216
})
207217

@@ -239,7 +249,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
239249

240250
#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \
241251
({ \
242-
unsigned int rel = (int)((target) - jit->prg) / 2; \
252+
unsigned int rel = off_to_pcrel(jit, target) / 2; \
243253
_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
244254
(op2) | (mask) << 12); \
245255
REG_SET_SEEN(b1); \
@@ -248,7 +258,7 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
248258

249259
#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \
250260
({ \
251-
unsigned int rel = (int)((target) - jit->prg) / 2; \
261+
unsigned int rel = off_to_pcrel(jit, target) / 2; \
252262
_EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \
253263
(rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
254264
REG_SET_SEEN(b1); \
@@ -257,29 +267,41 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
257267

258268
#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
259269
({ \
260-
int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
270+
int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\
261271
_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
262272
REG_SET_SEEN(b1); \
263273
REG_SET_SEEN(b2); \
264274
})
265275

276+
static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel)
277+
{
278+
u32 pc32dbl = (s32)(pcrel / 2);
279+
280+
_EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff);
281+
}
282+
283+
static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel)
284+
{
285+
emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel);
286+
REG_SET_SEEN(b);
287+
}
288+
266289
#define EMIT6_PCREL_RILB(op, b, target) \
267-
({ \
268-
unsigned int rel = (int)((target) - jit->prg) / 2; \
269-
_EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\
270-
REG_SET_SEEN(b); \
271-
})
290+
emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target))
272291

273-
#define EMIT6_PCREL_RIL(op, target) \
274-
({ \
275-
unsigned int rel = (int)((target) - jit->prg) / 2; \
276-
_EMIT6((op) | rel >> 16, rel & 0xffff); \
277-
})
292+
#define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \
293+
emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr))
294+
295+
static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel)
296+
{
297+
emit6_pcrel_ril(jit, op | mask << 20, pcrel);
298+
}
278299

279300
#define EMIT6_PCREL_RILC(op, mask, target) \
280-
({ \
281-
EMIT6_PCREL_RIL((op) | (mask) << 20, (target)); \
282-
})
301+
emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target))
302+
303+
#define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \
304+
emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr))
283305

284306
#define _EMIT6_IMM(op, imm) \
285307
({ \
@@ -503,7 +525,7 @@ static void bpf_skip(struct bpf_jit *jit, int size)
503525
{
504526
if (size >= 6 && !is_valid_rel(size)) {
505527
/* brcl 0xf,size */
506-
EMIT6_PCREL_RIL(0xc0f4000000, size);
528+
EMIT6_PCREL_RILC(0xc0040000, 0xf, size);
507529
size -= 6;
508530
} else if (size >= 4 && is_valid_rel(size)) {
509531
/* brc 0xf,size */
@@ -618,28 +640,17 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
618640
}
619641

620642
/*
621-
* Emit an expoline for a jump that follows
643+
* Jump using a register either directly or via an expoline thunk
622644
*/
623-
static void emit_expoline(struct bpf_jit *jit)
624-
{
625-
/* exrl %r0,.+10 */
626-
EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
627-
/* j . */
628-
EMIT4_PCREL(0xa7f40000, 0);
629-
}
630-
631-
/*
632-
* Emit __s390_indirect_jump_r1 thunk if necessary
633-
*/
634-
static void emit_r1_thunk(struct bpf_jit *jit)
635-
{
636-
if (nospec_uses_trampoline()) {
637-
jit->r1_thunk_ip = jit->prg;
638-
emit_expoline(jit);
639-
/* br %r1 */
640-
_EMIT2(0x07f1);
641-
}
642-
}
645+
#define EMIT_JUMP_REG(reg) do { \
646+
if (nospec_uses_trampoline()) \
647+
/* brcl 0xf,__s390_indirect_jump_rN */ \
648+
EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \
649+
__s390_indirect_jump_r ## reg); \
650+
else \
651+
/* br %rN */ \
652+
_EMIT2(0x07f0 | reg); \
653+
} while (0)
643654

644655
/*
645656
* Call r1 either directly or via __s390_indirect_jump_r1 thunk
@@ -648,7 +659,8 @@ static void call_r1(struct bpf_jit *jit)
648659
{
649660
if (nospec_uses_trampoline())
650661
/* brasl %r14,__s390_indirect_jump_r1 */
651-
EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
662+
EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14,
663+
__s390_indirect_jump_r1);
652664
else
653665
/* basr %r14,%r1 */
654666
EMIT2(0x0d00, REG_14, REG_1);
@@ -664,16 +676,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
664676
EMIT4(0xb9040000, REG_2, BPF_REG_0);
665677
/* Restore registers */
666678
save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);
667-
if (nospec_uses_trampoline()) {
668-
jit->r14_thunk_ip = jit->prg;
669-
/* Generate __s390_indirect_jump_r14 thunk */
670-
emit_expoline(jit);
671-
}
672-
/* br %r14 */
673-
_EMIT2(0x07fe);
674-
675-
if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
676-
emit_r1_thunk(jit);
679+
EMIT_JUMP_REG(14);
677680

678681
jit->prg = ALIGN(jit->prg, 8);
679682
jit->prologue_plt = jit->prg;
@@ -1875,7 +1878,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
18751878
/* aghi %r1,tail_call_start */
18761879
EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);
18771880
/* brcl 0xf,__s390_indirect_jump_r1 */
1878-
EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);
1881+
EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf,
1882+
__s390_indirect_jump_r1);
18791883
} else {
18801884
/* bc 0xf,tail_call_start(%r1) */
18811885
_EMIT4(0x47f01000 + jit->tail_call_start);
@@ -2844,17 +2848,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
28442848
0xf000 | tjit->tccnt_off);
28452849
/* aghi %r15,stack_size */
28462850
EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);
2847-
/* Emit an expoline for the following indirect jump. */
2848-
if (nospec_uses_trampoline())
2849-
emit_expoline(jit);
28502851
if (flags & BPF_TRAMP_F_SKIP_FRAME)
2851-
/* br %r14 */
2852-
_EMIT2(0x07fe);
2852+
EMIT_JUMP_REG(14);
28532853
else
2854-
/* br %r1 */
2855-
_EMIT2(0x07f1);
2856-
2857-
emit_r1_thunk(jit);
2854+
EMIT_JUMP_REG(1);
28582855

28592856
return 0;
28602857
}

0 commit comments

Comments
 (0)