From dae7df3d49e1cbb1825b6f7110567917b74ecb50 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 1 Oct 2024 09:19:52 -0700 Subject: [PATCH] [BPF] Use mul for certain div/mod operations The motivation example likes below $ cat t1.c struct S { int var[3]; }; int foo1 (struct S *a, struct S *b) { return a - b; } For cpu v1/v2/v3, the compilation will fail with the following error: $ clang --target=bpf -O2 -c t1.c -mcpu=v3 t1.c:4:5: error: unsupported signed division, please convert to unsigned div/mod. 4 | int foo1 (struct S *a, struct S *b) | ^ 1 error generated. The reason is that sdiv/smod is only supported at -mcpu=v4. At cpu v1/v2/v3, only udiv/umod is supported. But the above example (for func foo1()) is reasonable common and user has to workaround the compilation failure by using udiv with conditionals. For x86, for the above t1.c, compile and dump the asm code like below: $ clang -O2 -c t1.c && llvm-objdump -d t1.o 0000000000000000 : 0: 48 29 f7 subq %rsi, %rdi 3: 48 c1 ef 02 shrq $0x2, %rdi 7: 69 c7 ab aa aa aa imull $0xaaaaaaab, %edi, %eax # imm = 0xAAAAAAAB d: c3 retq Basically sdiv can be replaced with sub, shr and imul. Latest gcc-bpf is also able to generate code similar to x86 with -mcpu=v1. See https://godbolt.org/z/feP9ETbjj Generally multiplication is cheaper than divide. LLVM SelectionDAG already supports to generate alternative codes (mul etc.) for div/mod operations if the divisor is constant and if the cost of division is not cheap. For BPF backend, let us use multiplication instead of divide whenever possible. The following is the list of div/mod operations which can be converted to mul. - 32bit signed/unsigned div/mod - 64bit signed/unsigned div with 'exact' flag in IR where 'exact' means if replacing div with mod, it is guaranteed that modulo result will be 0. The above foo1() case belongs here. --- llvm/lib/Target/BPF/BPFISelLowering.h | 4 +- llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll | 4 +- llvm/test/CodeGen/BPF/sdiv_error.ll | 6 +-- llvm/test/CodeGen/BPF/sdiv_to_mul.ll | 57 ++++++++++++++++++++++ llvm/test/CodeGen/BPF/srem_error.ll | 6 +-- 5 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 llvm/test/CodeGen/BPF/sdiv_to_mul.ll diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h index 42707949e864c..d59098f9f569b 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/llvm/lib/Target/BPF/BPFISelLowering.h @@ -118,7 +118,9 @@ class BPFTargetLowering : public TargetLowering { return Op.size() >= 8 ? MVT::i64 : MVT::i32; } - bool isIntDivCheap(EVT VT, AttributeList Attr) const override { return true; } + bool isIntDivCheap(EVT VT, AttributeList Attr) const override { + return false; + } bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override { diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll index 42888ff58f5e2..abec34990044a 100644 --- a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll +++ b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll @@ -200,7 +200,7 @@ entry: define dso_local i32 @div_i(i32 %a) local_unnamed_addr #0 { entry: %div = udiv i32 %a, 15 -; CHECK: w{{[0-9]+}} /= 15 +; CHECK-NOT: w{{[0-9]+}} /= 15 ret i32 %div } @@ -216,7 +216,7 @@ entry: define dso_local i32 @rem_i(i32 %a) local_unnamed_addr #0 { entry: %rem = urem i32 %a, 15 -; CHECK: w{{[0-9]+}} %= 15 +; CHECK-NOT: w{{[0-9]+}} %= 15 ret i32 %rem } diff --git a/llvm/test/CodeGen/BPF/sdiv_error.ll b/llvm/test/CodeGen/BPF/sdiv_error.ll index a5dcc6271224b..05394f3e05de8 100644 --- a/llvm/test/CodeGen/BPF/sdiv_error.ll +++ b/llvm/test/CodeGen/BPF/sdiv_error.ll @@ -2,7 +2,7 @@ ; RUN: FileCheck %s < %t1 ; CHECK: unsupported signed division -define i32 @test(i32 %len) { - %1 = sdiv i32 %len, 15 - ret i32 %1 +define i64 @test(i64 %len) { + %1 = sdiv i64 %len, 15 + ret i64 %1 } diff --git a/llvm/test/CodeGen/BPF/sdiv_to_mul.ll b/llvm/test/CodeGen/BPF/sdiv_to_mul.ll new file mode 100644 index 0000000000000..7bb1b7badac1f --- /dev/null +++ b/llvm/test/CodeGen/BPF/sdiv_to_mul.ll @@ -0,0 +1,57 @@ +; RUN: llc -march=bpfel -mcpu=v1 < %s | FileCheck --check-prefix=CHECK-V1 %s +; RUN: llc -march=bpfel -mcpu=v4 < %s | FileCheck --check-prefix=CHECK-V4 %s + +target triple = "bpf" + +; struct S { +; int var[3]; +; }; +; int foo1 (struct S *a, struct S *b) +; { +; return a - b; +; } +define dso_local i32 @foo1(ptr noundef %a, ptr noundef %b) local_unnamed_addr { +entry: + %sub.ptr.lhs.cast = ptrtoint ptr %a to i64 + %sub.ptr.rhs.cast = ptrtoint ptr %b to i64 + %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast + %sub.ptr.div = sdiv exact i64 %sub.ptr.sub, 12 + %conv = trunc i64 %sub.ptr.div to i32 + ret i32 %conv +} +; CHECK-V1: r0 = r1 +; CHECK-V1: r0 -= r2 +; CHECK-V1: r0 s>>= 2 +; CHECK-V1: r1 = -6148914691236517205 ll +; CHECK-V1: r0 *= r1 +; CHECK-V1: exit + +; CHECK-V4: r0 = r1 +; CHECK-V4: r0 -= r2 +; CHECK-V4: r0 >>= 2 +; CHECK-V4: w0 *= -1431655765 +; CHECK-V4: exit + +define dso_local noundef range(i32 -143165576, 143165577) i32 @foo2(i32 noundef %a) local_unnamed_addr { +entry: + %div = sdiv i32 %a, 15 + ret i32 %div +} +; CHECK-V1-NOT: r[[#]] s/= 15 +; CHECK-V4-NOT: w[[#]] s/= 15 + +define dso_local noundef range(i32 -14, 15) i32 @foo3(i32 noundef %a) local_unnamed_addr { +entry: + %rem = srem i32 %a, 15 + ret i32 %rem +} +; CHECK-V1-NOT: r[[#]] s%= 15 +; CHECK-V4-NOT: w[[#]] s%= 15 + +define dso_local i64 @foo4(i64 noundef %a) local_unnamed_addr { +entry: + %div = udiv exact i64 %a, 15 + ret i64 %div +} +; CHECK-V1-NOT: r[[#]] /= 15 +; CHECK-V4-NOT: w[[#]] /= 15 diff --git a/llvm/test/CodeGen/BPF/srem_error.ll b/llvm/test/CodeGen/BPF/srem_error.ll index 883e72d32dc7e..8245a47b1ddd2 100644 --- a/llvm/test/CodeGen/BPF/srem_error.ll +++ b/llvm/test/CodeGen/BPF/srem_error.ll @@ -2,7 +2,7 @@ ; RUN: FileCheck %s < %t1 ; CHECK: unsupported signed division -define i32 @test(i32 %len) { - %1 = srem i32 %len, 15 - ret i32 %1 +define i64 @test(i64 %len) { + %1 = srem i64 %len, 15 + ret i64 %1 }