Skip to content

[AArch64][SVE] Tweak how SVE CFI expressions are emitted #151677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MacDue merged 4 commits into from
Aug 6, 2025
Merged

[AArch64][SVE] Tweak how SVE CFI expressions are emitted #151677

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d136e85
[AArch64][SVE] Tweak how SVE CFI expressions are emitted
MacDue Jul 31, 2025
214b2ab
Fixups
MacDue Aug 4, 2025
6a193fe
Fixups
MacDue Aug 4, 2025
142c8ec
Fixups
MacDue Aug 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions llvm/include/llvm/Support/LEB128.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,23 @@ inline uint64_t decodeULEB128AndIncUnsafe(const uint8_t *&p) {
return decodeULEB128AndInc(p, nullptr);
}

enum class LEB128Sign { Unsigned, Signed };

template <LEB128Sign Sign, typename T, typename U = char,
unsigned MaxLEB128SizeBytes = 16>
inline void appendLEB128(SmallVectorImpl<U> &Buffer, T Value) {
static_assert(sizeof(U) == 1, "Expected buffer of bytes");
unsigned LEB128ValueSize;
U TmpBuffer[MaxLEB128SizeBytes];
if constexpr (Sign == LEB128Sign::Signed)
LEB128ValueSize =
encodeSLEB128(Value, reinterpret_cast<uint8_t *>(TmpBuffer));
else
LEB128ValueSize =
encodeULEB128(Value, reinterpret_cast<uint8_t *>(TmpBuffer));
Buffer.append(TmpBuffer, TmpBuffer + LEB128ValueSize);
}

/// Utility function to get the size of the ULEB128-encoded value.
LLVM_ABI extern unsigned getULEB128Size(uint64_t Value);

Expand Down
94 changes: 59 additions & 35 deletions llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5861,33 +5861,42 @@ void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
}
}

// Convenience function to create a DWARF expression for
// Expr + NumBytes + NumVGScaledBytes * AArch64::VG
static void appendVGScaledOffsetExpr(SmallVectorImpl<char> &Expr, int NumBytes,
int NumVGScaledBytes, unsigned VG,
llvm::raw_string_ostream &Comment) {
uint8_t buffer[16];

if (NumBytes) {
// Convenience function to create a DWARF expression for: Constant `Operation`.
// This helper emits compact sequences for common cases. For example, for`-15
// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
dwarf::LocationAtom Operation) {
// + -constant (<= 31)
if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
Expr.push_back(dwarf::DW_OP_lit0 - Constant);
Operation = dwarf::DW_OP_minus;
} else if (Constant >= 0 && Constant <= 31) {
// `Op` literal value 0 to 31
Expr.push_back(dwarf::DW_OP_lit0 + Constant);
} else {
// `Op` constant
Expr.push_back(dwarf::DW_OP_consts);
Expr.append(buffer, buffer + encodeSLEB128(NumBytes, buffer));
Expr.push_back((uint8_t)dwarf::DW_OP_plus);
Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
appendLEB128<LEB128Sign::Signed>(Expr, Constant);
}
return Expr.push_back(Operation);
}

if (NumVGScaledBytes) {
Expr.push_back((uint8_t)dwarf::DW_OP_consts);
Expr.append(buffer, buffer + encodeSLEB128(NumVGScaledBytes, buffer));

Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
Expr.append(buffer, buffer + encodeULEB128(VG, buffer));
Expr.push_back(0);

Expr.push_back((uint8_t)dwarf::DW_OP_mul);
Expr.push_back((uint8_t)dwarf::DW_OP_plus);
// Convenience function to create a DWARF expression for VG
static void appendReadVGRegExpr(SmallVectorImpl<char> &Expr,
unsigned VGRegNum) {
Copy link
Contributor

@gbossu gbossu Aug 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Curious: Why not just appendVGExpr? Does a "write" expression exist?

Copy link
Member Author

@MacDue MacDue Aug 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In my next patch, I will add appendLoadIncomingVGExpr. I think the more explicit name here makes the distinction more apparent.

Expr.push_back(dwarf::DW_OP_bregx);
appendLEB128<LEB128Sign::Unsigned>(Expr, VGRegNum);
Expr.push_back(0);
}

Comment << (NumVGScaledBytes < 0 ? " - " : " + ")
<< std::abs(NumVGScaledBytes) << " * VG";
// Convenience function to create a comment for
// (+/-) NumBytes (* RegScale)?
static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
StringRef RegScale = {}) {
if (NumBytes) {
Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
if (!RegScale.empty())
Comment << " * " << RegScale;
}
}

Expand All @@ -5909,19 +5918,26 @@ static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
else
Comment << printReg(Reg, &TRI);

// Build up the expression (Reg + NumBytes + NumVGScaledBytes * AArch64::VG)
// Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
SmallString<64> Expr;
unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
assert(DwarfReg >= 0 && DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
// Reg + NumBytes
Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
Expr.push_back(0);
appendVGScaledOffsetExpr(Expr, NumBytes, NumVGScaledBytes,
TRI.getDwarfRegNum(AArch64::VG, true), Comment);
appendLEB128<LEB128Sign::Signed>(Expr, NumBytes);
appendOffsetComment(NumBytes, Comment);
if (NumVGScaledBytes) {
// + VG * NumVGScaledBytes
appendOffsetComment(NumVGScaledBytes, Comment, "VG");
appendReadVGRegExpr(Expr, TRI.getDwarfRegNum(AArch64::VG, true));
appendConstantExpr(Expr, NumVGScaledBytes, dwarf::DW_OP_mul);
Expr.push_back(dwarf::DW_OP_plus);
}

// Wrap this into DW_CFA_def_cfa.
SmallString<64> DefCfaExpr;
DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
uint8_t buffer[16];
DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer));
appendLEB128<LEB128Sign::Unsigned>(DefCfaExpr, Expr.size());
DefCfaExpr.append(Expr.str());
return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
Comment.str());
Expand Down Expand Up @@ -5958,17 +5974,25 @@ MCCFIInstruction llvm::createCFAOffset(const TargetRegisterInfo &TRI,
llvm::raw_string_ostream Comment(CommentBuffer);
Comment << printReg(Reg, &TRI) << " @ cfa";

// Build up expression (NumBytes + NumVGScaledBytes * AArch64::VG)
// Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
assert(NumVGScaledBytes && "Expected scalable offset");
SmallString<64> OffsetExpr;
appendVGScaledOffsetExpr(OffsetExpr, NumBytes, NumVGScaledBytes,
TRI.getDwarfRegNum(AArch64::VG, true), Comment);
// + VG * NumVGScaledBytes
appendOffsetComment(NumVGScaledBytes, Comment, "VG");
appendReadVGRegExpr(OffsetExpr, TRI.getDwarfRegNum(AArch64::VG, true));
appendConstantExpr(OffsetExpr, NumVGScaledBytes, dwarf::DW_OP_mul);
OffsetExpr.push_back(dwarf::DW_OP_plus);
if (NumBytes) {
// + NumBytes
appendOffsetComment(NumBytes, Comment);
appendConstantExpr(OffsetExpr, NumBytes, dwarf::DW_OP_plus);
}

// Wrap this into DW_CFA_expression
SmallString<64> CfaExpr;
CfaExpr.push_back(dwarf::DW_CFA_expression);
uint8_t buffer[16];
CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
CfaExpr.append(buffer, buffer + encodeULEB128(OffsetExpr.size(), buffer));
appendLEB128<LEB128Sign::Unsigned>(CfaExpr, DwarfReg);
appendLEB128<LEB128Sign::Unsigned>(CfaExpr, OffsetExpr.size());
CfaExpr.append(OffsetExpr.str());

return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), SMLoc(),
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ define void @array_1D(ptr %addr) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #2, mul vl]
Expand All @@ -34,7 +34,7 @@ define %my_subtype @array_1D_extract(ptr %addr) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #3
Expand All @@ -52,7 +52,7 @@ define void @array_1D_insert(ptr %addr, %my_subtype %elt) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldr z1, [x0, #2, mul vl]
; CHECK-NEXT: ldr z2, [x0]
Expand All @@ -75,7 +75,7 @@ define void @array_2D(ptr %addr) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-6
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #5, mul vl]
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ define void @test(ptr %addr) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #2, mul vl]
Expand Down
12 changes: 6 additions & 6 deletions llvm/test/CodeGen/AArch64/fp8-sme2-cvtn.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @cvtn_f16_tuple(i64 %stride, p
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str z11, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: str z10, [sp, #2, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 8 * VG - 16
; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 16 * VG - 16
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x8, x1, x0
; CHECK-NEXT: ld1h { z2.h, z10.h }, pn8/z, [x1]
Expand Down Expand Up @@ -52,10 +52,10 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @cvtnt_f32_tuple(i64 %stride,
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str z11, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: str z10, [sp, #2, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 8 * VG
; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 16 * VG
; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 8 * VG - 16
; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 16 * VG - 16
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x8, x1, x0
; CHECK-NEXT: mov z1.d, z0.d
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,16 @@
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: addvl sp, sp, #-2
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
; CHECK-NEXT: // implicit-def: $z8
; CHECK-NEXT: // implicit-def: $p4
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: addvl sp, sp, #2
Expand Down
Loading