[X86][NFC] Generalize the naming of "Retpoline Thunks" and related code to "Indirect Thunks"

There are applications for indirect call/branch thunks other than retpoline for Spectre v2, e.g.,

https://software.intel.com/security-software-guidance/software-guidance/load-value-injection

Therefore it makes sense to refactor X86RetpolineThunks as a more general capability.

Differential Revision: https://reviews.llvm.org/D76810

Author: scottconstable

llvm/lib/Target/X86/CMakeLists.txt

@@ -44,6 +44,7 @@ set(sources
   X86ISelDAGToDAG.cpp
   X86ISelLowering.cpp
   X86IndirectBranchTracking.cpp
+  X86IndirectThunks.cpp
   X86InterleavedAccess.cpp
   X86InsertPrefetch.cpp
   X86InstrFMA3Info.cpp
@@ -58,7 +59,6 @@ set(sources
   X86PadShortFunction.cpp
   X86RegisterBankInfo.cpp
   X86RegisterInfo.cpp
-  X86RetpolineThunks.cpp
   X86SelectionDAGInfo.cpp
   X86ShuffleDecodeConstantPool.cpp
   X86SpeculativeLoadHardening.cpp

llvm/lib/Target/X86/X86.h

@@ -120,7 +120,7 @@ FunctionPass *createX86DomainReassignmentPass();
 FunctionPass *createX86EvexToVexInsts();
 
 /// This pass creates the thunks for the retpoline feature.
-FunctionPass *createX86RetpolineThunksPass();
+FunctionPass *createX86IndirectThunksPass();
 
 /// This pass ensures instructions featuring a memory operand
 /// have distinctive <LineNumber, Discriminator> (with respect to eachother)

llvm/lib/Target/X86/X86FastISel.cpp

@@ -3202,8 +3202,8 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       (CalledFn && CalledFn->hasFnAttribute("no_caller_saved_registers")))
     return false;
 
-  // Functions using retpoline for indirect calls need to use SDISel.
-  if (Subtarget->useRetpolineIndirectCalls())
+  // Functions using thunks for indirect calls need to use SDISel.
+  if (Subtarget->useIndirectThunkCalls())
     return false;
 
   // Handle only C, fastcc, and webkit_js calling conventions for now.

llvm/lib/Target/X86/X86FrameLowering.cpp

@@ -765,10 +765,10 @@ void X86FrameLowering::emitStackProbeCall(MachineFunction &MF,
                                           bool InProlog) const {
   bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
 
-  // FIXME: Add retpoline support and remove this.
-  if (Is64Bit && IsLargeCodeModel && STI.useRetpolineIndirectCalls())
+  // FIXME: Add indirect thunk support and remove this.
+  if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
     report_fatal_error("Emitting stack probe calls on 64-bit with the large "
-                       "code model and retpoline not yet implemented.");
+                       "code model and indirect thunks not yet implemented.");
 
   unsigned CallOp;
   if (Is64Bit)
@@ -2493,9 +2493,9 @@ void X86FrameLowering::adjustForSegmentedStacks(
     // is laid out within 2^31 bytes of each function body, but this seems
     // to be sufficient for JIT.
     // FIXME: Add retpoline support and remove the error here..
-    if (STI.useRetpolineIndirectCalls())
+    if (STI.useIndirectThunkCalls())
       report_fatal_error("Emitting morestack calls on 64-bit with the large "
-                         "code model and retpoline not yet implemented.");
+                         "code model and thunks not yet implemented.");
     BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
         .addReg(X86::RIP)
         .addImm(0)

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

@@ -987,7 +987,7 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
     if (OptLevel != CodeGenOpt::None &&
         // Only do this when the target can fold the load into the call or
         // jmp.
-        !Subtarget->useRetpolineIndirectCalls() &&
+        !Subtarget->useIndirectThunkCalls() &&
         ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
          (N->getOpcode() == X86ISD::TC_RETURN &&
           (Subtarget->is64Bit() ||

llvm/lib/Target/X86/X86ISelLowering.cpp

@@ -30218,8 +30218,8 @@ bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
 }
 
 bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
-  // If the subtarget is using retpolines, we need to not generate jump tables.
-  if (Subtarget.useRetpolineIndirectBranches())
+  // If the subtarget is using thunks, we need to not generate jump tables.
+  if (Subtarget.useIndirectThunkBranches())
     return false;
 
   // Otherwise, fallback on the generic logic.
@@ -31342,22 +31342,22 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
   return BB;
 }
 
-static unsigned getOpcodeForRetpoline(unsigned RPOpc) {
+static unsigned getOpcodeForIndirectThunk(unsigned RPOpc) {
   switch (RPOpc) {
-  case X86::RETPOLINE_CALL32:
+  case X86::INDIRECT_THUNK_CALL32:
     return X86::CALLpcrel32;
-  case X86::RETPOLINE_CALL64:
+  case X86::INDIRECT_THUNK_CALL64:
     return X86::CALL64pcrel32;
-  case X86::RETPOLINE_TCRETURN32:
+  case X86::INDIRECT_THUNK_TCRETURN32:
     return X86::TCRETURNdi;
-  case X86::RETPOLINE_TCRETURN64:
+  case X86::INDIRECT_THUNK_TCRETURN64:
     return X86::TCRETURNdi64;
   }
-  llvm_unreachable("not retpoline opcode");
+  llvm_unreachable("not indirect thunk opcode");
 }
 
-static const char *getRetpolineSymbol(const X86Subtarget &Subtarget,
-                                      unsigned Reg) {
+static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget,
+                                          unsigned Reg) {
   if (Subtarget.useRetpolineExternalThunk()) {
     // When using an external thunk for retpolines, we pick names that match the
     // names GCC happens to use as well. This helps simplify the implementation
@@ -31389,39 +31389,43 @@ static const char *getRetpolineSymbol(const X86Subtarget &Subtarget,
       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
       return "__x86_indirect_thunk_r11";
     }
-    llvm_unreachable("unexpected reg for retpoline");
+    llvm_unreachable("unexpected reg for external indirect thunk");
   }
 
-  // When targeting an internal COMDAT thunk use an LLVM-specific name.
-  switch (Reg) {
-  case X86::EAX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_eax";
-  case X86::ECX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_ecx";
-  case X86::EDX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_edx";
-  case X86::EDI:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_edi";
-  case X86::R11:
-    assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
-    return "__llvm_retpoline_r11";
+  if (Subtarget.useRetpolineIndirectCalls() ||
+      Subtarget.useRetpolineIndirectBranches()) {
+    // When targeting an internal COMDAT thunk use an LLVM-specific name.
+    switch (Reg) {
+    case X86::EAX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_eax";
+    case X86::ECX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_ecx";
+    case X86::EDX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_edx";
+    case X86::EDI:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_edi";
+    case X86::R11:
+      assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
+      return "__llvm_retpoline_r11";
+    }
+    llvm_unreachable("unexpected reg for retpoline");
   }
-  llvm_unreachable("unexpected reg for retpoline");
+  llvm_unreachable("getIndirectThunkSymbol() invoked without thunk feature");
 }
 
 MachineBasicBlock *
-X86TargetLowering::EmitLoweredRetpoline(MachineInstr &MI,
-                                        MachineBasicBlock *BB) const {
+X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI,
+                                            MachineBasicBlock *BB) const {
   // Copy the virtual register into the R11 physical register and
   // call the retpoline thunk.
   DebugLoc DL = MI.getDebugLoc();
   const X86InstrInfo *TII = Subtarget.getInstrInfo();
   Register CalleeVReg = MI.getOperand(0).getReg();
-  unsigned Opc = getOpcodeForRetpoline(MI.getOpcode());
+  unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode());
 
   // Find an available scratch register to hold the callee. On 64-bit, we can
   // just use R11, but we scan for uses anyway to ensure we don't generate
@@ -31455,7 +31459,7 @@ X86TargetLowering::EmitLoweredRetpoline(MachineInstr &MI,
     report_fatal_error("calling convention incompatible with retpoline, no "
                        "available registers");
 
-  const char *Symbol = getRetpolineSymbol(Subtarget, AvailableReg);
+  const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg);
 
   BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), AvailableReg)
       .addReg(CalleeVReg);
@@ -32231,11 +32235,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case X86::TLS_base_addr32:
   case X86::TLS_base_addr64:
     return EmitLoweredTLSAddr(MI, BB);
-  case X86::RETPOLINE_CALL32:
-  case X86::RETPOLINE_CALL64:
-  case X86::RETPOLINE_TCRETURN32:
-  case X86::RETPOLINE_TCRETURN64:
-    return EmitLoweredRetpoline(MI, BB);
+  case X86::INDIRECT_THUNK_CALL32:
+  case X86::INDIRECT_THUNK_CALL64:
+  case X86::INDIRECT_THUNK_TCRETURN32:
+  case X86::INDIRECT_THUNK_TCRETURN64:
+    return EmitLoweredIndirectThunk(MI, BB);
   case X86::CATCHRET:
     return EmitLoweredCatchRet(MI, BB);
   case X86::CATCHPAD:

llvm/lib/Target/X86/X86ISelLowering.h

@@ -1482,8 +1482,8 @@ namespace llvm {
     MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
                                           MachineBasicBlock *BB) const;
 
-    MachineBasicBlock *EmitLoweredRetpoline(MachineInstr &MI,
-                                            MachineBasicBlock *BB) const;
+    MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
+                                                MachineBasicBlock *BB) const;
 
     MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
                                         MachineBasicBlock *MBB) const;

llvm/lib/Target/X86/X86RetpolineThunks.cpp renamed to llvm/lib/Target/X86/X86IndirectThunks.cpp

@@ -1,4 +1,4 @@
-//======- X86RetpolineThunks.cpp - Construct retpoline thunks for x86  --=====//
+//==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86  --=//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,12 +7,18 @@
 //===----------------------------------------------------------------------===//
 /// \file
 ///
-/// Pass that injects an MI thunk implementing a "retpoline". This is
-/// a RET-implemented trampoline that is used to lower indirect calls in a way
+/// Pass that injects an MI thunk that is used to lower indirect calls in a way
 /// that prevents speculation on some x86 processors and can be used to mitigate
 /// security vulnerabilities due to targeted speculative execution and side
 /// channels such as CVE-2017-5715.
 ///
+/// Currently supported thunks include:
+/// - Retpoline -- A RET-implemented trampoline that lowers indirect calls
+///
+/// Note that the reason that this is implemented as a MachineFunctionPass and
+/// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline
+/// serialize all transformations, which can consume lots of memory.
+///
 /// TODO(chandlerc): All of this code could use better comments and
 /// documentation.
 ///
@@ -37,21 +43,21 @@ using namespace llvm;
 
 #define DEBUG_TYPE "x86-retpoline-thunks"
 
-static const char ThunkNamePrefix[] = "__llvm_retpoline_";
-static const char R11ThunkName[]    = "__llvm_retpoline_r11";
-static const char EAXThunkName[]    = "__llvm_retpoline_eax";
-static const char ECXThunkName[]    = "__llvm_retpoline_ecx";
-static const char EDXThunkName[]    = "__llvm_retpoline_edx";
-static const char EDIThunkName[]    = "__llvm_retpoline_edi";
+static const char RetpolineNamePrefix[] = "__llvm_retpoline_";
+static const char R11RetpolineName[]    = "__llvm_retpoline_r11";
+static const char EAXRetpolineName[]    = "__llvm_retpoline_eax";
+static const char ECXRetpolineName[]    = "__llvm_retpoline_ecx";
+static const char EDXRetpolineName[]    = "__llvm_retpoline_edx";
+static const char EDIRetpolineName[]    = "__llvm_retpoline_edi";
 
 namespace {
-class X86RetpolineThunks : public MachineFunctionPass {
+class X86IndirectThunks : public MachineFunctionPass {
 public:
   static char ID;
 
-  X86RetpolineThunks() : MachineFunctionPass(ID) {}
+  X86IndirectThunks() : MachineFunctionPass(ID) {}
 
-  StringRef getPassName() const override { return "X86 Retpoline Thunks"; }
+  StringRef getPassName() const override { return "X86 Indirect Thunks"; }
 
   bool doInitialization(Module &M) override;
   bool runOnMachineFunction(MachineFunction &F) override;
@@ -72,24 +78,24 @@ class X86RetpolineThunks : public MachineFunctionPass {
   bool InsertedThunks = false;
 
   void createThunkFunction(Module &M, StringRef Name);
-  void insertRegReturnAddrClobber(MachineBasicBlock &MBB, unsigned Reg);
-  void populateThunk(MachineFunction &MF, unsigned Reg);
+  void insertRegReturnAddrClobber(MachineBasicBlock &MBB, Register Reg);
+  void populateThunk(MachineFunction &MF, Register Reg);
 };
 
 } // end anonymous namespace
 
-FunctionPass *llvm::createX86RetpolineThunksPass() {
-  return new X86RetpolineThunks();
+FunctionPass *llvm::createX86IndirectThunksPass() {
+  return new X86IndirectThunks();
 }
 
-char X86RetpolineThunks::ID = 0;
+char X86IndirectThunks::ID = 0;
 
-bool X86RetpolineThunks::doInitialization(Module &M) {
+bool X86IndirectThunks::doInitialization(Module &M) {
   InsertedThunks = false;
   return false;
 }
 
-bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
+bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << getPassName() << '\n');
 
   TM = &MF.getTarget();;
@@ -102,7 +108,7 @@ bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
 
   // If this function is not a thunk, check to see if we need to insert
   // a thunk.
-  if (!MF.getName().startswith(ThunkNamePrefix)) {
+  if (!MF.getName().startswith(RetpolineNamePrefix)) {
     // If we've already inserted a thunk, nothing else to do.
     if (InsertedThunks)
       return false;
@@ -124,10 +130,11 @@ bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
     // pass. We extract the module and insert a new function (and machine
     // function) directly into the module.
     if (Is64Bit)
-      createThunkFunction(M, R11ThunkName);
+      createThunkFunction(M, R11RetpolineName);
     else
       for (StringRef Name :
-           {EAXThunkName, ECXThunkName, EDXThunkName, EDIThunkName})
+           {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName,
+            EDIRetpolineName})
         createThunkFunction(M, Name);
     InsertedThunks = true;
     return true;
@@ -177,13 +184,13 @@ bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
     //   ... # Same setup
     //         movl %edi, (%esp)
     //         retl
-    if (MF.getName() == EAXThunkName)
+    if (MF.getName() == EAXRetpolineName)
       populateThunk(MF, X86::EAX);
-    else if (MF.getName() == ECXThunkName)
+    else if (MF.getName() == ECXRetpolineName)
       populateThunk(MF, X86::ECX);
-    else if (MF.getName() == EDXThunkName)
+    else if (MF.getName() == EDXRetpolineName)
       populateThunk(MF, X86::EDX);
-    else if (MF.getName() == EDIThunkName)
+    else if (MF.getName() == EDIRetpolineName)
       populateThunk(MF, X86::EDI);
     else
       llvm_unreachable("Invalid thunk name on x86-32!");
@@ -192,8 +199,8 @@ bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
   return true;
 }
 
-void X86RetpolineThunks::createThunkFunction(Module &M, StringRef Name) {
-  assert(Name.startswith(ThunkNamePrefix) &&
+void X86IndirectThunks::createThunkFunction(Module &M, StringRef Name) {
+  assert(Name.startswith(RetpolineNamePrefix) &&
          "Created a thunk with an unexpected prefix!");
 
   LLVMContext &Ctx = M.getContext();
@@ -226,16 +233,16 @@ void X86RetpolineThunks::createThunkFunction(Module &M, StringRef Name) {
   MF.insert(MF.end(), EntryMBB);
 }
 
-void X86RetpolineThunks::insertRegReturnAddrClobber(MachineBasicBlock &MBB,
-                                                    unsigned Reg) {
+void X86IndirectThunks::insertRegReturnAddrClobber(MachineBasicBlock &MBB,
+                                                   Register Reg) {
   const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
-  const unsigned SPReg = Is64Bit ? X86::RSP : X86::ESP;
+  const Register SPReg = Is64Bit ? X86::RSP : X86::ESP;
   addRegOffset(BuildMI(&MBB, DebugLoc(), TII->get(MovOpc)), SPReg, false, 0)
       .addReg(Reg);
 }
 
-void X86RetpolineThunks::populateThunk(MachineFunction &MF,
-                                       unsigned Reg) {
+void X86IndirectThunks::populateThunk(MachineFunction &MF,
+                                      Register Reg) {
   // Set MF properties. We never use vregs...
   MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
 
@@ -246,8 +253,10 @@ void X86RetpolineThunks::populateThunk(MachineFunction &MF,
   while (MF.size() > 1)
     MF.erase(std::next(MF.begin()));
 
-  MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
-  MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
+  MachineBasicBlock *CaptureSpec =
+      MF.CreateMachineBasicBlock(Entry->getBasicBlock());
+  MachineBasicBlock *CallTarget =
+      MF.CreateMachineBasicBlock(Entry->getBasicBlock());
   MCSymbol *TargetSym = MF.getContext().createTempSymbol();
   MF.push_back(CaptureSpec);
   MF.push_back(CallTarget);