Extract immediately-upstreamable changes from amd-staging

This is largely CFI only, but also includes definitions in
BinaryFormat/Dwarf.def that we should try to pin down ASAP.

An encoding change is required for some of the DW_AT_* extensions, which
I've limited to just:
    DW_AT_LLVM_lanes
    DW_AT_LLVM_lane_pc
    DW_AT_LLVM_vector_size

As these are not currently emitted by the compiler I don't think this
constitutes a breaking change, so we should be able to flow this back to
amd-staging without issue.

Author: slinder1

clang/lib/Driver/ToolChains/Clang.cpp

@@ -4920,6 +4920,10 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
   renderDwarfFormat(D, T, Args, CmdArgs, EffectiveDWARFVersion);
   RenderDebugInfoCompressionArgs(Args, CmdArgs, D, TC);
 
+  bool EmitDwarfForAMDGCN = EmitDwarf && T.isAMDGCN();
+  if (EmitDwarfForAMDGCN)
+    CmdArgs.append({"-mllvm", "-amdgpu-spill-cfi-saved-regs"});
+
   // This controls whether or not we perform JustMyCode instrumentation.
   if (Args.hasFlag(options::OPT_fjmc, options::OPT_fno_jmc, false)) {
     if (TC.getTriple().isOSBinFormatELF() ||

llvm/include/llvm/BinaryFormat/Dwarf.def

@@ -25,7 +25,9 @@
       defined HANDLE_DW_APPLE_PROPERTY || defined HANDLE_DW_UT ||              \
       defined HANDLE_DWARF_SECTION || defined HANDLE_DW_IDX ||                 \
       defined HANDLE_DW_END || defined HANDLE_DW_SECT ||                       \
-      defined HANDLE_DW_APPLE_ENUM_KIND)
+      defined HANDLE_DW_APPLE_ENUM_KIND ||                                     \
+      defined HANDLE_DW_MSPACE ||                                              \
+      ( defined HANDLE_DW_ASPACE && defined HANDLE_DW_ASPACE_PRED) )
 #error "Missing macro definition of HANDLE_DW*"
 #endif
 
@@ -151,6 +153,18 @@
 #define HANDLE_DW_APPLE_ENUM_KIND(ID, NAME)
 #endif
 
+#ifndef HANDLE_DW_MSPACE
+#define HANDLE_DW_MSPACE(ID, NAME)
+#endif
+
+#ifndef HANDLE_DW_ASPACE
+#define HANDLE_DW_ASPACE(ID, NAME)
+#endif
+
+#ifndef HANDLE_DW_ASPACE_PRED
+#define HANDLE_DW_ASPACE_PRED(ID, NAME, PRED)
+#endif
+
 HANDLE_DW_TAG(0x0000, null, 2, DWARF, DW_KIND_NONE)
 HANDLE_DW_TAG(0x0001, array_type, 2, DWARF, DW_KIND_TYPE)
 HANDLE_DW_TAG(0x0002, class_type, 2, DWARF, DW_KIND_TYPE)
@@ -624,6 +638,28 @@ HANDLE_DW_AT(0x3e09, LLVM_ptrauth_authenticates_null_values, 0, LLVM)
 HANDLE_DW_AT(0x3e0a, LLVM_ptrauth_authentication_mode, 0, LLVM)
 HANDLE_DW_AT(0x3e0b, LLVM_num_extra_inhabitants, 0, LLVM)
 HANDLE_DW_AT(0x3e0c, LLVM_stmt_sequence, 0, LLVM)
+// Heterogeneous Debugging Extension defined at
+// https://llvm.org/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.html.
+HANDLE_DW_AT(0x3e0d, LLVM_memory_space, 0, LLVM)
+HANDLE_DW_AT(0x3e0e, LLVM_address_space, 0, LLVM)
+HANDLE_DW_AT(0x3e0f, LLVM_lanes, 0, LLVM)
+HANDLE_DW_AT(0x3e10, LLVM_lane_pc, 0, LLVM)
+HANDLE_DW_AT(0x3e11, LLVM_vector_size, 0, LLVM)
+
+// https://www.llvm.org/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.html#a-7-15-memory-space-encodings
+HANDLE_DW_MSPACE(0x0, none)
+HANDLE_DW_MSPACE(0x1, global)
+HANDLE_DW_MSPACE(0x2, constant)
+HANDLE_DW_MSPACE(0x3, group)
+HANDLE_DW_MSPACE(0x4, private)
+
+// https://llvm.org/docs/AMDGPUUsage.html#address-space-identifier
+HANDLE_DW_ASPACE(0x0, none)
+HANDLE_DW_ASPACE_PRED(0x1, AMDGPU_generic, SELECT_AMDGPU)
+HANDLE_DW_ASPACE_PRED(0x2, AMDGPU_region, SELECT_AMDGPU)
+HANDLE_DW_ASPACE_PRED(0x3, AMDGPU_local, SELECT_AMDGPU)
+HANDLE_DW_ASPACE_PRED(0x5, AMDGPU_private_lane, SELECT_AMDGPU)
+HANDLE_DW_ASPACE_PRED(0x6, AMDGPU_private_wave, SELECT_AMDGPU)
 
 // Apple extensions.
 
@@ -911,6 +947,19 @@ HANDLE_DW_OP(0xe9, LLVM_user, -1, -1, 0, LLVM)
 // location stack or any of its values. It is defined as a placeholder for
 // testing purposes.
 HANDLE_DW_OP_LLVM_USEROP(0x0001, nop)
+// Heterogeneous Debugging Extension defined at
+// https://llvm.org/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.html.
+HANDLE_DW_OP_LLVM_USEROP(0x0002, form_aspace_address)
+HANDLE_DW_OP_LLVM_USEROP(0x0003, push_lane)
+HANDLE_DW_OP_LLVM_USEROP(0x0004, offset)
+HANDLE_DW_OP_LLVM_USEROP(0x0005, offset_uconst)
+HANDLE_DW_OP_LLVM_USEROP(0x0006, bit_offset)
+HANDLE_DW_OP_LLVM_USEROP(0x0007, call_frame_entry_reg)
+HANDLE_DW_OP_LLVM_USEROP(0x0008, undefined)
+HANDLE_DW_OP_LLVM_USEROP(0x0009, aspace_bregx)
+HANDLE_DW_OP_LLVM_USEROP(0x000a, piece_end)
+HANDLE_DW_OP_LLVM_USEROP(0x000b, extend)
+HANDLE_DW_OP_LLVM_USEROP(0x000c, select_bit_piece)
 
 // DWARF languages.
 HANDLE_DW_LANG(0x0001, C89, 0, 2, DWARF)
@@ -1380,3 +1429,6 @@ HANDLE_DW_SECT(8, RNGLISTS)
 #undef HANDLE_DW_END
 #undef HANDLE_DW_SECT
 #undef HANDLE_DW_APPLE_ENUM_KIND
+#undef HANDLE_DW_MSPACE
+#undef HANDLE_DW_ASPACE
+#undef HANDLE_DW_ASPACE_PRED

llvm/include/llvm/BinaryFormat/Dwarf.h

@@ -757,6 +757,19 @@ enum CallingConvention {
   DW_CC_hi_user = 0xff
 };
 
+enum MemorySpace {
+#define HANDLE_DW_MSPACE(ID, NAME) DW_MSPACE_LLVM_##NAME = ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+  DW_MSPACE_LLVM_lo_user = 0x8000,
+  DW_MSPACE_LLVM_hi_user = 0xffff
+};
+
+enum AddressSpace {
+#define HANDLE_DW_ASPACE(ID, NAME) DW_ASPACE_LLVM_##NAME = ID,
+#define HANDLE_DW_ASPACE_PRED(ID, NAME, PRED) DW_ASPACE_LLVM_##NAME = ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+};
+
 enum InlineAttribute {
   // Inline codes
   DW_INL_not_inlined = 0x00,
@@ -1011,6 +1024,8 @@ StringRef IndexString(unsigned Idx);
 StringRef FormatString(DwarfFormat Format);
 StringRef FormatString(bool IsDWARF64);
 StringRef RLEString(unsigned RLE);
+StringRef MemorySpaceString(unsigned MS);
+StringRef AddressSpaceString(unsigned AS, llvm::Triple TT);
 /// @}
 
 /// \defgroup DwarfConstantsParsing Dwarf constants parsing functions
@@ -1031,6 +1046,7 @@ unsigned getVirtuality(StringRef VirtualityString);
 unsigned getEnumKind(StringRef EnumKindString);
 unsigned getLanguage(StringRef LanguageString);
 unsigned getCallingConvention(StringRef LanguageString);
+unsigned getMemorySpace(StringRef LanguageString);
 unsigned getAttributeEncoding(StringRef EncodingString);
 unsigned getMacinfo(StringRef MacinfoString);
 unsigned getMacro(StringRef MacroString);

llvm/include/llvm/CodeGen/MachineFunction.h

@@ -1186,6 +1186,10 @@ class LLVM_ABI MachineFunction {
 
   [[nodiscard]] unsigned addFrameInst(const MCCFIInstruction &Inst);
 
+  /// Replace all references to register \param From with register \param To in
+  /// frame instructions. Note that .cfi_escape instructions will be left as-is.
+  void replaceFrameInstRegister(Register From, Register To);
+
   /// Returns a reference to a list of symbols immediately following calls to
   /// _setjmp in the function. Used to construct the longjmp target table used
   /// by Windows Control Flow Guard.

llvm/include/llvm/MC/MCAsmInfo.h

@@ -389,6 +389,11 @@ class MCAsmInfo {
   /// location is allowed.
   bool SupportsExtendedDwarfLocDirective = true;
 
+  /// True if the target supports the extensions defined at
+  /// https://llvm.org/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.html.
+  /// Defaults to false.
+  bool SupportsHeterogeneousDebuggingExtensions = false;
+
   //===--- Prologue State ----------------------------------------------===//
 
   std::vector<MCCFIInstruction> InitialFrameState;
@@ -618,6 +623,10 @@ class MCAsmInfo {
 
   bool doesSupportDebugInformation() const { return SupportsDebugInformation; }
 
+  bool doesSupportExceptionHandling() const {
+    return ExceptionsType != ExceptionHandling::None;
+  }
+
   ExceptionHandling getExceptionHandlingType() const { return ExceptionsType; }
   WinEH::EncodingType getWinEHEncodingType() const { return WinEHEncodingType; }
 
@@ -656,6 +665,9 @@ class MCAsmInfo {
   bool supportsExtendedDwarfLocDirective() const {
     return SupportsExtendedDwarfLocDirective;
   }
+  bool supportsHeterogeneousDebuggingExtensions() const {
+    return SupportsHeterogeneousDebuggingExtensions;
+  }
 
   bool usesDwarfFileAndLocDirectives() const { return !IsAIX; }
 

llvm/include/llvm/MC/MCDwarf.h

@@ -519,6 +519,42 @@ class MCCFIInstruction {
     OpGnuArgsSize,
     OpLabel,
     OpValOffset,
+    OpLLVMRegisterPair,
+    OpLLVMVectorRegisters,
+    OpLLVMVectorOffset,
+    OpLLVMVectorRegisterMask,
+  };
+
+  /// Some extra fields used when Operation is OpLLVMRegisterPair.
+  struct RegisterPairExtraFields {
+    unsigned Reg1, Reg2;
+    unsigned Reg1SizeInBits, Reg2SizeInBits;
+  };
+
+  struct VectorRegisterWithLane {
+    unsigned Register;
+    unsigned Lane;
+    unsigned SizeInBits;
+  };
+
+  /// Some extra fields used when Operation is OpLLVMVectorRegisters.
+  struct VectorRegistersExtraFields {
+    std::vector<VectorRegisterWithLane> VectorRegisters;
+  };
+
+  /// Some extra fields used when Operation is OpLLVMVectorOffset.
+  struct VectorOffsetExtraFields {
+    unsigned MaskRegister;
+    unsigned MaskRegisterSizeInBits;
+    unsigned RegisterSizeInBits;
+  };
+
+  /// Some extra fields used when Operation is OpLLVMVectorRegisterMask.
+  struct VectorRegisterMaskExtraFields {
+    unsigned SpillRegister;
+    unsigned SpillRegisterLaneSizeInBits;
+    unsigned MaskRegister;
+    unsigned MaskRegisterSizeInBits;
   };
 
 private:
@@ -544,6 +580,14 @@ class MCCFIInstruction {
   std::vector<char> Values;
   std::string Comment;
 
+  // FIXME: We could probably save some space and complexity by moving all
+  // Operation-specific fields to this variant. Leaving them as-is for now to
+  // avoid a diff with upstream.
+  std::variant<std::monostate, RegisterPairExtraFields,
+               VectorRegistersExtraFields, VectorOffsetExtraFields,
+               VectorRegisterMaskExtraFields>
+      ExtraFields;
+
   MCCFIInstruction(OpType Op, MCSymbol *L, unsigned R, int64_t O, SMLoc Loc,
                    StringRef V = "", StringRef Comment = "")
       : Label(L), Operation(Op), Loc(Loc), Values(V.begin(), V.end()),
@@ -563,6 +607,14 @@ class MCCFIInstruction {
     U.RIA = {R, O, AS};
   }
 
+  template <class ExtraFieldsTy>
+  MCCFIInstruction(OpType Op, MCSymbol *L, unsigned R, int O,
+                   ExtraFieldsTy &&ExtraFields, SMLoc Loc)
+      : Label(L), Operation(Op), Loc(Loc),
+        ExtraFields(std::forward<ExtraFieldsTy>(ExtraFields)) {
+    U.RI = {R, O};
+  }
+
   MCCFIInstruction(OpType Op, MCSymbol *L, MCSymbol *CfiLabel, SMLoc Loc)
       : Label(L), Operation(Op), Loc(Loc) {
     assert(Op == OpLabel);
@@ -700,6 +752,62 @@ class MCCFIInstruction {
     return MCCFIInstruction(OpLabel, L, CfiLabel, Loc);
   }
 
+  /// .cfi_llvm_register_pair Previous value of Register is saved in R1:R2.
+  static MCCFIInstruction
+  createLLVMRegisterPair(MCSymbol *L, unsigned Register, unsigned R1,
+                         unsigned R1SizeInBits, unsigned R2,
+                         unsigned R2SizeInBits, SMLoc Loc = {}) {
+    RegisterPairExtraFields Extra{R1, R2, R1SizeInBits, R2SizeInBits};
+    return MCCFIInstruction(OpLLVMRegisterPair, L, Register, 0, Extra, Loc);
+  }
+
+  /// .cfi_llvm_vector_registers Previous value of Register is saved in lanes of
+  /// vector registers.
+  static MCCFIInstruction
+  createLLVMVectorRegisters(MCSymbol *L, unsigned Register,
+                            std::vector<VectorRegisterWithLane> VectorRegisters,
+                            SMLoc Loc = {}) {
+    VectorRegistersExtraFields Extra{std::move(VectorRegisters)};
+    return MCCFIInstruction(OpLLVMVectorRegisters, L, Register, 0,
+                            std::move(Extra), Loc);
+  }
+
+  /// .cfi_llvm_vector_offset Previous value of Register is saved at Offset from
+  /// CFA. MaskRegister specifies the active lanes of register.
+  static MCCFIInstruction
+  createLLVMVectorOffset(MCSymbol *L, unsigned Register,
+                         unsigned RegisterSizeInBits, unsigned MaskRegister,
+                         unsigned MaskRegisterSizeInBits, int Offset,
+                         SMLoc Loc = {}) {
+    VectorOffsetExtraFields Extra{MaskRegister, MaskRegisterSizeInBits,
+                                  RegisterSizeInBits};
+    return MCCFIInstruction(OpLLVMVectorOffset, L, Register, Offset, Extra,
+                            Loc);
+  }
+
+  /// .cfi_llvm_vector_register_mask Previous value of Register is saved in
+  /// SpillRegister, predicated on the value of MaskRegister.
+  static MCCFIInstruction createLLVMVectorRegisterMask(
+      MCSymbol *L, unsigned Register, unsigned SpillRegister,
+      unsigned SpillRegisterLaneSizeInBits, unsigned MaskRegister,
+      unsigned MaskRegisterSizeInBits, SMLoc Loc = {}) {
+    VectorRegisterMaskExtraFields Extra{
+        SpillRegister,
+        SpillRegisterLaneSizeInBits,
+        MaskRegister,
+        MaskRegisterSizeInBits,
+    };
+    return MCCFIInstruction(OpLLVMVectorRegisterMask, L, Register, 0,
+                            std::move(Extra), Loc);
+  }
+
+  template <class ExtraFieldsTy> ExtraFieldsTy &getExtraFields() {
+    return std::get<ExtraFieldsTy>(ExtraFields);
+  }
+
+  template <class ExtraFieldsTy> const ExtraFieldsTy &getExtraFields() const {
+    return std::get<ExtraFieldsTy>(ExtraFields);
+  }
   /// .cfi_val_offset Previous value of Register is offset Offset from the
   /// current CFA register.
   static MCCFIInstruction createValOffset(MCSymbol *L, unsigned Register,
@@ -718,6 +826,9 @@ class MCCFIInstruction {
     assert(Operation == OpDefCfa || Operation == OpOffset ||
            Operation == OpRestore || Operation == OpUndefined ||
            Operation == OpSameValue || Operation == OpDefCfaRegister ||
+           Operation == OpLLVMVectorRegisters ||
+           Operation == OpLLVMRegisterPair || Operation == OpLLVMVectorOffset ||
+           Operation == OpLLVMVectorRegisterMask ||
            Operation == OpRelOffset || Operation == OpValOffset);
     return U.RI.Register;
   }
@@ -738,6 +849,7 @@ class MCCFIInstruction {
     assert(Operation == OpDefCfa || Operation == OpOffset ||
            Operation == OpRelOffset || Operation == OpDefCfaOffset ||
            Operation == OpAdjustCfaOffset || Operation == OpGnuArgsSize ||
+           Operation == OpLLVMVectorOffset ||
            Operation == OpValOffset);
     return U.RI.Offset;
   }
@@ -754,6 +866,9 @@ class MCCFIInstruction {
 
   StringRef getComment() const { return Comment; }
   SMLoc getLoc() const { return Loc; }
+
+  /// Replaces in place all references to FromReg with ToReg.
+  void replaceRegister(unsigned FromReg, unsigned ToReg);
 };
 
 struct MCDwarfFrameInfo {

llvm/include/llvm/MC/MCStreamer.h

@@ -1000,6 +1000,24 @@ class MCStreamer {
                                SMLoc Loc = {});
   virtual void emitCFIWindowSave(SMLoc Loc = {});
   virtual void emitCFINegateRAState(SMLoc Loc = {});
+  virtual void emitCFILLVMRegisterPair(int64_t Register, int64_t R1,
+                                       int64_t R1SizeInBits, int64_t R2,
+                                       int64_t R2SizeInBits, SMLoc Loc = {});
+  virtual void emitCFILLVMVectorRegisters(
+      int64_t Register,
+      std::vector<MCCFIInstruction::VectorRegisterWithLane> VRs,
+      SMLoc Loc = {});
+  virtual void emitCFILLVMVectorOffset(int64_t Register,
+                                       int64_t RegisterSizeInBits,
+                                       int64_t MaskRegister,
+                                       int64_t MaskRegisterSizeInBits,
+                                       int64_t Offset, SMLoc Loc = {});
+  virtual void
+  emitCFILLVMVectorRegisterMask(int64_t Register, int64_t SpillRegister,
+                                int64_t SpillRegisterLaneSizeInBits,
+                                int64_t MaskRegister,
+                                int64_t MaskRegisterSizeInBits, SMLoc Loc = {});
+
   virtual void emitCFINegateRAStateWithPC(SMLoc Loc = {});
   virtual void emitCFILabelDirective(SMLoc Loc, StringRef Name);
   virtual void emitCFIValOffset(int64_t Register, int64_t Offset,