[BOLT-AArch64] Support relocation mode for bzip2

Summary:
As we deal with incomplete addresses in address-computing
sequences of code in AArch64, we found it is easier to handle them in
relocation mode in the presence of relocations.

Incomplete addresses may mislead BOLT into thinking there are
instructions referring to a basic block when, in fact, this may be the
base address of a data reference. If the relocation is present, we can
easily spot such cases.

This diff contains extensions in relocation mode to understand and
deal with AArch64 relocations. It also adds code to process data inside
functions as marked by AArch64 ABI (symbol table entries named "$d").
In our code, this is called constant islands handling. Last, it extends
bughunter with a "cross" mode, in which the host generates the binaries
and the user test them (uploading to the target), useful when debugging
in AArch64.

(cherry picked from FBD6024570)

Author: rafaelauler

bolt/BinaryContext.cpp

@@ -489,13 +489,71 @@ size_t Relocation::getSizeForType(uint64_t Type) {
   case ELF::R_X86_64_TPOFF32:
   case ELF::R_X86_64_GOTPCRELX:
   case ELF::R_X86_64_REX_GOTPCRELX:
+  case ELF::R_AARCH64_CALL26:
+  case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+  case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
+  case ELF::R_AARCH64_ADD_ABS_LO12_NC:
+  case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
+  case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
+  case ELF::R_AARCH64_ADR_GOT_PAGE:
+  case ELF::R_AARCH64_LD64_GOT_LO12_NC:
+  case ELF::R_AARCH64_JUMP26:
     return 4;
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_64:
+  case ELF::R_AARCH64_ABS64:
     return 8;
   }
 }
 
+uint64_t Relocation::extractValue(uint64_t Type, uint64_t Contents) {
+  switch (Type) {
+  default:
+    llvm_unreachable("unsupported relocation type");
+  case ELF::R_AARCH64_ABS64:
+    return Contents;
+  case ELF::R_AARCH64_JUMP26:
+  case ELF::R_AARCH64_CALL26:
+    // Immediate goes in bits 25:0 of B and BL.
+    Contents &= ~0xfffffffffc000000ULL;
+    return SignExtend64<28>(Contents << 2);
+  case ELF::R_AARCH64_ADR_GOT_PAGE:
+  case ELF::R_AARCH64_ADR_PREL_PG_HI21: {
+    // Bits 32:12 of Symbol address goes in bits 30:29 + 23:5 of ADRP
+    // instruction
+    Contents &= ~0xffffffff9f00001fU;
+    auto LowBits = (Contents >> 29) & 0x3;
+    auto HighBits = (Contents >> 5) & 0x7ffff;
+    Contents = LowBits | (HighBits << 2);
+    return SignExtend64<32>(Contents << 12);
+  }
+  case ELF::R_AARCH64_LD64_GOT_LO12_NC:
+  case ELF::R_AARCH64_LDST64_ABS_LO12_NC: {
+    // Immediate goes in bits 21:10 of LD/ST instruction, taken
+    // from bits 11:3 of Symbol address
+    Contents &= ~0xffffffffffc003ffU;
+    return Contents >> (10 - 3);
+  }
+  case ELF::R_AARCH64_ADD_ABS_LO12_NC: {
+    // Immediate goes in bits 21:10 of ADD instruction
+    Contents &= ~0xffffffffffc003ffU;
+    return Contents >> (10 - 0);
+  }
+  case ELF::R_AARCH64_LDST32_ABS_LO12_NC: {
+    // Immediate goes in bits 21:10 of ADD instruction, taken
+    // from bits 11:2 of Symbol address
+    Contents &= ~0xffffffffffc003ffU;
+    return Contents >> (10 - 2);
+  }
+  case ELF::R_AARCH64_LDST8_ABS_LO12_NC: {
+    // Immediate goes in bits 21:10 of ADD instruction, taken
+    // from bits 11:0 of Symbol address
+    Contents &= ~0xffffffffffc003ffU;
+    return Contents >> (10 - 0);
+  }
+  }
+}
+
 bool Relocation::isPCRelative(uint64_t Type) {
   switch (Type) {
   default:
@@ -505,6 +563,12 @@ bool Relocation::isPCRelative(uint64_t Type) {
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
   case ELF::R_X86_64_TPOFF32:
+  case ELF::R_AARCH64_ABS64:
+  case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
+  case ELF::R_AARCH64_ADD_ABS_LO12_NC:
+  case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
+  case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
+  case ELF::R_AARCH64_LD64_GOT_LO12_NC:
     return false;
 
   case ELF::R_X86_64_PC8:
@@ -514,6 +578,10 @@ bool Relocation::isPCRelative(uint64_t Type) {
   case ELF::R_X86_64_GOTTPOFF:
   case ELF::R_X86_64_GOTPCRELX:
   case ELF::R_X86_64_REX_GOTPCRELX:
+  case ELF::R_AARCH64_CALL26:
+  case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+  case ELF::R_AARCH64_ADR_GOT_PAGE:
+  case ELF::R_AARCH64_JUMP26:
     return true;
   }
 }

bolt/BinaryContext.h

@@ -64,6 +64,11 @@ struct Relocation {
   /// Return size of the given relocation \p Type.
   static size_t getSizeForType(uint64_t Type);
 
+  /// Extract current relocated value from binary contents. This is used for
+  /// RISC architectures where values are encoded in specific bits depending
+  /// on the relocation value.
+  static uint64_t extractValue(uint64_t Type, uint64_t Contents);
+
   /// Return true if relocation type is PC-relative. Return false otherwise.
   static bool isPCRelative(uint64_t Type);
 
@@ -154,6 +159,11 @@ class BinaryContext {
   /// final addresses functions will have.
   uint64_t LayoutStartAddress{0};
 
+  /// Old .text info.
+  uint64_t OldTextSectionAddress{0};
+  uint64_t OldTextSectionOffset{0};
+  uint64_t OldTextSectionSize{0};
+
   /// True if the binary requires immediate relocation processing.
   bool RequiresZNow{false};
 

bolt/BinaryFunction.cpp

@@ -795,10 +795,21 @@ void BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData) {
       }
     }
 
+    if (BC.TheTriple->getArch() == llvm::Triple::aarch64 &&
+        isInConstantIsland(TargetAddress)) {
+      TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, "ISLANDat");
+      IslandSymbols[TargetAddress - getAddress()] = TargetSymbol;
+    }
+
     // Note that the address does not necessarily have to reside inside
     // a section, it could be an absolute address too.
     auto Section = BC.getSectionForAddress(TargetAddress);
-    if (Section && Section->isText()) {
+    // Assume AArch64's ADRP never references code - it does, but this is fixed
+    // after reading relocations. ADRP contents now are not really meaningful
+    // without its supporting relocation.
+    if (!TargetSymbol && Section && Section->isText() &&
+        (BC.TheTriple->getArch() != llvm::Triple::aarch64 ||
+         !BC.MIA->isADRP(Instruction))) {
       if (containsAddress(TargetAddress)) {
         if (TargetAddress != getAddress()) {
           // The address could potentially escape. Mark it as another entry
@@ -829,6 +840,16 @@ void BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData) {
     MCInst Instruction;
     const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
 
+    // Check for data inside code and ignore it
+    if (DataOffsets.find(Offset) != DataOffsets.end()) {
+      auto Iter = CodeOffsets.upper_bound(Offset);
+      if (Iter != CodeOffsets.end()) {
+        Size = *Iter - Offset;
+        continue;
+      }
+      break;
+    }
+
     if (!BC.DisAsm->getInstruction(Instruction,
                                    Size,
                                    FunctionData.slice(Offset),
@@ -985,10 +1006,16 @@ void BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData) {
               // code without re-assembly.
               size_t RelSize = (Size < 5) ? 1 : 4;
               auto RelOffset = Offset + Size - RelSize;
+              if (BC.TheTriple->getArch() == llvm::Triple::aarch64) {
+                RelSize = 0;
+                RelOffset = Offset;
+              }
               auto RI = MoveRelocations.find(RelOffset);
               if (RI == MoveRelocations.end()) {
                 uint64_t RelType = (RelSize == 1) ? ELF::R_X86_64_PC8
                                                   : ELF::R_X86_64_PC32;
+                if (BC.TheTriple->getArch() == llvm::Triple::aarch64)
+                  RelType = ELF::R_AARCH64_CALL26;
                 DEBUG(dbgs() << "BOLT-DEBUG: creating relocation for static"
                              << " function call to " << TargetSymbol->getName()
                              << " at offset 0x"
@@ -2485,6 +2512,9 @@ void BinaryFunction::emitBody(MCStreamer &Streamer, bool EmitColdPart) {
       LastIsPrefix = BC.MIA->isPrefix(Instr);
     }
   }
+
+  if (!EmitColdPart)
+    emitConstantIslands(Streamer);
 }
 
 void BinaryFunction::emitBodyRaw(MCStreamer *Streamer) {
@@ -2545,6 +2575,70 @@ void BinaryFunction::emitBodyRaw(MCStreamer *Streamer) {
   }
 }
 
+void BinaryFunction::emitConstantIslands(MCStreamer &Streamer) {
+  if (DataOffsets.empty())
+    return;
+
+  Streamer.EmitLabel(getFunctionConstantIslandLabel());
+  // Raw contents of the function.
+  StringRef SectionContents;
+  Section.getContents(SectionContents);
+
+  // Raw contents of the function.
+  StringRef FunctionContents =
+      SectionContents.substr(getAddress() - Section.getAddress(),
+                             getMaxSize());
+
+  if (opts::Verbosity)
+    outs() << "BOLT-INFO: emitting constant island for function " << *this
+           << "\n";
+
+  auto IS = IslandSymbols.begin();
+
+  // We split the island into smaller blocks and output labels between them.
+  for (auto DataIter = DataOffsets.begin(); DataIter != DataOffsets.end();
+       ++DataIter) {
+    uint64_t FunctionOffset = *DataIter;
+    uint64_t EndOffset = 0ULL;
+
+    // Determine size of this data chunk
+    auto NextData = std::next(DataIter);
+    auto CodeIter = CodeOffsets.lower_bound(*DataIter);
+    if (CodeIter == CodeOffsets.end() && NextData == DataOffsets.end()) {
+      EndOffset = getMaxSize();
+    } else if (CodeIter == CodeOffsets.end()) {
+      EndOffset = *NextData;
+    } else if (NextData == DataOffsets.end()) {
+      EndOffset = *CodeIter;
+    } else {
+      EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
+    }
+
+    if (FunctionOffset == EndOffset)
+      continue;    // Size is zero, nothing to emit
+
+    // Emit labels and data
+    while (IS != IslandSymbols.end() && IS->first < EndOffset) {
+      auto NextStop = IS->first;
+      assert(NextStop <= EndOffset && "internal overflow error");
+      if (FunctionOffset < NextStop) {
+        Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, NextStop));
+        FunctionOffset = NextStop;
+      }
+      DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << IS->second->getName()
+                   << " at offset 0x" << Twine::utohexstr(IS->first) << '\n');
+      Streamer.EmitLabel(IS->second);
+      ++IS;
+    }
+    assert(FunctionOffset <= EndOffset && "overflow error");
+    if (FunctionOffset < EndOffset) {
+      Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
+    }
+  }
+
+  assert(IS == IslandSymbols.end() && "some symbols were not emitted!");
+}
+
 namespace {
 
 #ifndef MAX_PATH
@@ -3334,10 +3428,37 @@ BinaryBasicBlock *BinaryFunction::splitEdge(BinaryBasicBlock *From,
   return NewBBPtr;
 }
 
+bool BinaryFunction::isDataMarker(const SymbolRef &Symbol,
+                                  uint64_t SymbolSize) const {
+  // For aarch64, the ABI defines mapping symbols so we identify data in the
+  // code section (see IHI0056B). $d identifies a symbol starting data contents.
+  if (BC.TheTriple->getArch() == llvm::Triple::aarch64 &&
+      Symbol.getType() == SymbolRef::ST_Unknown &&
+      SymbolSize == 0 &&
+      (!Symbol.getName().getError() && *Symbol.getName() == "$d"))
+    return true;
+  return false;
+}
+
+bool BinaryFunction::isCodeMarker(const SymbolRef &Symbol,
+                                  uint64_t SymbolSize) const {
+  // For aarch64, the ABI defines mapping symbols so we identify data in the
+  // code section (see IHI0056B). $x identifies a symbol starting code or the
+  // end of a data chunk inside code.
+  if (BC.TheTriple->getArch() == llvm::Triple::aarch64 &&
+      Symbol.getType() == SymbolRef::ST_Unknown &&
+      SymbolSize == 0 &&
+      (!Symbol.getName().getError() && *Symbol.getName() == "$x"))
+    return true;
+  return false;
+}
+
 bool BinaryFunction::isSymbolValidInScope(const SymbolRef &Symbol,
                                           uint64_t SymbolSize) const {
   // Some symbols are tolerated inside function bodies, others are not.
   // The real function boundaries may not be known at this point.
+  if (isDataMarker(Symbol, SymbolSize) || isCodeMarker(Symbol, SymbolSize))
+    return true;
 
   // It's okay to have a zero-sized symbol in the middle of non-zero-sized
   // function.