Merge from 'main' to 'sycl-web' (248 commits)

  CONFLICT (content): Merge conflict in llvm/test/CodeGen/X86/atom-pad-short-functions.ll

Author: Fznamznon

bolt/include/bolt/Passes/Hugify.h

@@ -0,0 +1,29 @@
+//===- bolt/Passes/Hugify.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_HUGIFY_H
+#define BOLT_PASSES_HUGIFY_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+class HugePage : public BinaryFunctionPass {
+public:
+  HugePage(const cl::opt<bool> &PrintPass) : BinaryFunctionPass(PrintPass) {}
+
+  void runOnFunctions(BinaryContext &BC) override;
+
+  const char *getName() const override { return "HugePage"; }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h

@@ -22,13 +22,11 @@ class HugifyRuntimeLibrary : public RuntimeLibrary {
 public:
   /// Add custom section names generated by the runtime libraries to \p
   /// SecNames.
-  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {
-    SecNames.push_back(".bolt.hugify.entries");
-  }
+  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {}
 
   void adjustCommandLineOptions(const BinaryContext &BC) const final;
 
-  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final;
+  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final {}
 
   void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld,
             std::function<void(RuntimeDyld &)> OnLoad) final;

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -44,6 +44,7 @@ extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;
+extern llvm::cl::opt<bool> Hugify;
 extern llvm::cl::opt<bool> Instrument;
 extern llvm::cl::opt<std::string> OutputFilename;
 extern llvm::cl::opt<std::string> PerfData;

bolt/lib/Core/BinaryEmitter.cpp

@@ -740,10 +740,12 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 
   for (auto &JTI : BF.jumpTables()) {
     JumpTable &JT = *JTI.second;
+    // Only emit shared jump tables once, when processing the first parent
+    if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
+      continue;
     if (opts::PrintJumpTables)
       JT.print(outs());
-    if ((opts::JumpTables == JTS_BASIC || !BF.isSimple()) &&
-        BC.HasRelocations) {
+    if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
       JT.updateOriginal();
     } else {
       MCSection *HotSection, *ColdSection;

bolt/lib/Passes/CMakeLists.txt

@@ -15,6 +15,7 @@ add_llvm_library(LLVMBOLTPasses
   FrameOptimizer.cpp
   HFSort.cpp
   HFSortPlus.cpp
+  Hugify.cpp
   IdenticalCodeFolding.cpp
   IndirectCallPromotion.cpp
   Inliner.cpp

bolt/lib/Passes/Hugify.cpp

@@ -0,0 +1,50 @@
+//===--- bolt/Passes/Hugify.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Passes/Hugify.h"
+#include "llvm/Support/CommandLine.h"
+
+#define DEBUG_TYPE "bolt-hugify"
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+void HugePage::runOnFunctions(BinaryContext &BC) {
+  auto *RtLibrary = BC.getRuntimeLibrary();
+  if (!RtLibrary || !BC.isELF() || !BC.StartFunctionAddress) {
+    return;
+  }
+
+  auto createSimpleFunction =
+      [&](std::string Title, std::vector<MCInst> Instrs) -> BinaryFunction * {
+    BinaryFunction *Func = BC.createInjectedBinaryFunction(Title);
+
+    std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;
+    BBs.emplace_back(Func->createBasicBlock(nullptr));
+    BBs.back()->addInstructions(Instrs.begin(), Instrs.end());
+    BBs.back()->setCFIState(0);
+    BBs.back()->setOffset(BinaryBasicBlock::INVALID_OFFSET);
+
+    Func->insertBasicBlocks(nullptr, std::move(BBs),
+                            /*UpdateLayout=*/true,
+                            /*UpdateCFIState=*/false);
+    Func->updateState(BinaryFunction::State::CFG_Finalized);
+    return Func;
+  };
+
+  const BinaryFunction *const Start =
+      BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
+  assert(Start && "Entry point function not found");
+  const MCSymbol *StartSym = Start->getSymbol();
+  createSimpleFunction("__bolt_hugify_start_program",
+                       BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));
+}
+} // namespace bolt
+} // namespace llvm

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -13,6 +13,7 @@
 #include "bolt/Passes/AsmDump.h"
 #include "bolt/Passes/CMOVConversion.h"
 #include "bolt/Passes/FrameOptimizer.h"
+#include "bolt/Passes/Hugify.h"
 #include "bolt/Passes/IdenticalCodeFolding.h"
 #include "bolt/Passes/IndirectCallPromotion.h"
 #include "bolt/Passes/Inliner.h"
@@ -333,6 +334,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
 
   if (opts::Instrument)
     Manager.registerPass(std::make_unique<Instrumentation>(NeverPrint));
+  else if (opts::Hugify)
+    Manager.registerPass(std::make_unique<HugePage>(NeverPrint));
 
   Manager.registerPass(std::make_unique<ShortenInstructions>(NeverPrint));
 

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -479,6 +479,11 @@ Error RewriteInstance::discoverStorage() {
   NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);
   NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);
 
+  // Hugify: Additional huge page from left side due to
+  // weird ASLR mapping addresses (4KB aligned)
+  if (opts::Hugify && !BC->HasFixedLoadAddress)
+    NextAvailableAddress += BC->PageAlign;
+
   if (!opts::UseGnuStack) {
     // This is where the black magic happens. Creating PHDR table in a segment
     // other than that containing ELF header is tricky. Some loaders and/or
@@ -3719,6 +3724,12 @@ void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) {
         Address = alignTo(Address, Section->getAlignment());
         Section->setOutputAddress(Address);
         Address += Section->getOutputSize();
+
+        // Hugify: Additional huge page from right side due to
+        // weird ASLR mapping addresses (4KB aligned)
+        if (opts::Hugify && !BC->HasFixedLoadAddress &&
+            Section->getName() == BC->getMainCodeSectionName())
+          Address = alignTo(Address, Section->getAlignment());
       }
 
       // Make sure we allocate enough space for huge pages.

bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp

@@ -60,35 +60,6 @@ void HugifyRuntimeLibrary::adjustCommandLineOptions(
   }
 }
 
-void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
-  const BinaryFunction *StartFunction =
-      BC.getBinaryFunctionAtAddress(*(BC.StartFunctionAddress));
-  assert(!StartFunction->isFragment() && "expected main function fragment");
-  if (!StartFunction) {
-    errs() << "BOLT-ERROR: failed to locate function at binary start address\n";
-    exit(1);
-  }
-
-  const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
-                                             /*IsText=*/false,
-                                             /*IsAllocatable=*/true);
-  MCSectionELF *Section =
-      BC.Ctx->getELFSection(".bolt.hugify.entries", ELF::SHT_PROGBITS, Flags);
-
-  // __bolt_hugify_init_ptr stores the poiter the hugify library needs to
-  // jump to after finishing the init code.
-  MCSymbol *InitPtr = BC.Ctx->getOrCreateSymbol("__bolt_hugify_init_ptr");
-
-  Section->setAlignment(llvm::Align(BC.RegularPageSize));
-  Streamer.switchSection(Section);
-
-  Streamer.emitLabel(InitPtr);
-  Streamer.emitSymbolAttribute(InitPtr, MCSymbolAttr::MCSA_Global);
-  Streamer.emitValue(
-      MCSymbolRefExpr::create(StartFunction->getSymbol(), *(BC.Ctx)),
-      /*Size=*/8);
-}
-
 void HugifyRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath,
                                 RuntimeDyld &RTDyld,
                                 std::function<void(RuntimeDyld &)> OnLoad) {

bolt/runtime/CMakeLists.txt

@@ -27,10 +27,11 @@ set(BOLT_RT_FLAGS
   -fno-exceptions
   -fno-rtti
   -fno-stack-protector
-  -mno-sse)
+  -mno-sse
+  -fPIE)
 
 # Don't let the compiler think it can create calls to standard libs
-target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS} -fPIE)
+target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_instr PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_compile_options(bolt_rt_hugify PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_hugify PRIVATE ${CMAKE_CURRENT_BINARY_DIR})