Merge branch 'main' into 138902-define-buffer

Author: V-FEXrt

.ci/generate_test_report_lib.py

@@ -92,7 +92,9 @@ def plural(num_tests):
             ]
         )
     elif failures:
-        report.extend(["", "## Failed Tests", "(click on a test name to see its output)"])
+        report.extend(
+            ["", "## Failed Tests", "(click on a test name to see its output)"]
+        )
 
         for testsuite_name, failures in failures.items():
             report.extend(["", f"### {testsuite_name}"])
@@ -120,6 +122,17 @@ def plural(num_tests):
             ]
         )
 
+    if failures or return_code != 0:
+        report.extend(
+            [
+                "",
+                "If these failures are unrelated to your changes (for example "
+                "tests are broken or flaky at HEAD), please open an issue at "
+                "https://github.com/llvm/llvm-project/issues and add the "
+                "`infrastructure` label.",
+            ]
+        )
+
     report = "\n".join(report)
     if len(report.encode("utf-8")) > size_limit:
         return generate_report(

.ci/generate_test_report_lib_test.py

@@ -109,7 +109,9 @@ def test_no_failures_build_failed(self):
 
               All tests passed but another part of the build **failed**.
 
-              [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
+              [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+              
+              If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -169,7 +171,9 @@ def test_report_single_file_single_testsuite(self):
           ```
           Other output goes here
           ```
-          </details>"""
+          </details>
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -203,7 +207,9 @@ def test_report_single_file_single_testsuite(self):
         ```
         DEF/test_2 output goes here
         ```
-        </details>"""
+        </details>
+        
+        If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
         ),
         "error",
     )
@@ -311,7 +317,9 @@ def test_report_dont_list_failures(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+          Failed tests and their output was too large to report. Download the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -352,13 +360,16 @@ def test_report_dont_list_failures_link_to_log(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
+          Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
         )
 
     def test_report_size_limit(self):
+        test_output = "f" * 1000
         self.assertEqual(
             generate_test_report_lib.generate_report(
                 "Foo",
@@ -371,14 +382,16 @@ def test_report_size_limit(self):
           <testsuites time="0.02">
           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
           <testcase classname="Bar/test_1" name="test_1" time="0.02">
-            <failure><![CDATA[Some long output goes here...]]></failure>
+            <failure><![CDATA[{output}]]></failure>
           </testcase>
           </testsuite>
-          </testsuites>"""
+          </testsuites>""".format(
+                                output=test_output
+                            )
                         )
                     )
                 ],
-                size_limit=128,
+                size_limit=512,
             ),
             (
                 dedent(
@@ -387,7 +400,9 @@ def test_report_size_limit(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+          Failed tests and their output was too large to report. Download the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),

.ci/metrics/metrics.py

@@ -67,6 +67,7 @@
 # remain small.
 BUILDKITE_GRAPHQL_BUILDS_PER_PAGE = 50
 
+
 @dataclass
 class JobMetrics:
     job_name: str
@@ -77,6 +78,7 @@ class JobMetrics:
     workflow_id: int
     workflow_name: str
 
+
 @dataclass
 class GaugeMetric:
     name: str
@@ -258,6 +260,7 @@ def buildkite_get_metrics(
 
     return output, incomplete_now
 
+
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:

.github/new-prs-labeler.yml

@@ -702,6 +702,7 @@ mlgo:
   - llvm/unittests/CodeGen/ML*
   - llvm/test/CodeGen/MLRegAlloc/**
   - llvm/utils/mlgo-utils/**
+  - llvm/docs/MLGO.rst
 
 tools:llvm-exegesis:
   - llvm/tools/llvm-exegesis/**

bolt/CMakeLists.txt

@@ -46,6 +46,10 @@ if(BOLT_BUILT_STANDALONE)
   set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin)
   set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX})
 
+  separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
+  add_definitions(${LLVM_DEFINITIONS_LIST})
+  list(APPEND CMAKE_REQUIRED_DEFINITIONS ${LLVM_DEFINITIONS_LIST})
+
   include(AddLLVM)
   include(TableGen)
   include_directories(${LLVM_INCLUDE_DIRS})

bolt/include/bolt/Core/BinaryFunction.h

@@ -804,6 +804,19 @@ class BinaryFunction {
     return iterator_range<const_cfi_iterator>(cie_begin(), cie_end());
   }
 
+  /// Iterate over instructions (only if CFG is unavailable or not built yet).
+  iterator_range<InstrMapType::iterator> instrs() {
+    assert(!hasCFG() && "Iterate over basic blocks instead");
+    return make_range(Instructions.begin(), Instructions.end());
+  }
+  iterator_range<InstrMapType::const_iterator> instrs() const {
+    assert(!hasCFG() && "Iterate over basic blocks instead");
+    return make_range(Instructions.begin(), Instructions.end());
+  }
+
+  /// Returns whether there are any labels at Offset.
+  bool hasLabelAt(unsigned Offset) const { return Labels.count(Offset) != 0; }
+
   /// Iterate over all jump tables associated with this function.
   iterator_range<std::map<uint64_t, JumpTable *>::const_iterator>
   jumpTables() const {

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -49,6 +49,7 @@ class MCSymbol;
 class raw_ostream;
 
 namespace bolt {
+class BinaryBasicBlock;
 class BinaryFunction;
 
 /// Different types of indirect branches encountered during disassembly.
@@ -572,6 +573,11 @@ class MCPlusBuilder {
     return false;
   }
 
+  virtual MCPhysReg getSignedReg(const MCInst &Inst) const {
+    llvm_unreachable("not implemented");
+    return getNoRegister();
+  }
+
   virtual ErrorOr<MCPhysReg> getRegUsedAsRetDest(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
     return getNoRegister();
@@ -622,6 +628,54 @@ class MCPlusBuilder {
     return std::make_pair(getNoRegister(), getNoRegister());
   }
 
+  /// Analyzes if a pointer is checked to be authenticated successfully
+  /// by the end of the basic block.
+  ///
+  /// It is possible for pointer authentication instructions not to terminate
+  /// the program abnormally on authentication failure and return some invalid
+  /// pointer instead (like it is done on AArch64 when FEAT_FPAC is not
+  /// implemented). This might be enough to crash on invalid memory access when
+  /// the pointer is later used as the destination of a load, store, or branch
+  /// instruction. On the other hand, when the pointer is not used right away,
+  /// it may be important for the compiler to check the address explicitly not
+  /// to introduce a signing or authentication oracle.
+  ///
+  /// This function is intended to detect a complex, multi-instruction pointer-
+  /// checking sequence spanning a contiguous range of instructions at the end
+  /// of the basic block (as these sequences are expected to end with a
+  /// conditional branch - this is how they are implemented on AArch64 by LLVM).
+  /// If a (Reg, FirstInst) pair is returned and before execution of FirstInst
+  /// Reg was last written to by an authentication instruction, then it is known
+  /// that in any successor of BB either
+  /// * the authentication instruction that last wrote to Reg succeeded, or
+  /// * the program is terminated abnormally without introducing any signing
+  ///   or authentication oracles
+  ///
+  /// Note that this function is not expected to repeat the results returned
+  /// by getAuthCheckedReg(Inst, MayOverwrite) function below.
+  virtual std::optional<std::pair<MCPhysReg, MCInst *>>
+  getAuthCheckedReg(BinaryBasicBlock &BB) const {
+    llvm_unreachable("not implemented");
+    return std::nullopt;
+  }
+
+  /// Returns the register that is checked to be authenticated successfully.
+  ///
+  /// If the returned register was last written to by an authentication
+  /// instruction and that authentication failed, then the program is known
+  /// to be terminated abnormally as a result of execution of Inst.
+  ///
+  /// Additionally, if MayOverwrite is false, it is known that the authenticated
+  /// pointer is not clobbered by Inst itself.
+  ///
+  /// Use this function for simple, single-instruction patterns instead of
+  /// its getAuthCheckedReg(BB) counterpart.
+  virtual MCPhysReg getAuthCheckedReg(const MCInst &Inst,
+                                      bool MayOverwrite) const {
+    llvm_unreachable("not implemented");
+    return getNoRegister();
+  }
+
   virtual bool isTerminator(const MCInst &Inst) const;
 
   virtual bool isNoop(const MCInst &Inst) const {

bolt/include/bolt/Passes/PAuthGadgetScanner.h

@@ -12,7 +12,6 @@
 #include "bolt/Core/BinaryContext.h"
 #include "bolt/Core/BinaryFunction.h"
 #include "bolt/Passes/BinaryPasses.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
 
@@ -65,6 +64,14 @@ struct MCInstInBFReference {
   uint64_t Offset;
   MCInstInBFReference(BinaryFunction *BF, uint64_t Offset)
       : BF(BF), Offset(Offset) {}
+
+  static MCInstInBFReference get(const MCInst *Inst, BinaryFunction &BF) {
+    for (auto &I : BF.instrs())
+      if (Inst == &I.second)
+        return MCInstInBFReference(&BF, I.first);
+    return {};
+  }
+
   MCInstInBFReference() : BF(nullptr), Offset(0) {}
   bool operator==(const MCInstInBFReference &RHS) const {
     return BF == RHS.BF && Offset == RHS.Offset;
@@ -104,6 +111,12 @@ struct MCInstReference {
   MCInstReference(BinaryFunction *BF, uint32_t Offset)
       : MCInstReference(MCInstInBFReference(BF, Offset)) {}
 
+  static MCInstReference get(const MCInst *Inst, BinaryFunction &BF) {
+    if (BF.hasCFG())
+      return MCInstInBBReference::get(Inst, BF);
+    return MCInstInBFReference::get(Inst, BF);
+  }
+
   bool operator<(const MCInstReference &RHS) const {
     if (ParentKind != RHS.ParentKind)
       return ParentKind < RHS.ParentKind;
@@ -138,6 +151,16 @@ struct MCInstReference {
     llvm_unreachable("");
   }
 
+  operator bool() const {
+    switch (ParentKind) {
+    case BasicBlockParent:
+      return U.BBRef.BB != nullptr;
+    case FunctionParent:
+      return U.BFRef.BF != nullptr;
+    }
+    llvm_unreachable("");
+  }
+
   uint64_t getAddress() const {
     switch (ParentKind) {
     case BasicBlockParent:
@@ -173,9 +196,6 @@ raw_ostream &operator<<(raw_ostream &OS, const MCInstReference &);
 
 namespace PAuthGadgetScanner {
 
-class SrcSafetyAnalysis;
-struct SrcState;
-
 /// Description of a gadget kind that can be detected. Intended to be
 /// statically allocated to be attached to reports by reference.
 class GadgetKind {
@@ -184,7 +204,7 @@ class GadgetKind {
 public:
   GadgetKind(const char *Description) : Description(Description) {}
 
-  const StringRef getDescription() const { return Description; }
+  StringRef getDescription() const { return Description; }
 };
 
 /// Base report located at some instruction, without any additional information.
@@ -199,8 +219,8 @@ struct Report {
 
   // The two methods below are called by Analysis::computeDetailedInfo when
   // iterating over the reports.
-  virtual const ArrayRef<MCPhysReg> getAffectedRegisters() const { return {}; }
-  virtual void setOverwritingInstrs(const ArrayRef<MCInstReference> Instrs) {}
+  virtual ArrayRef<MCPhysReg> getAffectedRegisters() const { return {}; }
+  virtual void setOverwritingInstrs(ArrayRef<MCInstReference> Instrs) {}
 
   void printBasicInfo(raw_ostream &OS, const BinaryContext &BC,
                       StringRef IssueKind) const;
@@ -223,19 +243,19 @@ struct GadgetReport : public Report {
 
   void generateReport(raw_ostream &OS, const BinaryContext &BC) const override;
 
-  const ArrayRef<MCPhysReg> getAffectedRegisters() const override {
+  ArrayRef<MCPhysReg> getAffectedRegisters() const override {
     return AffectedRegisters;
   }
 
-  void setOverwritingInstrs(const ArrayRef<MCInstReference> Instrs) override {
+  void setOverwritingInstrs(ArrayRef<MCInstReference> Instrs) override {
     OverwritingInstrs.assign(Instrs.begin(), Instrs.end());
   }
 };
 
 /// Report with a free-form message attached.
 struct GenericReport : public Report {
   std::string Text;
-  GenericReport(MCInstReference Location, const std::string &Text)
+  GenericReport(MCInstReference Location, StringRef Text)
       : Report(Location), Text(Text) {}
   virtual void generateReport(raw_ostream &OS,
                               const BinaryContext &BC) const override;

bolt/include/bolt/Profile/DataAggregator.h

@@ -212,11 +212,6 @@ class DataAggregator : public DataReader {
   uint64_t NumTraces{0};
   uint64_t NumInvalidTraces{0};
   uint64_t NumLongRangeTraces{0};
-  /// Specifies how many samples were recorded in cold areas if we are dealing
-  /// with profiling data collected in a bolted binary. For LBRs, incremented
-  /// for the source of the branch to avoid counting cold activity twice (one
-  /// for source and another for destination).
-  uint64_t NumColdSamples{0};
   uint64_t NumTotalSamples{0};
 
   /// Looks into system PATH for Linux Perf and set up the aggregator to use it
@@ -473,7 +468,6 @@ class DataAggregator : public DataReader {
   void dump(const PerfMemSample &Sample) const;
 
   /// Profile diagnostics print methods
-  void printColdSamplesDiagnostic() const;
   void printLongRangeTracesDiagnostic() const;
   void printBranchSamplesDiagnostics() const;
   void printBasicSamplesDiagnostics(uint64_t OutOfRangeSamples) const;