Extract SipHash implementation into a header.

llvm/lib/Support/CMakeLists.txt

@@ -379,3 +379,8 @@ if(LLVM_WITH_Z3)
     ${Z3_INCLUDE_DIR}
     )
 endif()
+
+target_include_directories(LLVMSupport SYSTEM
+  PRIVATE
+  ${LLVM_THIRD_PARTY_DIR}/siphash/include
+  )

llvm/lib/Support/SipHash.cpp

@@ -12,10 +12,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/SipHash.h"
+#include "siphash/SipHash.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include <cstdint>
@@ -25,135 +25,6 @@ using namespace support;
 
 #define DEBUG_TYPE "llvm-siphash"
 
-// Lightly adapted from the SipHash reference C implementation:
-//   https://github.com/veorq/SipHash
-// by Jean-Philippe Aumasson and Daniel J. Bernstein
-
-#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
-
-#define SIPROUND                                                               \
-  do {                                                                         \
-    v0 += v1;                                                                  \
-    v1 = ROTL(v1, 13);                                                         \
-    v1 ^= v0;                                                                  \
-    v0 = ROTL(v0, 32);                                                         \
-    v2 += v3;                                                                  \
-    v3 = ROTL(v3, 16);                                                         \
-    v3 ^= v2;                                                                  \
-    v0 += v3;                                                                  \
-    v3 = ROTL(v3, 21);                                                         \
-    v3 ^= v0;                                                                  \
-    v2 += v1;                                                                  \
-    v1 = ROTL(v1, 17);                                                         \
-    v1 ^= v2;                                                                  \
-    v2 = ROTL(v2, 32);                                                         \
-  } while (0)
-
-namespace {
-
-/// Computes a SipHash value
-///
-/// \param in: pointer to input data (read-only)
-/// \param inlen: input data length in bytes (any size_t value)
-/// \param k: reference to the key data 16-byte array (read-only)
-/// \returns output data, must be 8 or 16 bytes
-///
-template <int cROUNDS, int dROUNDS, size_t outlen>
-void siphash(const unsigned char *in, uint64_t inlen,
-             const unsigned char (&k)[16], unsigned char (&out)[outlen]) {
-
-  const unsigned char *ni = (const unsigned char *)in;
-  const unsigned char *kk = (const unsigned char *)k;
-
-  static_assert(outlen == 8 || outlen == 16, "result should be 8 or 16 bytes");
-
-  uint64_t v0 = UINT64_C(0x736f6d6570736575);
-  uint64_t v1 = UINT64_C(0x646f72616e646f6d);
-  uint64_t v2 = UINT64_C(0x6c7967656e657261);
-  uint64_t v3 = UINT64_C(0x7465646279746573);
-  uint64_t k0 = endian::read64le(kk);
-  uint64_t k1 = endian::read64le(kk + 8);
-  uint64_t m;
-  int i;
-  const unsigned char *end = ni + inlen - (inlen % sizeof(uint64_t));
-  const int left = inlen & 7;
-  uint64_t b = ((uint64_t)inlen) << 56;
-  v3 ^= k1;
-  v2 ^= k0;
-  v1 ^= k1;
-  v0 ^= k0;
-
-  if (outlen == 16)
-    v1 ^= 0xee;
-
-  for (; ni != end; ni += 8) {
-    m = endian::read64le(ni);
-    v3 ^= m;
-
-    for (i = 0; i < cROUNDS; ++i)
-      SIPROUND;
-
-    v0 ^= m;
-  }
-
-  switch (left) {
-  case 7:
-    b |= ((uint64_t)ni[6]) << 48;
-    LLVM_FALLTHROUGH;
-  case 6:
-    b |= ((uint64_t)ni[5]) << 40;
-    LLVM_FALLTHROUGH;
-  case 5:
-    b |= ((uint64_t)ni[4]) << 32;
-    LLVM_FALLTHROUGH;
-  case 4:
-    b |= ((uint64_t)ni[3]) << 24;
-    LLVM_FALLTHROUGH;
-  case 3:
-    b |= ((uint64_t)ni[2]) << 16;
-    LLVM_FALLTHROUGH;
-  case 2:
-    b |= ((uint64_t)ni[1]) << 8;
-    LLVM_FALLTHROUGH;
-  case 1:
-    b |= ((uint64_t)ni[0]);
-    break;
-  case 0:
-    break;
-  }
-
-  v3 ^= b;
-
-  for (i = 0; i < cROUNDS; ++i)
-    SIPROUND;
-
-  v0 ^= b;
-
-  if (outlen == 16)
-    v2 ^= 0xee;
-  else
-    v2 ^= 0xff;
-
-  for (i = 0; i < dROUNDS; ++i)
-    SIPROUND;
-
-  b = v0 ^ v1 ^ v2 ^ v3;
-  endian::write64le(out, b);
-
-  if (outlen == 8)
-    return;
-
-  v1 ^= 0xdd;
-
-  for (i = 0; i < dROUNDS; ++i)
-    SIPROUND;
-
-  b = v0 ^ v1 ^ v2 ^ v3;
-  endian::write64le(out + 8, b);
-}
-
-} // end anonymous namespace
-
 void llvm::getSipHash_2_4_64(ArrayRef<uint8_t> In, const uint8_t (&K)[16],
                              uint8_t (&Out)[8]) {
   siphash<2, 4>(In.data(), In.size(), K, Out);

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

@@ -169,7 +169,12 @@ class AArch64AsmPrinter : public AsmPrinter {
   void emitPtrauthTailCallHardening(const MachineInstr *TC);
 
   // Emit the sequence for AUT or AUTPAC.
-  void emitPtrauthAuthResign(const MachineInstr *MI);
+  void emitPtrauthAuthResign(Register AUTVal, AArch64PACKey::ID AUTKey,
+                             uint64_t AUTDisc,
+                             const MachineOperand *AUTAddrDisc,
+                             Register Scratch,
+                             std::optional<AArch64PACKey::ID> PACKey,
+                             uint64_t PACDisc, Register PACAddrDisc);
 
   // Emit the sequence to compute the discriminator.
   //
@@ -1867,7 +1872,8 @@ Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
                                                      Register AddrDisc,
                                                      Register ScratchReg,
                                                      bool MayUseAddrAsScratch) {
-  assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17);
+  assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17 ||
+         !STI->isX16X17Safer());
   // So far we've used NoRegister in pseudos.  Now we need real encodings.
   if (AddrDisc == AArch64::NoRegister)
     AddrDisc = AArch64::XZR;
@@ -1887,7 +1893,8 @@ Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
 
   // Check if we can save one MOV instruction.
   assert(MayUseAddrAsScratch || ScratchReg != AddrDisc);
-  bool AddrDiscIsSafe = AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
+  bool AddrDiscIsSafe = AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17 ||
+                        !STI->isX16X17Safer();
   if (MayUseAddrAsScratch && AddrDiscIsSafe)
     ScratchReg = AddrDisc;
   else
@@ -2065,8 +2072,12 @@ void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
       /*ShouldTrap=*/true, /*OnFailure=*/nullptr);
 }
 
-void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
-  const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
+void AArch64AsmPrinter::emitPtrauthAuthResign(
+    Register AUTVal, AArch64PACKey::ID AUTKey, uint64_t AUTDisc,
+    const MachineOperand *AUTAddrDisc, Register Scratch,
+    std::optional<AArch64PACKey::ID> PACKey, uint64_t PACDisc,
+    Register PACAddrDisc) {
+  const bool IsAUTPAC = PACKey.has_value();
 
   // We expand AUT/AUTPAC into a sequence of the form
   //
@@ -2105,23 +2116,19 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
     break;
   }
 
-  auto AUTKey = (AArch64PACKey::ID)MI->getOperand(0).getImm();
-  uint64_t AUTDisc = MI->getOperand(1).getImm();
-  unsigned AUTAddrDisc = MI->getOperand(2).getReg();
-
-  // Compute aut discriminator into x17
+  // Compute aut discriminator
   assert(isUInt<16>(AUTDisc));
-  Register AUTDiscReg =
-      emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc, AArch64::X17);
+  Register AUTDiscReg = emitPtrauthDiscriminator(
+      AUTDisc, AUTAddrDisc->getReg(), Scratch, AUTAddrDisc->isKill());
   bool AUTZero = AUTDiscReg == AArch64::XZR;
   unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero);
 
   //  autiza x16      ; if  AUTZero
   //  autia x16, x17  ; if !AUTZero
   MCInst AUTInst;
   AUTInst.setOpcode(AUTOpc);
-  AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
-  AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
+  AUTInst.addOperand(MCOperand::createReg(AUTVal));
+  AUTInst.addOperand(MCOperand::createReg(AUTVal));
   if (!AUTZero)
     AUTInst.addOperand(MCOperand::createReg(AUTDiscReg));
   EmitToStreamer(*OutStreamer, AUTInst);
@@ -2136,7 +2143,7 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
     if (IsAUTPAC && !ShouldTrap)
       EndSym = createTempSymbol("resign_end_");
 
-    emitPtrauthCheckAuthenticatedValue(AArch64::X16, AArch64::X17, AUTKey,
+    emitPtrauthCheckAuthenticatedValue(AUTVal, Scratch, AUTKey,
                                        AArch64PAuth::AuthCheckMethod::XPAC,
                                        ShouldTrap, EndSym);
   }
@@ -2147,23 +2154,19 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
   if (!IsAUTPAC)
     return;
 
-  auto PACKey = (AArch64PACKey::ID)MI->getOperand(3).getImm();
-  uint64_t PACDisc = MI->getOperand(4).getImm();
-  unsigned PACAddrDisc = MI->getOperand(5).getReg();
-
-  // Compute pac discriminator into x17
+  // Compute pac discriminator
   assert(isUInt<16>(PACDisc));
   Register PACDiscReg =
-      emitPtrauthDiscriminator(PACDisc, PACAddrDisc, AArch64::X17);
+      emitPtrauthDiscriminator(PACDisc, PACAddrDisc, Scratch);
   bool PACZero = PACDiscReg == AArch64::XZR;
-  unsigned PACOpc = getPACOpcodeForKey(PACKey, PACZero);
+  unsigned PACOpc = getPACOpcodeForKey(*PACKey, PACZero);
 
   //  pacizb x16      ; if  PACZero
   //  pacib x16, x17  ; if !PACZero
   MCInst PACInst;
   PACInst.setOpcode(PACOpc);
-  PACInst.addOperand(MCOperand::createReg(AArch64::X16));
-  PACInst.addOperand(MCOperand::createReg(AArch64::X16));
+  PACInst.addOperand(MCOperand::createReg(AUTVal));
+  PACInst.addOperand(MCOperand::createReg(AUTVal));
   if (!PACZero)
     PACInst.addOperand(MCOperand::createReg(PACDiscReg));
   EmitToStreamer(*OutStreamer, PACInst);
@@ -2862,9 +2865,26 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
   }
 
-  case AArch64::AUT:
+  case AArch64::AUTx16x17:
+    emitPtrauthAuthResign(AArch64::X16,
+                          (AArch64PACKey::ID)MI->getOperand(0).getImm(),
+                          MI->getOperand(1).getImm(), &MI->getOperand(2),
+                          AArch64::X17, std::nullopt, 0, 0);
+    return;
+
+  case AArch64::AUTxMxN:
+    emitPtrauthAuthResign(MI->getOperand(0).getReg(),
+                          (AArch64PACKey::ID)MI->getOperand(3).getImm(),
+                          MI->getOperand(4).getImm(), &MI->getOperand(5),
+                          MI->getOperand(1).getReg(), std::nullopt, 0, 0);
+    return;
+
   case AArch64::AUTPAC:
-    emitPtrauthAuthResign(MI);
+    emitPtrauthAuthResign(
+        AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(),
+        MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17,
+        (AArch64PACKey::ID)MI->getOperand(3).getImm(),
+        MI->getOperand(4).getImm(), MI->getOperand(5).getReg());
     return;
 
   case AArch64::LOADauthptrstatic:

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

@@ -1534,12 +1534,20 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
   std::tie(AUTConstDisc, AUTAddrDisc) =
       extractPtrauthBlendDiscriminators(AUTDisc, CurDAG);
 
-  SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
-                                         AArch64::X16, Val, SDValue());
-  SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)};
+  if (!Subtarget->isX16X17Safer()) {
+    SDValue Ops[] = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
 
-  SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops);
-  ReplaceNode(N, AUT);
+    SDNode *AUT =
+        CurDAG->getMachineNode(AArch64::AUTxMxN, DL, MVT::i64, MVT::i64, Ops);
+    ReplaceNode(N, AUT);
+  } else {
+    SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
+                                           AArch64::X16, Val, SDValue());
+    SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)};
+
+    SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTx16x17, DL, MVT::i64, Ops);
+    ReplaceNode(N, AUT);
+  }
 }
 
 void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -2129,10 +2129,11 @@ let Predicates = [HasPAuth] in {
   defm LDRAB  : AuthLoad<1, "ldrab", simm10Scaled>;
 
   // AUT pseudo.
-  // This directly manipulates x16/x17, which are the only registers the OS
-  // guarantees are safe to use for sensitive operations.
-  def AUT : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc, GPR64noip:$AddrDisc),
-                   []>, Sched<[WriteI, ReadI]> {
+  // This directly manipulates x16/x17, which are the only registers that
+  // certain OSs guarantee are safe to use for sensitive operations.
+  def AUTx16x17 : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc,
+                                      GPR64noip:$AddrDisc),
+                         []>, Sched<[WriteI, ReadI]> {
     let isCodeGenOnly = 1;
     let hasSideEffects = 1;
     let mayStore = 0;
@@ -2142,9 +2143,22 @@ let Predicates = [HasPAuth] in {
     let Uses = [X16];
   }
 
+  def AUTxMxN : Pseudo<(outs GPR64:$AuthVal, GPR64common:$Scratch),
+                       (ins GPR64:$Val, i32imm:$Key,
+                            i64imm:$Disc, GPR64:$AddrDisc),
+                       [], "$AuthVal = $Val">, Sched<[WriteI, ReadI]> {
+    let isCodeGenOnly = 1;
+    let hasSideEffects = 0;
+    let mayStore = 0;
+    let mayLoad = 0;
+    let Size = 32;
+    let Defs = [NZCV];
+    let Uses = [];
+  }
+
   // AUT and re-PAC a value, using different keys/data.
-  // This directly manipulates x16/x17, which are the only registers the OS
-  // guarantees are safe to use for sensitive operations.
+  // This directly manipulates x16/x17, which are the only registers that
+  // certain OSs guarantee are safe to use for sensitive operations.
   def AUTPAC
       : Pseudo<(outs),
                (ins i32imm:$AUTKey, i64imm:$AUTDisc, GPR64noip:$AUTAddrDisc,

llvm/lib/Target/AArch64/AArch64Subtarget.cpp

@@ -664,6 +664,12 @@ AArch64Subtarget::getPtrAuthBlockAddressDiscriminatorIfEnabled(
       (Twine(ParentFn.getName()) + " blockaddress").str());
 }
 
+bool AArch64Subtarget::isX16X17Safer() const {
+  // The Darwin kernel implements special protections for x16 and x17 so we
+  // should prefer to use those registers on that platform.
+  return isTargetDarwin();
+}
+
 bool AArch64Subtarget::enableMachinePipeliner() const {
   return getSchedModel().hasInstrSchedModel();
 }

llvm/lib/Target/AArch64/AArch64Subtarget.h

@@ -318,6 +318,10 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
     }
   }
 
+  /// Returns whether the operating system makes it safer to store sensitive
+  /// values in x16 and x17 as opposed to other registers.
+  bool isX16X17Safer() const;
+
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);