|
| 1 | +diff --git a/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp b/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp |
| 2 | +index d6f4c5220f71d..9297711d53612 100644 |
| 3 | +--- a/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp |
| 4 | ++++ b/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp |
| 5 | +@@ -8,10 +8,17 @@ |
| 6 | + #include "../Target.h" |
| 7 | + #include "AArch64.h" |
| 8 | + #include "AArch64RegisterInfo.h" |
| 9 | ++#include "llvm/Support/CommandLine.h" |
| 10 | + |
| 11 | + #if defined(__aarch64__) && defined(__linux__) |
| 12 | +-#include <linux/prctl.h> // For PR_PAC_* constants |
| 13 | +-#include <sys/prctl.h> |
| 14 | ++#include <errno.h> |
| 15 | ++#include <sys/prctl.h> // For PR_PAC_* constants |
| 16 | ++#ifndef PR_PAC_SET_ENABLED_KEYS |
| 17 | ++#define PR_PAC_SET_ENABLED_KEYS 60 |
| 18 | ++#endif |
| 19 | ++#ifndef PR_PAC_GET_ENABLED_KEYS |
| 20 | ++#define PR_PAC_GET_ENABLED_KEYS 61 |
| 21 | ++#endif |
| 22 | + #ifndef PR_PAC_APIAKEY |
| 23 | + #define PR_PAC_APIAKEY (1UL << 0) |
| 24 | + #endif |
| 25 | +@@ -32,6 +39,53 @@ |
| 26 | + namespace llvm { |
| 27 | + namespace exegesis { |
| 28 | + |
| 29 | ++static cl::opt<bool> AArch64DisablePacControl( |
| 30 | ++ "aarch64-disable-pac-control", |
| 31 | ++ cl::desc("Disable PAC key control at runtime for benchmarking. Use this if " |
| 32 | ++ "llvm-exegesis crashes or instruction timings are affected."), |
| 33 | ++ cl::init(false)); |
| 34 | ++ |
| 35 | ++bool isPointerAuth(unsigned Opcode) { |
| 36 | ++ switch (Opcode) { |
| 37 | ++ default: |
| 38 | ++ return false; |
| 39 | ++ |
| 40 | ++ // FIXME: Pointer Authentication instructions. |
| 41 | ++ // We would like to measure these instructions, but they can behave |
| 42 | ++ // differently on different platforms, and maybe the snippets need to look |
| 43 | ++ // different for these instructions, |
| 44 | ++ // Platform-specific handling: On Linux, we disable authentication, may |
| 45 | ++ // interfere with measurements. On non-Linux platforms, disable opcodes for |
| 46 | ++ // now. |
| 47 | ++ case AArch64::AUTDA: |
| 48 | ++ case AArch64::AUTDB: |
| 49 | ++ case AArch64::AUTDZA: |
| 50 | ++ case AArch64::AUTDZB: |
| 51 | ++ case AArch64::AUTIA: |
| 52 | ++ case AArch64::AUTIA1716: |
| 53 | ++ case AArch64::AUTIASP: |
| 54 | ++ case AArch64::AUTIAZ: |
| 55 | ++ case AArch64::AUTIB: |
| 56 | ++ case AArch64::AUTIB1716: |
| 57 | ++ case AArch64::AUTIBSP: |
| 58 | ++ case AArch64::AUTIBZ: |
| 59 | ++ case AArch64::AUTIZA: |
| 60 | ++ case AArch64::AUTIZB: |
| 61 | ++ return true; |
| 62 | ++ } |
| 63 | ++} |
| 64 | ++ |
| 65 | ++bool isLoadTagMultiple(unsigned Opcode) { |
| 66 | ++ switch (Opcode) { |
| 67 | ++ default: |
| 68 | ++ return false; |
| 69 | ++ |
| 70 | ++ // Load tag multiple instruction |
| 71 | ++ case AArch64::LDGM: |
| 72 | ++ return true; |
| 73 | ++ } |
| 74 | ++} |
| 75 | ++ |
| 76 | + static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) { |
| 77 | + switch (RegBitWidth) { |
| 78 | + case 32: |
| 79 | +@@ -151,6 +205,72 @@ class ExegesisAArch64Target : public ExegesisTarget { |
| 80 | + // Function return is a pseudo-instruction that needs to be expanded |
| 81 | + PM.add(createAArch64ExpandPseudoPass()); |
| 82 | + } |
| 83 | ++ |
| 84 | ++#if defined(__aarch64__) && defined(__linux__) |
| 85 | ++ // Converts variadic arguments to `long` and passes zeros for the unused |
| 86 | ++ // arg2-arg5, as tested by the Linux kernel. |
| 87 | ++ static long prctl_wrapper(int op, long arg2 = 0, long arg3 = 0) { |
| 88 | ++ return prctl(op, arg2, arg3, /*arg4=*/0L, /*arg5=*/0L); |
| 89 | ++ } |
| 90 | ++#endif |
| 91 | ++ |
| 92 | ++ const char *getIgnoredOpcodeReasonOrNull(const LLVMState &State, |
| 93 | ++ unsigned Opcode) const override { |
| 94 | ++ if (const char *Reason = |
| 95 | ++ ExegesisTarget::getIgnoredOpcodeReasonOrNull(State, Opcode)) |
| 96 | ++ return Reason; |
| 97 | ++ |
| 98 | ++ if (isPointerAuth(Opcode)) { |
| 99 | ++#if defined(__aarch64__) && defined(__linux__) |
| 100 | ++ // Only proceed with PAC key control if explicitly requested |
| 101 | ++ if (!AArch64DisablePacControl) { |
| 102 | ++ // For some systems with existing PAC keys set, it is better to |
| 103 | ++ // check the existing state of the key before setting it. |
| 104 | ++ // If the CPU implements FEAT_FPAC, |
| 105 | ++ // authentication instructions almost certainly crash when being |
| 106 | ++ // benchmarked, so disable all the keys by default. On the other hand, |
| 107 | ++ // disabling the keys at run-time can probably crash llvm-exegesis at |
| 108 | ++ // some later point, depending on how it was built. For that reason, the |
| 109 | ++ // user may pass --aarch64-disable-pac-control in case |
| 110 | ++ // llvm-exegesis crashes or instruction timings are affected. |
| 111 | ++ // Hence the guard for switching. |
| 112 | ++ errno = 0; |
| 113 | ++ long PacKeys = prctl_wrapper(PR_PAC_GET_ENABLED_KEYS); |
| 114 | ++ if (PacKeys < 0 || errno == EINVAL) |
| 115 | ++ return nullptr; |
| 116 | ++ |
| 117 | ++ // Disable all PAC keys. Note that while we expect the measurements to |
| 118 | ++ // be the same with PAC keys disabled, they could potentially be lower |
| 119 | ++ // since authentication checks are bypassed.PR_PAC_* prctl operations |
| 120 | ++ // return EINVAL when Pointer Authentication is not available, but no |
| 121 | ++ // more errors are expected if we got here. |
| 122 | ++ if (PacKeys != 0) { |
| 123 | ++ // Operate on all keys. |
| 124 | ++ const long KeysToControl = |
| 125 | ++ PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY; |
| 126 | ++ // PR_PAC_* prctl operations return EINVAL when Pointer Authentication |
| 127 | ++ // is not available but no more errors are expected if we got here. |
| 128 | ++ const long EnabledBitMask = 0; |
| 129 | ++ if (prctl_wrapper(PR_PAC_SET_ENABLED_KEYS, KeysToControl, |
| 130 | ++ EnabledBitMask) < 0) { |
| 131 | ++ return "Failed to disable PAC keys"; |
| 132 | ++ } |
| 133 | ++ llvm::errs() |
| 134 | ++ << "llvm-exegesis: PAC keys were disabled at runtime for " |
| 135 | ++ "benchmarking.\n"; |
| 136 | ++ } |
| 137 | ++ } |
| 138 | ++#else |
| 139 | ++ // Silently return nullptr to ensure forward progress |
| 140 | ++ return nullptr; |
| 141 | ++#endif |
| 142 | ++ } |
| 143 | ++ |
| 144 | ++ if (isLoadTagMultiple(Opcode)) |
| 145 | ++ return "Unsupported opcode: load tag multiple"; |
| 146 | ++ |
| 147 | ++ return nullptr; |
| 148 | ++ } |
| 149 | + }; |
| 150 | + |
| 151 | + } // namespace |
0 commit comments