[llvm][aarch64][x86] Implement a lightweight spectre v1 mitigation, like MSVC /Qspectre

llvm/include/llvm/IR/Intrinsics.td

@@ -885,6 +885,9 @@ def int_readcyclecounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>;
 
 def int_readsteadycounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>;
 
+def int_speculative_data_barrier  : DefaultAttrsIntrinsic<[], [],
+                                            [IntrHasSideEffects]>;
+
 // The assume intrinsic is marked InaccessibleMemOnly so that proper control
 // dependencies will be maintained.
 def int_assume : DefaultAttrsIntrinsic<

llvm/include/llvm/InitializePasses.h

@@ -126,6 +126,7 @@ void initializeGVNLegacyPassPass(PassRegistry &);
 void initializeGlobalMergeFuncPassWrapperPass(PassRegistry &);
 void initializeGlobalMergePass(PassRegistry &);
 void initializeGlobalsAAWrapperPassPass(PassRegistry &);
+void initializeGuardedLoadHardeningPass(PassRegistry &);
 void initializeHardwareLoopsLegacyPass(PassRegistry &);
 void initializeMIRProfileLoaderPassPass(PassRegistry &);
 void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry &);

llvm/include/llvm/Transforms/Utils/GuardedLoadHardening.h

@@ -0,0 +1,31 @@
+//=== GuardedLoadHardening.h - Lightweight spectre v1 mitigation *- C++ -*===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+// Lightweight load hardening as a mitigation against Spectre v1.
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_GUARDEDLOADHARDENING_H
+#define LLVM_TRANSFORMS_GUARDEDLOADHARDENING_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class FunctionPass;
+
+class GuardedLoadHardeningPass
+    : public PassInfoMixin<GuardedLoadHardeningPass> {
+public:
+  GuardedLoadHardeningPass() = default;
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+FunctionPass *createGuardedLoadHardeningPass();
+
+} // namespace llvm
+
+#endif

llvm/lib/CodeGen/IntrinsicLowering.cpp

@@ -324,6 +324,10 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;
   }
 
+  case Intrinsic::speculative_data_barrier:
+    break; // Simply strip out speculative_data_barrier on unsupported
+           // architectures
+
   case Intrinsic::dbg_declare:
   case Intrinsic::dbg_label:
     break;    // Simply strip out debugging intrinsics

llvm/lib/Passes/PassBuilder.cpp

@@ -308,6 +308,7 @@
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 #include "llvm/Transforms/Utils/FixIrreducible.h"
+#include "llvm/Transforms/Utils/GuardedLoadHardening.h"
 #include "llvm/Transforms/Utils/HelloWorld.h"
 #include "llvm/Transforms/Utils/IRNormalizer.h"
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"

llvm/lib/Passes/PassRegistry.def

@@ -370,6 +370,7 @@ FUNCTION_PASS("flatten-cfg", FlattenCFGPass())
 FUNCTION_PASS("float2int", Float2IntPass())
 FUNCTION_PASS("gc-lowering", GCLoweringPass())
 FUNCTION_PASS("guard-widening", GuardWideningPass())
+FUNCTION_PASS("guarded-load-hardening", GuardedLoadHardeningPass())
 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
 FUNCTION_PASS("gvn-sink", GVNSinkPass())
 FUNCTION_PASS("helloworld", HelloWorldPass())

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -10607,6 +10607,9 @@ let Predicates = [HasLSFE] in {
 let Uses = [FPMR, FPCR] in
 defm FMMLA : SIMDThreeSameVectorFP8MatrixMul<"fmmla">;
 
+// Use the CSDB instruction as a barrier.
+def : Pat<(int_speculative_data_barrier), (HINT 0x14)>;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
 include "AArch64SMEInstrInfo.td"

llvm/lib/Target/AArch64/AArch64TargetMachine.cpp

@@ -49,6 +49,7 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/CFGuard.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/GuardedLoadHardening.h"
 #include "llvm/Transforms/Utils/LowerIFunc.h"
 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
 #include <memory>
@@ -669,6 +670,9 @@ void AArch64PassConfig::addIRPasses() {
       addPass(createCFGuardCheckPass());
   }
 
+  // Lightweight spectre v1 mitigation.
+  addPass(createGuardedLoadHardeningPass());
+
   if (TM->Options.JMCInstrument)
     addPass(createJMCInstrumenterPass());
 }

llvm/lib/Target/X86/X86InstrCompiler.td

@@ -2213,3 +2213,6 @@ def : Pat<(cttz_zero_undef (loadi64 addr:$src)), (BSF64rm addr:$src)>;
 let Predicates = [HasMOVBE] in {
  def : Pat<(bswap GR16:$src), (ROL16ri GR16:$src, (i8 8))>;
 }
+
+// Use the LFENCE instruction as a barrier.
+def : Pat<(int_speculative_data_barrier), (LFENCE)>;

llvm/lib/Target/X86/X86TargetMachine.cpp

@@ -48,6 +48,7 @@
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/CFGuard.h"
+#include "llvm/Transforms/Utils/GuardedLoadHardening.h"
 #include <memory>
 #include <optional>
 #include <string>
@@ -492,6 +493,9 @@ void X86PassConfig::addIRPasses() {
     }
   }
 
+  // Lightweight spectre v1 mitigation.
+  addPass(createGuardedLoadHardeningPass());
+
   if (TM->Options.JMCInstrument)
     addPass(createJMCInstrumenterPass());
 }

llvm/lib/Transforms/Utils/CMakeLists.txt

@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMTransformUtils
   FunctionComparator.cpp
   FunctionImportUtils.cpp
   GlobalStatus.cpp
+  GuardedLoadHardening.cpp
   GuardUtils.cpp
   HelloWorld.cpp
   InlineFunction.cpp