[MLIR][LLVMIR] Import unregistered intrinsics via llvm.intrinsic_call

mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt

@@ -81,3 +81,29 @@ mlir_tablegen(VCIXConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(VCIXOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=vcix)
 mlir_tablegen(VCIXOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=vcix)
 add_public_tablegen_target(MLIRVCIXConversionsIncGen)
+
+# FIXME: Should emit extra file with intrinsics_gen if MLIR is enabled? Or maybe find
+# a better way to get the path for Intrinsics.td?
+set(LLVM_TARGET_DEFINITIONS ../../../../../llvm/include/llvm/IR/Intrinsics.td)
+
+mlir_tablegen(LLVMUnregisteredLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics)
+mlir_tablegen(LLVMUnregisteredAArch64LLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=aarch64)
+mlir_tablegen(LLVMUnregisteredAMDGPULLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=amdgcn)
+mlir_tablegen(LLVMUnregisteredARMLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=arm)
+mlir_tablegen(LLVMUnregisteredBPFLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=bpf)
+mlir_tablegen(LLVMUnregisteredDirectXLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=dx)
+mlir_tablegen(LLVMUnregisteredHexagonLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=hexagon)
+mlir_tablegen(LLVMUnregisteredLoongArchLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=loongarch)
+mlir_tablegen(LLVMUnregisteredMipsLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=mips)
+mlir_tablegen(LLVMUnregisteredNVPTXLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=nvvm)
+mlir_tablegen(LLVMUnregisteredPowerPCLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=ppc)
+mlir_tablegen(LLVMUnregisteredR600LLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=r600)
+mlir_tablegen(LLVMUnregisteredRISCVLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=riscv)
+mlir_tablegen(LLVMUnregisteredSPIRVLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=spv)
+mlir_tablegen(LLVMUnregisteredS390LLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=s390)
+mlir_tablegen(LLVMUnregisteredWebAssemblyLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=wasm)
+mlir_tablegen(LLVMUnregisteredX86LLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=x86)
+mlir_tablegen(LLVMUnregisteredXCoreLLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=xcore)
+mlir_tablegen(LLVMUnregisteredVELLVMIRIntrinsics.inc -gen-unregistered-llvmir-intrinsics -intrinsic-prefix=ve)
+
+add_public_tablegen_target(MLIRLLVMIntrinsicUnregisteredIncGen)

mlir/include/mlir/Target/LLVMIR/LLVMImportInterface.h

@@ -74,6 +74,10 @@ class LLVMImportDialectInterface
   /// returns the list of supported intrinsic identifiers.
   virtual ArrayRef<unsigned> getSupportedIntrinsics() const { return {}; }
 
+  /// Returns the list of LLVM IR intrinsic identifiers that are unsupported
+  /// but dialects might have a generic way to represent them.
+  virtual ArrayRef<unsigned> getUnregisteredIntrinsics() const { return {}; }
+
   /// Hook for derived dialect interfaces to publish the supported instructions.
   /// As every LLVM IR instruction has a unique integer identifier, the function
   /// returns the list of supported instruction identifiers. These identifiers
@@ -139,6 +143,9 @@ class LLVMImportInterface
       // Add a mapping for all supported intrinsic identifiers.
       for (unsigned id : iface.getSupportedIntrinsics())
         intrinsicToDialect[id] = iface.getDialect();
+      // Add a mapping for all unregistered intrinsic identifiers.
+      for (unsigned id : iface.getUnregisteredIntrinsics())
+        unregisteredIntrinscToDialect[id] = iface.getDialect();
       // Add a mapping for all supported instruction identifiers.
       for (unsigned id : iface.getSupportedInstructions())
         instructionToDialect[id] = &iface;
@@ -155,7 +162,19 @@ class LLVMImportInterface
   LogicalResult convertIntrinsic(OpBuilder &builder, llvm::CallInst *inst,
                                  LLVM::ModuleImport &moduleImport) const {
     // Lookup the dialect interface for the given intrinsic.
-    Dialect *dialect = intrinsicToDialect.lookup(inst->getIntrinsicID());
+    llvm::Intrinsic::ID intrinId = inst->getIntrinsicID();
+    if (intrinId == llvm::Intrinsic::not_intrinsic)
+      return failure();
+
+    // First lookup intrinsic across different dialects for known
+    // supported converstions, examples include arm-neon, nvm-sve, etc
+    Dialect *dialect = intrinsicToDialect.lookup(intrinId);
+
+    // No specialized (supported) intrinsics, attempt to generate a generic
+    // version via llvm.call_intrinsic (if available).
+    if (!dialect)
+      dialect = unregisteredIntrinscToDialect.lookup(intrinId);
+
     if (!dialect)
       return failure();
 
@@ -227,6 +246,9 @@ class LLVMImportInterface
   DenseMap<unsigned, Dialect *> intrinsicToDialect;
   DenseMap<unsigned, const LLVMImportDialectInterface *> instructionToDialect;
   DenseMap<unsigned, SmallVector<Dialect *, 1>> metadataToDialect;
+
+  /// Unregistered generic and target independent intrinsics.
+  DenseMap<unsigned, Dialect *> unregisteredIntrinscToDialect;
 };
 
 } // namespace mlir

mlir/lib/Dialect/LLVMIR/CMakeLists.txt

@@ -17,6 +17,9 @@ add_mlir_dialect_library(MLIRLLVMDialect
   MLIRLLVMTypesIncGen
   MLIRLLVMIntrinsicOpsIncGen
   MLIRLLVMInterfacesIncGen
+  MLIRLLVMConversionsIncGen
+  MLIRLLVMIntrinsicConversionsIncGen
+  MLIRLLVMIntrinsicUnregisteredIncGen
   MLIROpenMPOpsIncGen
   intrinsics_gen
 

mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp

@@ -24,6 +24,24 @@
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/IntrinsicsARM.h"
+#include "llvm/IR/IntrinsicsBPF.h"
+#include "llvm/IR/IntrinsicsDirectX.h"
+#include "llvm/IR/IntrinsicsHexagon.h"
+#include "llvm/IR/IntrinsicsLoongArch.h"
+#include "llvm/IR/IntrinsicsMips.h"
+#include "llvm/IR/IntrinsicsNVPTX.h"
+#include "llvm/IR/IntrinsicsPowerPC.h"
+#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
+#include "llvm/IR/IntrinsicsS390.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/IR/IntrinsicsVE.h"
+#include "llvm/IR/IntrinsicsWebAssembly.h"
+#include "llvm/IR/IntrinsicsX86.h"
+#include "llvm/IR/IntrinsicsXCore.h"
 #include "llvm/Support/ModRef.h"
 
 using namespace mlir;
@@ -56,6 +74,105 @@ static ArrayRef<unsigned> getSupportedIntrinsicsImpl() {
   return convertibleIntrinsics;
 }
 
+/// Returns true if the LLVM IR intrinsic is convertible to llvm.intrinsic_call
+/// Returns false otherwise.
+static bool isConvertibleUnregisteredIntrinsic(llvm::Intrinsic::ID id) {
+  static const DenseSet<unsigned> convertibleTargetIntrinsics = {
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredAArch64LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredAMDGPULLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredARMLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredBPFLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredDirectXLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredHexagonLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredLoongArchLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredMipsLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredNVPTXLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredPowerPCLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredR600LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredRISCVLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredS390LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredSPIRVLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredVELLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredWebAssemblyLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredX86LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredXCoreLLVMIRIntrinsics.inc"
+  };
+  return convertibleTargetIntrinsics.contains(id);
+}
+
+/// Returns the list of LLVM IR intrinsic identifiers that are not registered
+/// by any dialect but can be convertible to llvm.intrinsic_call operation.
+static ArrayRef<unsigned> getUnregisteredIntrinsicsImpl() {
+  static const SmallVector<unsigned> convertibleTargetIntrinsics = {
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredAArch64LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredAMDGPULLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredARMLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredBPFLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredDirectXLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredHexagonLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredLoongArchLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredMipsLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredNVPTXLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredPowerPCLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredR600LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredRISCVLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredS390LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredSPIRVLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredVELLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredWebAssemblyLLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredX86LLVMIRIntrinsics.inc"
+#include "mlir/Dialect/LLVMIR/LLVMUnregisteredXCoreLLVMIRIntrinsics.inc"
+  };
+  return convertibleTargetIntrinsics;
+}
+
+/// Converts the LLVM intrinsic to a generic LLVM intrinsic call using
+/// llvm.intrinsic_call. Returns failure otherwise.
+static LogicalResult
+convertUnregisteredIntrinsicImpl(OpBuilder &odsBuilder, llvm::CallInst *inst,
+                                 LLVM::ModuleImport &moduleImport) {
+  llvm::Intrinsic::ID intrinsicID = inst->getIntrinsicID();
+  StringRef intrinName = inst->getCalledFunction()->getName();
+
+  // Sanity check the intrinsic ID.
+  assert(isConvertibleUnregisteredIntrinsic(intrinsicID));
+  SmallVector<llvm::Value *> args(inst->args());
+  ArrayRef<llvm::Value *> llvmOperands(args);
+
+  SmallVector<llvm::OperandBundleUse> llvmOpBundles;
+  llvmOpBundles.reserve(inst->getNumOperandBundles());
+  for (unsigned i = 0; i < inst->getNumOperandBundles(); ++i)
+    llvmOpBundles.push_back(inst->getOperandBundleAt(i));
+
+  SmallVector<Value> mlirOperands;
+  SmallVector<NamedAttribute> mlirAttrs;
+  if (failed(moduleImport.convertIntrinsicArguments(
+          llvmOperands, llvmOpBundles, false, {}, {}, mlirOperands, mlirAttrs)))
+    return failure();
+
+  mlir::Type results = moduleImport.convertType(inst->getType());
+  auto op = odsBuilder.create<::mlir::LLVM::CallIntrinsicOp>(
+      moduleImport.translateLoc(inst->getDebugLoc()), results,
+      StringAttr::get(odsBuilder.getContext(), intrinName),
+      ValueRange{mlirOperands}, FastmathFlagsAttr{});
+
+  moduleImport.setFastmathFlagsAttr(inst, op);
+
+  // Update importer tracking of results.
+  unsigned numRes = op.getNumResults();
+  if (numRes == 1)
+    moduleImport.mapValue(inst) = op.getResult(0);
+  else if (numRes == 0)
+    moduleImport.mapNoResultOp(inst);
+  else
+    return op.emitError(
+        "expected at most one result from target intrinsic call");
+
+  return success();
+}
+
 /// Converts the LLVM intrinsic to an MLIR LLVM dialect operation if a
 /// conversion exits. Returns failure otherwise.
 static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder,
@@ -75,6 +192,8 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder,
       llvmOpBundles.push_back(inst->getOperandBundleAt(i));
 
 #include "mlir/Dialect/LLVMIR/LLVMIntrinsicFromLLVMIRConversions.inc"
+  } else if (isConvertibleUnregisteredIntrinsic(intrinsicID)) {
+    return convertUnregisteredIntrinsicImpl(odsBuilder, inst, moduleImport);
   }
 
   return failure();
@@ -422,6 +541,12 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface {
     return getSupportedIntrinsicsImpl();
   }
 
+  /// Returns the list of LLVM IR intrinsic identifiers that are unsupported
+  /// by existing dialects by are convertible to generic llvm.call_intrinsic.
+  ArrayRef<unsigned> getUnregisteredIntrinsics() const final {
+    return getUnregisteredIntrinsicsImpl();
+  }
+
   /// Returns the list of LLVM IR metadata kinds that are convertible to MLIR
   /// LLVM dialect attributes.
   ArrayRef<unsigned>

mlir/test/Target/LLVMIR/Import/import-failure.ll

@@ -38,18 +38,6 @@ bb1:
 
 ; // -----
 
-declare void @llvm.gcroot(ptr %arg1, ptr %arg2)
-
-; CHECK:      <unknown>
-; CHECK-SAME: error: unhandled intrinsic: call void @llvm.gcroot(ptr %arg1, ptr null)
-define void @unhandled_intrinsic() gc "example" {
-  %arg1 = alloca ptr
-  call void @llvm.gcroot(ptr %arg1, ptr null)
-  ret void
-}
-
-; // -----
-
 ; Check that debug intrinsics with an unsupported argument are dropped.
 
 declare void @llvm.dbg.value(metadata, metadata, metadata)

mlir/test/Target/LLVMIR/Import/intrinsic-unregistered.ll

@@ -0,0 +1,68 @@
+; RUN: mlir-translate -import-llvm %s -split-input-file | FileCheck %s
+
+declare i64 @llvm.aarch64.ldxr.p0(ptr)
+
+define dso_local void @t0(ptr %a) {
+  %x = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i8) %a)
+  ret void
+}
+
+; CHECK-LABEL: llvm.func @llvm.aarch64.ldxr.p0(!llvm.ptr)
+; CHECK-LABEL: llvm.func @t0
+; CHECK:   llvm.call_intrinsic "llvm.aarch64.ldxr.p0"({{.*}}) : (!llvm.ptr) -> i64
+; CHECK:   llvm.return
+; CHECK: }
+
+; -----
+
+declare <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8>, <8 x i8>)
+
+define dso_local <8 x i8> @t1(<8 x i8> %lhs, <8 x i8> %rhs) {
+  %r = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %lhs, <8 x i8> %rhs)
+  ret <8 x i8> %r
+}
+
+; CHECK: llvm.func @t1(%[[A0:.*]]: vector<8xi8>, %[[A1:.*]]: vector<8xi8>) -> vector<8xi8> {{.*}} {
+; CHECK:   %[[R:.*]] = llvm.call_intrinsic "llvm.aarch64.neon.uabd.v8i8"(%[[A0]], %[[A1]]) : (vector<8xi8>, vector<8xi8>) -> vector<8xi8>
+; CHECK:   llvm.return %[[R]] : vector<8xi8>
+; CHECK: }
+
+; -----
+
+declare void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8>, <8 x i8>, ptr)
+
+define dso_local void @t2(<8 x i8> %lhs, <8 x i8> %rhs, ptr %a) {
+  call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> %lhs, <8 x i8> %rhs, ptr %a)
+  ret void
+}
+
+; CHECK: llvm.func @t2(%[[A0:.*]]: vector<8xi8>, %[[A1:.*]]: vector<8xi8>, %[[A2:.*]]: !llvm.ptr) {{.*}} {
+; CHECK:   llvm.call_intrinsic "llvm.aarch64.neon.st2.v8i8.p0"(%[[A0]], %[[A1]], %[[A2]]) : (vector<8xi8>, vector<8xi8>, !llvm.ptr) -> !llvm.void
+; CHECK:   llvm.return
+; CHECK: }
+
+; -----
+
+declare void @llvm.gcroot(ptr %arg1, ptr %arg2)
+define void @gctest() gc "example" {
+  %arg1 = alloca ptr
+  call void @llvm.gcroot(ptr %arg1, ptr null)
+  ret void
+}
+
+; CHECK-LABEL: @gctest
+; CHECK: llvm.call_intrinsic "llvm.gcroot"({{.*}}, {{.*}}) : (!llvm.ptr, !llvm.ptr) -> !llvm.void
+
+; -----
+
+; Test we get the supported version, not the unregistered one.
+
+declare i32 @llvm.lround.i32.f32(float)
+
+; CHECK-LABEL: llvm.func @lround_test
+define void @lround_test(float %0, double %1) {
+  ; CHECK-NOT: llvm.call_intrinsic "llvm.lround
+  ; CHECK: llvm.intr.lround(%{{.*}}) : (f32) -> i32
+  %3 = call i32 @llvm.lround.i32.f32(float %0)
+  ret void
+}