[CIR][CIRGen][Builtin][Neon] Lower neon_vldap1_lane_s64 and vldap1q_lane_s64 (#1346)

Lower `neon_vldap1_lane_s64` and `vldap1q_lane_s64`

To add atomic `MemOrder` I changed the return type of builder to return
LoadOp similar to our builders for StoreOp.

Author: AmrDeveloper

clang/lib/CIR/CodeGen/CIRGenBuilder.h

@@ -871,7 +871,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
         /*mem_order=*/cir::MemOrderAttr{}, /*tbaa=*/cir::TBAAAttr{});
   }
 
-  mlir::Value createAlignedLoad(mlir::Location loc, mlir::Type ty,
+  cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
                                 mlir::Value ptr, llvm::MaybeAlign align,
                                 bool isVolatile) {
     if (ty != mlir::cast<cir::PointerType>(ptr.getType()).getPointee())
@@ -880,14 +880,14 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     return CIRBaseBuilderTy::createLoad(loc, ptr, isVolatile, alignment);
   }
 
-  mlir::Value createAlignedLoad(mlir::Location loc, mlir::Type ty,
+  cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
                                 mlir::Value ptr, llvm::MaybeAlign align) {
     // TODO: make sure callsites shouldn't be really passing volatile.
     assert(!cir::MissingFeatures::volatileLoadOrStore());
     return createAlignedLoad(loc, ty, ptr, align, /*isVolatile=*/false);
   }
 
-  mlir::Value
+  cir::LoadOp
   createAlignedLoad(mlir::Location loc, mlir::Type ty, mlir::Value addr,
                     clang::CharUnits align = clang::CharUnits::One()) {
     return createAlignedLoad(loc, ty, addr, align.getAsAlign());

clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp

@@ -4453,7 +4453,12 @@ CIRGenFunction::emitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
   }
   case NEON::BI__builtin_neon_vldap1_lane_s64:
   case NEON::BI__builtin_neon_vldap1q_lane_s64: {
-    llvm_unreachable("NEON::BI__builtin_neon_vldap1q_lane_s64 NYI");
+    cir::LoadOp Load = builder.createAlignedLoad(
+        Ops[0].getLoc(), vTy.getEltType(), Ops[0], PtrOp0.getAlignment());
+    Load.setAtomic(cir::MemOrder::Acquire);
+    return builder.create<cir::VecInsertOp>(getLoc(E->getExprLoc()),
+                                            builder.createBitcast(Ops[1], vTy),
+                                            Load, Ops[2]);
   }
   case NEON::BI__builtin_neon_vld1_dup_v:
   case NEON::BI__builtin_neon_vld1q_dup_v: {

clang/test/CIR/CodeGen/AArch64/neon-ldst.c

@@ -629,3 +629,140 @@ void test_vstl1_lane_p64(poly64_t  *a, poly64x1_t b) {
 // LLVM: [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
 // LLVM: [[TMP2:%.*]] = extractelement <1 x i64> [[TMP1]], i32 0
 // LLVM: store atomic i64 [[TMP2]], ptr [[PTR]] release, align 8
+
+uint64x2_t test_vldap1q_lane_u64(uint64_t  *a, uint64x2_t b) {
+  return vldap1q_lane_u64(a, b, 1);
+}
+
+// CIR-LABEL:test_vldap1q_lane_u64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!u64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!u64i>, !u64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 16>), !cir.vector<!u64i x 2>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!u64i x 2>
+
+// LLVM: {{.*}}test_vldap1q_lane_u64(ptr{{.*}}[[PTR:%.*]], <2 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <2 x i64> [[SRC]] to <16 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[TMP2]], i32 1
+
+int64x2_t test_vldap1q_lane_s64(int64_t  *a, int64x2_t b) {
+  return vldap1q_lane_s64(a, b, 1);
+}
+
+// CIR-LABEL:test_vldap1q_lane_s64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!s64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!s64i>, !s64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 16>), !cir.vector<!s64i x 2>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!s64i x 2>
+
+// LLVM: {{.*}}test_vldap1q_lane_s64(ptr{{.*}}[[PTR:%.*]], <2 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <2 x i64> [[SRC]] to <16 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[TMP2]], i32 1
+
+float64x2_t test_vldap1q_lane_f64(float64_t  *a, float64x2_t b) {
+  return vldap1q_lane_f64(a, b, 1);
+}
+
+// CIR-LABEL:test_vldap1q_lane_f64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!cir.double>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!cir.double>, !cir.double
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 16>), !cir.vector<!cir.double x 2>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!cir.double x 2>
+
+// LLVM: {{.*}}test_vldap1q_lane_f64(ptr{{.*}}[[PTR:%.*]], <2 x double>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <2 x double> [[SRC]] to <16 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic double, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x double>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <2 x double> [[TMP1]], double [[TMP2]], i32 1
+
+poly64x2_t test_vldap1q_lane_p64(poly64_t  *a, poly64x2_t b) {
+  return vldap1q_lane_p64(a, b, 1);
+}
+
+// CIR-LABEL:test_vldap1q_lane_p64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!s64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!s64i>, !s64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 16>), !cir.vector<!s64i x 2>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!s64i x 2>
+
+// LLVM: {{.*}}test_vldap1q_lane_p64(ptr{{.*}}[[PTR:%.*]], <2 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <2 x i64> [[SRC]] to <16 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[TMP2]], i32 1
+
+uint64x1_t test_vldap1_lane_u64(uint64_t  *a, uint64x1_t b) {
+  return vldap1_lane_u64(a, b, 0);
+}
+
+// CIR-LABEL:test_vldap1_lane_u64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!u64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!u64i>, !u64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 8>), !cir.vector<!u64i x 1>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!u64i x 1>
+
+// LLVM: {{.*}}test_vldap1_lane_u64(ptr{{.*}}[[PTR:%.*]], <1 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <1 x i64> [[SRC]] to <8 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <1 x i64> [[TMP1]], i64 [[TMP2]], i32 0
+
+int64x1_t test_vldap1_lane_s64(int64_t  *a, int64x1_t b) {
+  return vldap1_lane_s64(a, b, 0);
+}
+
+// CIR-LABEL:test_vldap1_lane_s64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!s64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!s64i>, !s64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 8>), !cir.vector<!s64i x 1>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!s64i x 1>
+
+// LLVM: {{.*}}test_vldap1_lane_s64(ptr{{.*}}[[PTR:%.*]], <1 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <1 x i64> [[SRC]] to <8 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <1 x i64> [[TMP1]], i64 [[TMP2]], i32 0
+
+
+float64x1_t test_vldap1_lane_f64(float64_t  *a, float64x1_t b) {
+  return vldap1_lane_f64(a, b, 0);
+}
+
+// CIR-LABEL: test_vldap1_lane_f64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!cir.double>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!cir.double>, !cir.double
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 8>), !cir.vector<!cir.double x 1>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!cir.double x 1>
+
+// LLVM: {{.*}}test_vldap1_lane_f64(ptr{{.*}}[[PTR:%.*]], <1 x double>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <1 x double> [[SRC]] to <8 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic double, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x double>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <1 x double> [[TMP1]], double [[TMP2]], i32 0
+
+poly64x1_t test_vldap1_lane_p64(poly64_t  *a, poly64x1_t b) {
+  return vldap1_lane_p64(a, b, 0);
+}
+
+// CIR-LABEL: test_vldap1_lane_p64
+// CIR: [[LANE:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR: [[TMP0:%.*]] = cir.cast(bitcast, {{.*}} : !cir.ptr<!void>), !cir.ptr<!s64i>
+// CIR: [[VAL:%.*]] = cir.load align(8) atomic(acquire) [[TMP0]] : !cir.ptr<!s64i>, !s64
+// CIR: [[VEC:%.*]] = cir.cast(bitcast, {{.*}} : !cir.vector<!s8i x 8>), !cir.vector<!s64i x 1>
+// CIR: [[TMP:%.*]]  = cir.vec.insert [[VAL]], {{.*}}[[[LANE]] : !s32i] : !cir.vector<!s64i x 1>
+
+// LLVM: {{.*}}test_vldap1_lane_p64(ptr{{.*}}[[PTR:%.*]], <1 x i64>{{.*}}[[SRC:%.*]])
+// LLVM: [[TMP0:%.*]] = bitcast <1 x i64> [[SRC]] to <8 x i8>
+// LLVM: [[TMP2:%.*]] = load atomic i64, ptr [[PTR]] acquire, align 8
+// LLVM: [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+// LLVM: [[VLDAP1_LANE:%.*]] = insertelement <1 x i64> [[TMP1]], i64 [[TMP2]], i32 0