diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index b00aa11f8499d..35b222e2d55eb 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1702,6 +1702,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::MLOAD, VT, Custom); setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); + setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); + setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); if (Subtarget->hasSVEB16B16()) { diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index e443c5ab150bd..5af2f98f18132 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -3079,6 +3079,8 @@ multiclass sve_int_perm_bin_perm_zz opc, string asm, def : SVE_2_Op_Pat(NAME # _D)>; def : SVE_2_Op_Pat(NAME # _H)>; + def : SVE_2_Op_Pat(NAME # _S)>; + def : SVE_2_Op_Pat(NAME # _D)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll index fd1365d56fee4..adf1b48b6998a 100644 --- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll @@ -10,7 +10,7 @@ define {, } @vector_deinterleave_nxv2f16_n ; CHECK-NEXT: uunpklo z1.d, z2.s ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv4f16( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv4f16_nxv8f16( %vec) { @@ -22,7 +22,7 @@ define {, } @vector_deinterleave_nxv4f16_n ; CHECK-NEXT: uunpklo z1.s, z2.h ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv8f16( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv8f16_nxv16f16( %vec) { @@ -33,7 +33,7 @@ define {, } @vector_deinterleave_nxv8f16_n ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv16f16( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv2f32_nxv4f32( %vec) { @@ -45,7 +45,7 @@ define {, } @vector_deinterleave_nxv2f32 ; CHECK-NEXT: uunpklo z1.d, z2.s ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv4f32( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv4f32_nxv8f32( %vec) { @@ -56,7 +56,7 @@ define {, } @vector_deinterleave_nxv4f32 ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv8f32( %vec) -ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv2f64_nxv4f64( %vec) { @@ -67,7 +67,42 @@ define {, } @vector_deinterleave_nxv2f ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv4f64( %vec) - ret {, } %retval + ret {, } %retval +} + +define {, } @vector_deinterleave_nxv2bf16_nxv4bf16( %vec) { +; CHECK-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s +; CHECK-NEXT: uzp2 z2.s, z0.s, z0.s +; CHECK-NEXT: uunpklo z0.d, z1.s +; CHECK-NEXT: uunpklo z1.d, z2.s +; CHECK-NEXT: ret + %retval = call {, } @llvm.vector.deinterleave2.nxv4bf16( %vec) + ret {, } %retval +} + +define {, } @vector_deinterleave_nxv4bf16_nxv8bf16( %vec) { +; CHECK-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z1.h, z0.h, z0.h +; CHECK-NEXT: uzp2 z2.h, z0.h, z0.h +; CHECK-NEXT: uunpklo z0.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z2.h +; CHECK-NEXT: ret + %retval = call {, } @llvm.vector.deinterleave2.nxv8bf16( %vec) + ret {, } %retval +} + +define {, } @vector_deinterleave_nxv8bf16_nxv16bf16( %vec) { +; CHECK-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z2.h, z0.h, z1.h +; CHECK-NEXT: uzp2 z1.h, z0.h, z1.h +; CHECK-NEXT: mov z0.d, z2.d +; CHECK-NEXT: ret + %retval = call {, } @llvm.vector.deinterleave2.nxv16bf16( %vec) + ret {, } %retval } ; Integers @@ -80,7 +115,7 @@ define {, } @vector_deinterleave_nxv16i8_nxv ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv32i8( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv8i16_nxv16i16( %vec) { @@ -91,7 +126,7 @@ define {, } @vector_deinterleave_nxv8i16_nxv ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv16i16( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv4i32_nxvv8i32( %vec) { @@ -102,7 +137,7 @@ define {, } @vector_deinterleave_nxv4i32_nxv ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv8i32( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv2i64_nxv4i64( %vec) { @@ -113,7 +148,7 @@ define {, } @vector_deinterleave_nxv2i64_nxv ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv4i64( %vec) - ret {, } %retval + ret {, } %retval } ; Predicated @@ -125,7 +160,7 @@ define {, } @vector_deinterleave_nxv16i1_nxv ; CHECK-NEXT: mov p0.b, p2.b ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv32i1( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv8i1_nxv16i1( %vec) { @@ -137,7 +172,7 @@ define {, } @vector_deinterleave_nxv8i1_nxv16i ; CHECK-NEXT: punpklo p1.h, p2.b ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv16i1( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv4i1_nxv8i1( %vec) { @@ -149,7 +184,7 @@ define {, } @vector_deinterleave_nxv4i1_nxv8i1 ; CHECK-NEXT: punpklo p1.h, p2.b ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv8i1( %vec) - ret {, } %retval + ret {, } %retval } define {, } @vector_deinterleave_nxv2i1_nxv4i1( %vec) { @@ -161,7 +196,7 @@ define {, } @vector_deinterleave_nxv2i1_nxv4i1 ; CHECK-NEXT: punpklo p1.h, p2.b ; CHECK-NEXT: ret %retval = call {, } @llvm.vector.deinterleave2.nxv4i1( %vec) - ret {, } %retval + ret {, } %retval } @@ -178,11 +213,11 @@ define {, } @vector_deinterleave_nxv4i64_nxv ; CHECK-NEXT: mov z1.d, z4.d ; CHECK-NEXT: mov z2.d, z6.d ; CHECK-NEXT: ret -%retval = call {, } @llvm.vector.deinterleave2.nxv8i64( %vec) -ret {, } %retval + %retval = call {, } @llvm.vector.deinterleave2.nxv8i64( %vec) + ret {, } %retval } -define {, } @vector_deinterleave_nxv8i64_nxv16i64( %vec) { +define {, } @vector_deinterleave_nxv8i64_nxv16i64( %vec) { ; CHECK-LABEL: vector_deinterleave_nxv8i64_nxv16i64: ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1 z24.d, z2.d, z3.d @@ -201,8 +236,8 @@ define {, } @vector_deinterleave_nxv8i64_nx ; CHECK-NEXT: mov z5.d, z29.d ; CHECK-NEXT: mov z6.d, z30.d ; CHECK-NEXT: ret -%retval = call {, } @llvm.vector.deinterleave2.nxv16i64( %vec) -ret {, } %retval + %retval = call {, } @llvm.vector.deinterleave2.nxv16i64( %vec) + ret {, } %retval } @@ -216,8 +251,8 @@ define {, } @vector_deinterleave_nxv8i8_nxv16i ; CHECK-NEXT: uzp1 z0.h, z2.h, z1.h ; CHECK-NEXT: uzp2 z1.h, z2.h, z1.h ; CHECK-NEXT: ret -%retval = call {, } @llvm.vector.deinterleave2.nxv16i8( %vec) -ret {, } %retval + %retval = call {, } @llvm.vector.deinterleave2.nxv16i8( %vec) + ret {, } %retval } define {, } @vector_deinterleave_nxv4i16_nxv8i16( %vec) { @@ -228,8 +263,8 @@ define {, } @vector_deinterleave_nxv4i16_nxv ; CHECK-NEXT: uzp1 z0.s, z2.s, z1.s ; CHECK-NEXT: uzp2 z1.s, z2.s, z1.s ; CHECK-NEXT: ret -%retval = call {, } @llvm.vector.deinterleave2.nxv8i16( %vec) -ret {, } %retval + %retval = call {, } @llvm.vector.deinterleave2.nxv8i16( %vec) + ret {, } %retval } define {, } @vector_deinterleave_nxv2i32_nxv4i32( %vec) { @@ -240,8 +275,8 @@ define {, } @vector_deinterleave_nxv2i32_nxv ; CHECK-NEXT: uzp1 z0.d, z2.d, z1.d ; CHECK-NEXT: uzp2 z1.d, z2.d, z1.d ; CHECK-NEXT: ret -%retval = call {,} @llvm.vector.deinterleave2.nxv4i32( %vec) -ret {, } %retval + %retval = call {,} @llvm.vector.deinterleave2.nxv4i32( %vec) + ret {, } %retval } diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll index e2c3b0abe21aa..288034422d9c0 100644 --- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll @@ -9,7 +9,7 @@ define @interleave2_nxv4f16( %vec0, @llvm.vector.interleave2.nxv4f16( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv8f16( %vec0, %vec1) { @@ -20,7 +20,7 @@ define @interleave2_nxv8f16( %vec0, @llvm.vector.interleave2.nxv8f16( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv16f16( %vec0, %vec1) { @@ -31,7 +31,7 @@ define @interleave2_nxv16f16( %vec0, @llvm.vector.interleave2.nxv16f16( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv4f32( %vec0, %vec1) { @@ -42,7 +42,7 @@ define @interleave2_nxv4f32( %vec0, @llvm.vector.interleave2.nxv4f32( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv8f32( %vec0, %vec1) { @@ -53,7 +53,7 @@ define @interleave2_nxv8f32( %vec0, @llvm.vector.interleave2.nxv8f32( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv4f64( %vec0, %vec1) { @@ -64,7 +64,40 @@ define @interleave2_nxv4f64( %vec0, < ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %retval = call @llvm.vector.interleave2.nxv4f64( %vec0, %vec1) - ret %retval + ret %retval +} + +define @interleave2_nxv4bf16( %vec0, %vec1) { +; CHECK-LABEL: interleave2_nxv4bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: zip2 z2.d, z0.d, z1.d +; CHECK-NEXT: zip1 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp1 z0.s, z0.s, z2.s +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave2.nxv4bf16( %vec0, %vec1) + ret %retval +} + +define @interleave2_nxv8bf16( %vec0, %vec1) { +; CHECK-LABEL: interleave2_nxv8bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: zip2 z2.s, z0.s, z1.s +; CHECK-NEXT: zip1 z0.s, z0.s, z1.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave2.nxv8bf16( %vec0, %vec1) + ret %retval +} + +define @interleave2_nxv16bf16( %vec0, %vec1) { +; CHECK-LABEL: interleave2_nxv16bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: zip1 z2.h, z0.h, z1.h +; CHECK-NEXT: zip2 z1.h, z0.h, z1.h +; CHECK-NEXT: mov z0.d, z2.d +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave2.nxv16bf16( %vec0, %vec1) + ret %retval } ; Integers @@ -76,8 +109,8 @@ define @interleave2_nxv32i8( %vec0, @llvm.vector.interleave2.nxv32i8( %vec0, %vec1) - ret %retval + %retval = call @llvm.vector.interleave2.nxv32i8( %vec0, %vec1) + ret %retval } define @interleave2_nxv16i16( %vec0, %vec1) { @@ -88,7 +121,7 @@ define @interleave2_nxv16i16( %vec0, @llvm.vector.interleave2.nxv16i16( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv8i32( %vec0, %vec1) { @@ -99,7 +132,7 @@ define @interleave2_nxv8i32( %vec0, @llvm.vector.interleave2.nxv8i32( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv4i64( %vec0, %vec1) { @@ -110,7 +143,7 @@ define @interleave2_nxv4i64( %vec0, @llvm.vector.interleave2.nxv4i64( %vec0, %vec1) - ret %retval + ret %retval } ; Predicated @@ -122,8 +155,8 @@ define @interleave2_nxv32i1( %vec0, @llvm.vector.interleave2.nxv32i1( %vec0, %vec1) - ret %retval + %retval = call @llvm.vector.interleave2.nxv32i1( %vec0, %vec1) + ret %retval } define @interleave2_nxv16i1( %vec0, %vec1) { @@ -134,7 +167,7 @@ define @interleave2_nxv16i1( %vec0, @llvm.vector.interleave2.nxv16i1( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv8i1( %vec0, %vec1) { @@ -145,7 +178,7 @@ define @interleave2_nxv8i1( %vec0, @llvm.vector.interleave2.nxv8i1( %vec0, %vec1) - ret %retval + ret %retval } define @interleave2_nxv4i1( %vec0, %vec1) { @@ -156,7 +189,7 @@ define @interleave2_nxv4i1( %vec0, @llvm.vector.interleave2.nxv4i1( %vec0, %vec1) - ret %retval + ret %retval } ; Split illegal type size