[Hexagon] Custom lower v128 bf16 vectors

Adds custom load/store and masked/vector-building lowering for
v128bf16, updates HVX patterns to use the proper VecPBF16 leaf.

Patch By: Fateme Hosseini

Co-authored-by: Muntasir Mallick <[email protected]>

Author: mallick-qc

llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp

@@ -184,11 +184,19 @@ HexagonTargetLowering::initializeHVXLowering() {
       setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64bf16, Custom);
       setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v64bf16, Custom);
 
+      setOperationAction(ISD::LOAD, MVT::v128bf16, Custom);
+      setOperationAction(ISD::STORE, MVT::v128bf16, Custom);
+
       setOperationAction(ISD::MLOAD, MVT::v64bf16, Custom);
       setOperationAction(ISD::MSTORE, MVT::v64bf16, Custom);
       setOperationAction(ISD::BUILD_VECTOR, MVT::v64bf16, Custom);
       setOperationAction(ISD::CONCAT_VECTORS, MVT::v64bf16, Custom);
 
+      setOperationAction(ISD::MLOAD, MVT::v128bf16, Custom);
+      setOperationAction(ISD::MSTORE, MVT::v128bf16, Custom);
+      setOperationAction(ISD::BUILD_VECTOR, MVT::v128bf16, Custom);
+      setOperationAction(ISD::CONCAT_VECTORS, MVT::v128bf16, Custom);
+
       setOperationAction(ISD::SPLAT_VECTOR, MVT::bf16, Custom);
       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
       setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);

llvm/lib/Target/Hexagon/HexagonPatternsHVX.td

@@ -22,7 +22,7 @@ def HWI16:  PatLeaf<(VecPI16 HvxWR:$R)>;
 def HWI32:  PatLeaf<(VecPI32 HvxWR:$R)>;
 def HWF16:  PatLeaf<(VecPF16 HvxWR:$R)>;
 def HWF32:  PatLeaf<(VecPF32 HvxWR:$R)>;
-def HWBF16: PatLeaf<(VecBF16 HvxWR:$R)>;
+def HWBF16: PatLeaf<(VecPBF16 HvxWR:$R)>;
 
 def SDTVecUnaryOp:
   SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
@@ -354,11 +354,15 @@ let Predicates = [UseHVX, UseHVXFloatingPoint] in {
 
   def: Pat<(VecPF16 (concat_vectors HVF16:$Vs, HVF16:$Vt)),
            (Combinev HvxVR:$Vt, HvxVR:$Vs)>;
+  def: Pat<(VecPBF16 (concat_vectors HVBF16:$Vs, HVBF16:$Vt)),
+           (Combinev HvxVR:$Vt, HvxVR:$Vs)>;
   def: Pat<(VecPF32 (concat_vectors HVF32:$Vs, HVF32:$Vt)),
            (Combinev HvxVR:$Vt, HvxVR:$Vs)>;
 
   def: Pat<(HexagonVINSERTW0 HVF16:$Vu, I32:$Rt),
            (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>;
+  def: Pat<(HexagonVINSERTW0 HVBF16:$Vu, I32:$Rt),
+           (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>;
   def: Pat<(HexagonVINSERTW0 HVF32:$Vu, I32:$Rt),
            (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>;
 }

llvm/test/CodeGen/Hexagon/bfloat_vec.ll

@@ -95,20 +95,11 @@ entry:
 
 define dso_local void @copy1d(ptr noundef readonly captures(none) %X, ptr noundef writeonly captures(none) %Y) local_unnamed_addr #0 {
 ; CHECK-LABEL: copy1d:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    {
-; CHECK-NEXT:     v0 = vmemu(r0+#1)
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     v1 = vmemu(r0+#0)
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     vmemu(r1+#1) = v0
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     jumpr r31
-; CHECK-NEXT:     vmemu(r1+#0) = v1
-; CHECK-NEXT:    }
+; CHECK: v[[X_HI:[0-9]+]] = vmemu(r0+#1)
+; CHECK: v[[X_LO:[0-9]+]] = vmemu(r0+#0)
+; CHECK: vmemu(r1+#1) = v[[X_HI]]
+; CHECK: jumpr [[RET:r[0-9]+]]
+; CHECK: vmemu(r1+#0) = v[[X_LO]]
 entry:
   %0 = load <128 x half>, ptr %X, align 2
   store <128 x half> %0, ptr %Y, align 2