[MLIR][ArmNeon] Add an ArmNeon operation which maps to bfmmla (llvm#145038)

Author: momchil-velikov

mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td

@@ -222,6 +222,34 @@ def UsmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"usmmla",[1], [
     "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
 }
 
+def BfmmlaOp : ArmNeon_IntrOp<"bfmmla", [], [], 1, [
+                 Pure,
+                 AllTypesMatch<["src1", "src2"]>,
+                 AllTypesMatch<["acc", "res"]>,
+               ]> {
+  let summary = "BFloat16 matrix multiply-accumulate to single-precision";
+  let description = [{
+    BFMMLA: BFloat16 matrix multiply-accumulate to single-precision.
+
+    The operation multiplies the 2x4 BFloat16 matrix in the first source vector
+    with the 4x2 BFloat16 matrix in the second source vector, then accumulates
+    this intermediate result with the 2x2 Float32 matrix in the accumulator
+    vector, yielding the final 2x2 Float32 result.
+
+    Source:
+    https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmmlaq_f32
+  }];
+  // Supports (vector<8xbf16>, vector<8xbf16>) -> (vector<2xf32>)
+  let arguments = (ins
+    NeonVectorOfLength<4, F32>:$acc,
+    NeonVectorOfLength<8, BF16>:$src1,
+    NeonVectorOfLength<8, BF16>:$src2
+  );
+  let results = (outs NeonVectorOfLength<4, F32>:$res);
+  let assemblyFormat =
+    "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
+}
+
 class ArmNeon_2dOp<string mnemonic, list<Trait> traits = []>
     : Op</*dialect=*/ArmNeon_Dialect,
          /*opName=*/"2d." # mnemonic,

mlir/test/Dialect/ArmNeon/invalid.mlir

@@ -91,3 +91,43 @@ func.func @usmmla_invalid_dimensions(%a: vector<8xi32>,
   %0 = arm_neon.intr.usmmla %a, %b, %c : vector<32xi8> to vector<8xi32>
   return %0 : vector<8xi32>
 }
+
+// -----
+
+func.func @bfmmla_invalid_element_type_lhs_rhs(%acc: vector<4xf32>,
+                                               %lhs: vector<8xf16>,
+                                               %rhs: vector<8xf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #1 must be a vector with length 8 of bfloat16 type values, but got 'vector<8xf16>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_lhs_rhs(%acc: vector<4xf32>,
+                                            %lhs: vector<4xbf16>,
+                                            %rhs: vector<4xbf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #1 must be a vector with length 8 of bfloat16 type values, but got 'vector<4xbf16>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<4xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_element_type_acc(%acc: vector<4xi32>,
+                                           %lhs: vector<8xbf16>,
+                                           %rhs: vector<8xbf16>) -> vector<4xi32> {
+  // expected-error@+1 {{op operand #0 must be a vector with length 4 of 32-bit float values, but got 'vector<4xi32>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<4xi32>
+  return %0 : vector<4xi32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_acc(%acc: vector<8xf32>,
+                                        %lhs: vector<8xbf16>,
+                                        %rhs: vector<8xbf16>) -> vector<8xf32> {
+  // expected-error@+1 {{op operand #0 must be a vector with length 4 of 32-bit float values, but got 'vector<8xf32>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<8xf32>
+  return %0 : vector<8xf32>
+}

mlir/test/Dialect/ArmNeon/roundtrip.mlir

@@ -60,3 +60,15 @@ func.func @arm_neon_usmmla(%a: vector<16xi8>,
   %0 = arm_neon.intr.usmmla %c, %a, %b : vector<16xi8> to vector<4xi32>
   return %0 : vector<4xi32>
 }
+
+
+// -----
+
+// CHECK-LABEL: arm_neon_bfmmla
+func.func @arm_neon_bfmmla(%a: vector<8xbf16>,
+                           %b: vector<8xbf16>,
+                           %c: vector<4xf32>) -> vector<4xf32> {
+  // CHECK: arm_neon.intr.bfmmla {{.*}}: vector<8xbf16> to vector<4xf32>
+  %0 = arm_neon.intr.bfmmla %c, %a, %b : vector<8xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}

mlir/test/Target/LLVMIR/arm-neon.mlir

@@ -82,3 +82,16 @@ llvm.func @arm_neon_usmmla(%arg0: vector<16xi8>,
         -> vector<4xi32>
   llvm.return %0 : vector<4xi32>
 }
+
+// -----
+
+// CHECK-LABEL: arm_neon_bfmmla
+llvm.func @arm_neon_bfmmla(%arg0: vector<8xbf16>,
+                           %arg1: vector<8xbf16>,
+                           %arg2: vector<4xf32>) -> vector<4xf32> {
+  // CHECK: <4 x float> @llvm.aarch64.neon.bfmmla(<4 x float
+  %0 = "arm_neon.intr.bfmmla"(%arg2, %arg0, %arg1) :
+    (vector<4xf32>, vector<8xbf16>, vector<8xbf16>)
+        -> vector<4xf32>
+  llvm.return %0 : vector<4xf32>
+}