1- // RUN: mlir-opt %s -transform-interpreter -split-input-file | FileCheck %s
1+ // RUN: mlir-opt -split-input-file \
2+ // RUN: -transform-preload-library='transform-library-paths=%p/td/vectorize-with-patterns.mlir' \
3+ // RUN: -transform-interpreter=entry-point=vectorize_with_patterns %s | FileCheck %s
24
35#map0 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d2 )>
46#map1 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d1 , d2 , d3 )>
@@ -27,15 +29,6 @@ func.func @vectorize_1d_tensor_extract(%arg0: tensor<3xf32>, %arg1: tensor<4x3xi
2729// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]]] [%[[INDICES]]], %[[MASK]], %[[PASSTHRU]]
2830// CHECK: vector.transfer_write %[[GATHER]]
2931
30- module attributes {transform.with_named_sequence } {
31- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
32- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
33- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
34- %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op ) -> !transform.any_op
35- transform.yield
36- }
37- }
38-
3932// -----
4033
4134#map = affine_map <() -> ()>
@@ -59,15 +52,6 @@ func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>)
5952// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
6053// CHECK: vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
6154
62- module attributes {transform.with_named_sequence } {
63- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
64- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
65- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
66- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
67- transform.yield
68- }
69- }
70-
7155// -----
7256
7357#map = affine_map <(n ) -> (n )>
@@ -92,15 +76,6 @@ func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32
9276// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
9377// CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
9478
95- module attributes {transform.with_named_sequence } {
96- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
97- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
98- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
99- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
100- transform.yield
101- }
102- }
103-
10479// -----
10580
10681#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -130,14 +105,6 @@ func.func @vectorize_nd_tensor_extract_scalar_broadcast(%arg0: tensor<3x3xf32>,
130105// CHECK: %[[C0_2:.*]] = arith.constant 0 : index
131106// CHECK: vector.transfer_write %[[READ]], %[[ARG_1]]{{\[}}%[[C0_2]], %[[C0_2]], %[[C0_2]]] : vector<1x1x3xf32>, tensor<1x1x3xf32>
132107
133- module attributes {transform.with_named_sequence } {
134- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
135- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
136- transform.structured.vectorize %0 { vectorize_nd_extract } : !transform.any_op
137- transform.yield
138- }
139- }
140-
141108// -----
142109
143110#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -207,15 +174,6 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
207174// CHECK: %[[RES:.*]] = vector.transfer_write %[[BCAST]], %[[OUTPUT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32>
208175// CHECK: return %[[RES]] : tensor<3x1x1xf32>
209176
210- module attributes {transform.with_named_sequence } {
211- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
212- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
213- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
214- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
215- transform.yield
216- }
217- }
218-
219177 // -----
220178
221179func.func @vectorize_nd_tensor_extract_transfer_read_complex (%6: tensor <45 x80 x16 xf32 >, %arg0: index , %arg2: index , %arg1: index , %arg4: index , %extracted_slice : tensor <1 x4 xf32 >) -> tensor <1 x4 xf32 > {
@@ -259,15 +217,6 @@ func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16
259217// CHECK: return %[[VAL_21]] : tensor<1x4xf32>
260218// CHECK: }
261219
262- module attributes {transform.with_named_sequence } {
263- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
264- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
265- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
266- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
267- transform.yield
268- }
269- }
270-
271220// -----
272221
273222#map0 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d2 )>
@@ -309,15 +258,6 @@ func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>,
309258// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]]] [%[[T]]], %[[CST_1]], %[[PASSTHRU]] : tensor<3x3xf32>, vector<4x7x3x2xindex>, vector<4x7x3x2xi1>, vector<4x7x3x2xf32> into vector<4x7x3x2xf32>
310259// CHECK: vector.transfer_write %[[GATHER]], %[[ARG4]][%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true, true]} : vector<4x7x3x2xf32>, tensor<4x7x3x2xf32>
311260
312- module attributes {transform.with_named_sequence } {
313- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
314- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
315- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
316- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
317- transform.yield
318- }
319- }
320-
321261// -----
322262
323263#map = affine_map <(d0 , d1 ) -> (d0 , d1 )>
@@ -339,15 +279,6 @@ func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load(%
339279 return %1 : tensor <8 x1 xf32 >
340280}
341281
342- module attributes {transform.with_named_sequence } {
343- transform.named_sequence @__transform_main (%arg0: !transform.any_op {transform.readonly }) {
344- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg0 : (!transform.any_op ) -> !transform.any_op
345- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
346- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
347- transform.yield
348- }
349- }
350-
351282// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load
352283// CHECK-SAME: %[[ARG0:.*]]: tensor<8x128x768xf32>
353284// CHECK-SAME: %[[ARG1:.*]]: index
@@ -390,15 +321,6 @@ func.func @index_from_output_column_vector_gather_load(%src: tensor<8x128xf32>)
390321 return %res : tensor <8 x1 xf32 >
391322}
392323
393- module attributes {transform.with_named_sequence } {
394- transform.named_sequence @__transform_main (%arg2: !transform.any_op {transform.readonly }) {
395- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg2 : (!transform.any_op ) -> !transform.any_op
396- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
397- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
398- transform.yield
399- }
400- }
401-
402324// CHECK-LABEL: func.func @index_from_output_column_vector_gather_load(
403325// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
404326// CHECK: %[[C128:.*]] = arith.constant dense<128> : vector<1x8xindex>
@@ -437,15 +359,6 @@ func.func @index_from_output_column_vector_contiguous_load(%src: tensor<8x128xf3
437359 return %res : tensor <8 x1 xf32 >
438360}
439361
440- module attributes {transform.with_named_sequence } {
441- transform.named_sequence @__transform_main (%arg2: !transform.any_op {transform.readonly }) {
442- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg2 : (!transform.any_op ) -> !transform.any_op
443- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
444- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
445- transform.yield
446- }
447- }
448-
449362// CHECK-LABEL: func.func @index_from_output_column_vector_contiguous_load(
450363// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
451364// CHECK: %[[C0:.*]] = arith.constant 0 : index
@@ -497,15 +410,6 @@ func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32
497410// CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_8]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32>
498411// CHECK: return %[[VAL_14]] : tensor<5xf32>
499412
500- module attributes {transform.with_named_sequence } {
501- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
502- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
503- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
504- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
505- transform.yield
506- }
507- }
508-
509413// -----
510414
511415// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Contiguous load.
@@ -540,15 +444,6 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<8
540444// CHECK: return %[[VAL_12]] : tensor<1x4xf32>
541445// CHECK: }
542446
543- module attributes {transform.with_named_sequence } {
544- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
545- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
546- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
547- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
548- transform.yield
549- }
550- }
551-
552447// -----
553448
554449func.func @vectorize_nd_tensor_extract_with_tensor_extract (%input_1: tensor <1 x20 xi32 >, %input_2: tensor <257 x24 xf32 >, %arg0 : index , %arg1 : index , %arg2 : index , %arg3 : index ) -> tensor <1 x1 x4 xf32 > {
@@ -585,16 +480,6 @@ func.func @vectorize_nd_tensor_extract_with_tensor_extract(%input_1: tensor<1x20
585480// for address calculation also satisfy the required conditions).
586481// CHECK: vector.transfer_read %[[INPUT_2]][%{{.*}}, %{{.*}}, %{{.*}} {in_bounds = [true, true]} : tensor<257x24xf32>, vector<1x4xf32>
587482
588-
589- module attributes {transform.with_named_sequence } {
590- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
591- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
592- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
593- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
594- transform.yield
595- }
596- }
597-
598483// -----
599484
600485// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Gather load.
@@ -632,15 +517,6 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tensor<80x16
632517// CHECK: return %[[VAL_14]] : tensor<1x4xf32>
633518// CHECK: }
634519
635- module attributes {transform.with_named_sequence } {
636- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
637- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
638- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
639- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
640- transform.yield
641- }
642- }
643-
644520// -----
645521
646522// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Gather load.
@@ -674,15 +550,6 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_gather(%arg0: tensor<80x16xf32
674550// CHECK: return %[[VAL_10]] : tensor<1x4xf32>
675551// CHECK: }
676552
677- module attributes {transform.with_named_sequence } {
678- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
679- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
680- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
681- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
682- transform.yield
683- }
684- }
685-
686553// -----
687554
688555// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Contiguous load.
@@ -718,15 +585,6 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16
718585// CHECK: return %[[VAL_9]] : tensor<1x4xf32>
719586// CHECK: }
720587
721- module attributes {transform.with_named_sequence } {
722- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
723- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
724- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
725- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
726- transform.yield
727- }
728- }
729-
730588// -----
731589
732590// The vectorizer assumes it's a gather load whenever using a block argument to calculate an index.
@@ -759,15 +617,6 @@ func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1:
759617// CHECK: return %[[VAL_12]] : tensor<5xf32>
760618// CHECK: }
761619
762- module attributes {transform.with_named_sequence } {
763- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
764- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
765- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
766- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
767- transform.yield
768- }
769- }
770-
771620// -----
772621
773622#map1 = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -788,15 +637,6 @@ func.func @vectorize_0d_tensor_extract(%arg0: tensor<f32>, %arg2: tensor<1x1x3xf
788637// CHECK: %[[EXTRACT:.*]] = vector.transfer_read %[[ARG_0]][], %{{.+}} : tensor<f32>
789638// CHECK: vector.broadcast %[[EXTRACT]] : vector<f32> to vector<1x1x3xf32>
790639
791- module attributes {transform.with_named_sequence } {
792- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
793- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
794- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
795- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
796- transform.yield
797- }
798- }
799-
800640// -----
801641
802642#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -833,16 +673,6 @@ func.func @vectorize_reverse_like_tensor_extract(%arg0: tensor<1x2x3xf32>, %arg1
833673// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[T3]]], %[[MASK]], %[[PASSTHRU]]
834674// CHECK: vector.transfer_write %[[GATHER]]
835675
836- module attributes {transform.with_named_sequence } {
837- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
838- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
839- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
840- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
841- transform.yield
842- }
843- }
844-
845-
846676// -----
847677
848678func.func @vectorize_scalar_read_with_broadcast_from_column_tensor (%init: tensor <1 x1 x4 xi32 >) -> tensor <1 x1 x4 xi32 > {
@@ -874,12 +704,3 @@ func.func @vectorize_scalar_read_with_broadcast_from_column_tensor(%init: tensor
874704// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[IDX_ELT]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
875705// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<i32> to vector<1x1x4xi32>
876706// CHECK: %[[RES:.*]] = vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x4xi32>
877-
878- module attributes {transform.with_named_sequence } {
879- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
880- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
881- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
882- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
883- transform.yield
884- }
885- }
0 commit comments