Skip to content

Commit 1e85f2a

Browse files
LukacmaLukacma
committed
[AArch64] Model ZA array using inaccessible memory
1 parent 4508d6a commit 1e85f2a

File tree

4 files changed

+483
-257
lines changed

4 files changed

+483
-257
lines changed

‎llvm/include/llvm/IR/IntrinsicsAArch64.td‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2940,7 +2940,7 @@ def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
29402940
let TargetPrefix = "aarch64" in {
29412941
class SME_Load_Store_Intrinsic<LLVMType pred_ty>
29422942
: DefaultAttrsIntrinsic<[],
2943-
[pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
2943+
[pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<2>>]>;
29442944

29452945
// Loads
29462946
def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
@@ -2968,18 +2968,18 @@ let TargetPrefix = "aarch64" in {
29682968

29692969
// Spill + fill
29702970
class SME_LDR_STR_ZA_Intrinsic
2971-
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty]>;
2971+
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOrArgMemOnly]>;
29722972
def int_aarch64_sme_ldr : SME_LDR_STR_ZA_Intrinsic;
29732973
def int_aarch64_sme_str : SME_LDR_STR_ZA_Intrinsic;
29742974

29752975
class SME_TileToVector_Intrinsic
29762976
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
29772977
[LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2978-
llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
2978+
llvm_i32_ty, llvm_i32_ty], [IntrReadMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<2>>]>;
29792979
class SME_VectorToTile_Intrinsic
29802980
: DefaultAttrsIntrinsic<[],
29812981
[llvm_i32_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2982-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
2982+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
29832983

29842984
def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic;
29852985
def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic;
@@ -2994,13 +2994,13 @@ let TargetPrefix = "aarch64" in {
29942994
class SME_MOVAZ_TileToVector_X2_Intrinsic
29952995
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
29962996
[llvm_i32_ty, llvm_i32_ty],
2997-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
2997+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
29982998

29992999
class SME_MOVAZ_TileToVector_X4_Intrinsic
30003000
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
30013001
LLVMMatchType<0>,LLVMMatchType<0>],
30023002
[llvm_i32_ty, llvm_i32_ty],
3003-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
3003+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30043004

30053005
def int_aarch64_sme_readz_horiz_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic;
30063006
def int_aarch64_sme_readz_vert_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic;
@@ -3011,7 +3011,7 @@ let TargetPrefix = "aarch64" in {
30113011
class SME_MOVAZ_TileToVector_Intrinsic
30123012
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
30133013
[llvm_i32_ty, llvm_i32_ty],
3014-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
3014+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30153015

30163016
def int_aarch64_sme_readz_horiz : SME_MOVAZ_TileToVector_Intrinsic;
30173017
def int_aarch64_sme_readz_vert : SME_MOVAZ_TileToVector_Intrinsic;
@@ -3022,12 +3022,12 @@ let TargetPrefix = "aarch64" in {
30223022
def int_aarch64_sme_readz_x2
30233023
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
30243024
[llvm_i32_ty],
3025-
[IntrNoMem, IntrHasSideEffects]>;
3025+
[IntrInaccessibleMemOnly]>;
30263026

30273027
def int_aarch64_sme_readz_x4
30283028
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
30293029
[llvm_i32_ty],
3030-
[IntrNoMem, IntrHasSideEffects]>;
3030+
[IntrInaccessibleMemOnly]>;
30313031

30323032
def int_aarch64_sme_write_lane_zt
30333033
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_anyvector_ty, llvm_i32_ty],
@@ -3038,7 +3038,7 @@ let TargetPrefix = "aarch64" in {
30383038
[ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects]>;
30393039

30403040

3041-
def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
3041+
def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30423042
def int_aarch64_sme_in_streaming_mode : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrNoMem]>, ClangBuiltin<"__builtin_arm_in_streaming_mode">;
30433043

30443044
class SME_OuterProduct_Intrinsic
@@ -3047,7 +3047,7 @@ let TargetPrefix = "aarch64" in {
30473047
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
30483048
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
30493049
LLVMMatchType<0>,
3050-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
3050+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30513051

30523052
def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic;
30533053
def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic;
@@ -3069,7 +3069,7 @@ let TargetPrefix = "aarch64" in {
30693069
[llvm_i32_ty,
30703070
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
30713071
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
3072-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
3072+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30733073

30743074
def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic;
30753075
def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic;
@@ -3189,56 +3189,56 @@ let TargetPrefix = "aarch64" in {
31893189
: DefaultAttrsIntrinsic<[],
31903190
[llvm_i32_ty,
31913191
llvm_anyvector_ty, LLVMMatchType<0>],
3192-
[]>;
3192+
[IntrInaccessibleMemOnly]>;
31933193

31943194
class SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic
31953195
: DefaultAttrsIntrinsic<[],
31963196
[llvm_i32_ty,
31973197
llvm_anyvector_ty, LLVMMatchType<0>,
31983198
LLVMMatchType<0>],
3199-
[]>;
3199+
[IntrInaccessibleMemOnly]>;
32003200

32013201
class SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic
32023202
: DefaultAttrsIntrinsic<[],
32033203
[llvm_i32_ty,
32043204
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32053205
LLVMMatchType<0>],
3206-
[]>;
3206+
[IntrInaccessibleMemOnly]>;
32073207

32083208
class SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic
32093209
: DefaultAttrsIntrinsic<[],
32103210
[llvm_i32_ty,
32113211
llvm_anyvector_ty, LLVMMatchType<0>,
32123212
LLVMMatchType<0>, LLVMMatchType<0>],
3213-
[]>;
3213+
[IntrInaccessibleMemOnly]>;
32143214

32153215
class SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic
32163216
: DefaultAttrsIntrinsic<[],
32173217
[llvm_i32_ty,
32183218
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32193219
LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
3220-
[]>;
3220+
[IntrInaccessibleMemOnly]>;
32213221

32223222
class SME2_Matrix_ArrayVector_Single_Index_Intrinsic
32233223
: DefaultAttrsIntrinsic<[],
32243224
[llvm_i32_ty,
32253225
llvm_anyvector_ty,
32263226
LLVMMatchType<0>, llvm_i32_ty],
3227-
[ImmArg<ArgIndex<3>>]>;
3227+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
32283228

32293229
class SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic
32303230
: DefaultAttrsIntrinsic<[],
32313231
[llvm_i32_ty,
32323232
llvm_anyvector_ty, LLVMMatchType<0>,
32333233
LLVMMatchType<0>, llvm_i32_ty],
3234-
[ImmArg<ArgIndex<4>>]>;
3234+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<4>>]>;
32353235

32363236
class SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic
32373237
: DefaultAttrsIntrinsic<[],
32383238
[llvm_i32_ty,
32393239
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32403240
LLVMMatchType<0>, llvm_i32_ty],
3241-
[ImmArg<ArgIndex<6>>]>;
3241+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<6>>]>;
32423242

32433243
class SME2_VG2_Multi_Imm_Intrinsic
32443244
: DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
@@ -3257,14 +3257,14 @@ let TargetPrefix = "aarch64" in {
32573257
: DefaultAttrsIntrinsic<[],
32583258
[llvm_i32_ty,
32593259
llvm_anyvector_ty, LLVMMatchType<0>],
3260-
[]>;
3260+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
32613261

32623262
class SME2_ZA_Write_VG4_Intrinsic
32633263
: DefaultAttrsIntrinsic<[],
32643264
[llvm_i32_ty,
32653265
llvm_anyvector_ty, LLVMMatchType<0>,
32663266
LLVMMatchType<0>, LLVMMatchType<0>],
3267-
[]>;
3267+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
32683268

32693269
class SME2_VG2_Multi_Single_Intrinsic
32703270
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
@@ -3353,50 +3353,50 @@ let TargetPrefix = "aarch64" in {
33533353
class SME2_ZA_ArrayVector_Read_VG2_Intrinsic
33543354
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
33553355
[llvm_i32_ty],
3356-
[]>;
3356+
[IntrReadMem, IntrInaccessibleMemOnly]>;
33573357

33583358
class SME2_ZA_ArrayVector_Read_VG4_Intrinsic
33593359
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
33603360
LLVMMatchType<0>, LLVMMatchType<0>],
33613361
[llvm_i32_ty],
3362-
[]>;
3362+
[IntrReadMem, IntrInaccessibleMemOnly]>;
33633363

33643364
class SME2_Matrix_TileVector_Read_VG2_Intrinsic
33653365
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
33663366
[llvm_i32_ty, llvm_i32_ty],
3367-
[]>;
3367+
[IntrReadMem, IntrInaccessibleMemOnly]>;
33683368

33693369
class SME2_Matrix_TileVector_Read_VG4_Intrinsic
33703370
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
33713371
LLVMMatchType<0>, LLVMMatchType<0>],
33723372
[llvm_i32_ty, llvm_i32_ty],
3373-
[]>;
3373+
[IntrReadMem, IntrInaccessibleMemOnly]>;
33743374

33753375
class SME2_ZA_ArrayVector_Write_VG2_Intrinsic
33763376
: DefaultAttrsIntrinsic<[],
33773377
[llvm_i32_ty,
33783378
llvm_anyvector_ty, LLVMMatchType<0>],
3379-
[]>;
3379+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
33803380

33813381
class SME2_ZA_ArrayVector_Write_VG4_Intrinsic
33823382
: DefaultAttrsIntrinsic<[],
33833383
[llvm_i32_ty,
33843384
llvm_anyvector_ty, LLVMMatchType<0>,
33853385
LLVMMatchType<0>, LLVMMatchType<0>],
3386-
[]>;
3386+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
33873387

33883388
class SME2_Matrix_TileVector_Write_VG2_Intrinsic
33893389
: DefaultAttrsIntrinsic<[],
33903390
[llvm_i32_ty, llvm_i32_ty,
33913391
llvm_anyvector_ty, LLVMMatchType<0>],
3392-
[ImmArg<ArgIndex<0>>]>;
3392+
[IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
33933393

33943394
class SME2_Matrix_TileVector_Write_VG4_Intrinsic
33953395
: DefaultAttrsIntrinsic<[],
33963396
[llvm_i32_ty, llvm_i32_ty,
33973397
llvm_anyvector_ty, LLVMMatchType<0>,
33983398
LLVMMatchType<0>, LLVMMatchType<0>],
3399-
[ImmArg<ArgIndex<0>>]>;
3399+
[IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
34003400

34013401
class SME2_VG2_Multi_Single_Single_Intrinsic
34023402
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
@@ -3562,7 +3562,7 @@ let TargetPrefix = "aarch64" in {
35623562
// Multi-vector zeroing
35633563

35643564
foreach vg = ["vg1x2", "vg1x4", "vg2x1", "vg2x2", "vg2x4", "vg4x1", "vg4x2", "vg4x4"] in {
3565-
def int_aarch64_sme_zero_za64_ # vg : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects]>;
3565+
def int_aarch64_sme_zero_za64_ # vg : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;
35663566
}
35673567

35683568
// Multi-vector signed saturating doubling multiply high
@@ -4002,57 +4002,57 @@ let TargetPrefix = "aarch64" in {
40024002
[llvm_i32_ty,
40034003
llvm_nxv16i1_ty, llvm_nxv16i1_ty,
40044004
llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4005-
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4005+
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly]>;
40064006

40074007
class SME_FP8_ZA_LANE_VGx1_Intrinsic
40084008
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40094009
llvm_nxv16i8_ty,
40104010
llvm_nxv16i8_ty,
40114011
llvm_i32_ty],
4012-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<3>>]>;
4012+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
40134013

40144014
class SME_FP8_ZA_LANE_VGx2_Intrinsic
40154015
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40164016
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40174017
llvm_nxv16i8_ty,
40184018
llvm_i32_ty],
4019-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<4>>]>;
4019+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<4>>]>;
40204020

40214021
class SME_FP8_ZA_LANE_VGx4_Intrinsic
40224022
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40234023
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40244024
llvm_nxv16i8_ty,
40254025
llvm_i32_ty],
4026-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<6>>]>;
4026+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<6>>]>;
40274027
class SME_FP8_ZA_SINGLE_VGx1_Intrinsic
40284028
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40294029
llvm_nxv16i8_ty,
40304030
llvm_nxv16i8_ty],
4031-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4031+
[IntrInaccessibleMemOnly]>;
40324032

40334033
class SME_FP8_ZA_SINGLE_VGx2_Intrinsic
40344034
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40354035
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40364036
llvm_nxv16i8_ty],
4037-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4037+
[IntrInaccessibleMemOnly]>;
40384038

40394039
class SME_FP8_ZA_SINGLE_VGx4_Intrinsic
40404040
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40414041
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40424042
llvm_nxv16i8_ty],
4043-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4043+
[IntrInaccessibleMemOnly]>;
40444044

40454045
class SME_FP8_ZA_MULTI_VGx2_Intrinsic
40464046
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40474047
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40484048
llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4049-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4049+
[IntrInaccessibleMemOnly]>;
40504050

40514051
class SME_FP8_ZA_MULTI_VGx4_Intrinsic
40524052
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40534053
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40544054
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4055-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4055+
[IntrInaccessibleMemOnly]>;
40564056
//
40574057
// CVT from FP8 to half-precision/BFloat16 multi-vector
40584058
//

0 commit comments

Comments
 (0)