diff --git a/clang/test/Driver/aarch64-vfat.c b/clang/test/Driver/aarch64-vfat.c index fa268641a86e0..63096336ceb76 100644 --- a/clang/test/Driver/aarch64-vfat.c +++ b/clang/test/Driver/aarch64-vfat.c @@ -13,3 +13,7 @@ // RUN: %clang -target aarch64 -march=armv9.7a+btie -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-BTIE %s // RUN: %clang -target aarch64 -march=armv9.7-a+btie -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-BTIE %s // VFAT-BTIE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.7a"{{.*}} "-target-feature" "+btie" + +// RUN: %clang -target aarch64 -march=armv9.7a+mops-go -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-MOPS-GO %s +// RUN: %clang -target aarch64 -march=armv9.7-a+mops-go -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-MOPS-GO %s +// VFAT-MOPS-GO: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.7a"{{.*}} "-target-feature" "+mops-go" diff --git a/clang/test/Driver/print-supported-extensions-aarch64.c b/clang/test/Driver/print-supported-extensions-aarch64.c index d0c86c7065281..93373f41ad2cf 100644 --- a/clang/test/Driver/print-supported-extensions-aarch64.c +++ b/clang/test/Driver/print-supported-extensions-aarch64.c @@ -50,6 +50,7 @@ // CHECK-NEXT: lsui FEAT_LSUI Enable Armv9.6-A unprivileged load/store instructions // CHECK-NEXT: lut FEAT_LUT Enable Lookup Table instructions // CHECK-NEXT: mops FEAT_MOPS Enable Armv8.8-A memcpy and memset acceleration instructions +// CHECK-NEXT: mops-go FEAT_MOPS_GO Enable memset acceleration granule only // CHECK-NEXT: mpamv2 FEAT_MPAMv2 Enable Armv9.7-A MPAMv2 Lookaside Buffer Invalidate instructions // CHECK-NEXT: memtag FEAT_MTE, FEAT_MTE2 Enable Memory Tagging Extension // CHECK-NEXT: mtetc FEAT_MTETC Enable Virtual Memory Tagging Extension diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td index c4f6e000dff66..51e602ad7e0f2 100644 --- a/llvm/lib/Target/AArch64/AArch64Features.td +++ b/llvm/lib/Target/AArch64/AArch64Features.td @@ -638,6 +638,9 @@ def FeatureS1POE2: ExtensionWithMArch<"poe2", "POE2", "FEAT_S1POE2", def FeatureTEV: ExtensionWithMArch<"tev", "TEV", "FEAT_TEV", "Enable TIndex Exception-like Vector instructions">; +def FeatureMOPS_GO: ExtensionWithMArch<"mops-go", "MOPS_GO", "FEAT_MOPS_GO", + "Enable memset acceleration granule only">; + //===----------------------------------------------------------------------===// // Other Features //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index a39bb31e61196..a1431b24f992e 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -12604,7 +12604,7 @@ class MOPSMemoryMove opcode, bits<2> op1, bits<2> op2, string asm> : MOPSMemoryCopyMoveBase<1, opcode, op1, op2, asm>; class MOPSMemorySetBase opcode, bit op1, bit op2, - string asm> + bit op3, string asm> : I<(outs GPR64common:$Rd_wb, GPR64:$Rn_wb), (ins GPR64common:$Rd, GPR64:$Rn, GPR64:$Rm), asm, "\t[$Rd]!, $Rn!, $Rm", @@ -12620,7 +12620,8 @@ class MOPSMemorySetBase opcode, bit op1, bit op2, let Inst{15-14} = opcode; let Inst{13} = op2; let Inst{12} = op1; - let Inst{11-10} = 0b01; + let Inst{11} = 0b0; + let Inst{10} = op3; let Inst{9-5} = Rn; let Inst{4-0} = Rd; @@ -12629,11 +12630,18 @@ class MOPSMemorySetBase opcode, bit op1, bit op2, let mayStore = 1; } -class MOPSMemorySet opcode, bit op1, bit op2, string asm> - : MOPSMemorySetBase<0, opcode, op1, op2, asm>; +class MOPSMemorySet opcode, bit op1, bit op2, bit op3, string asm> + : MOPSMemorySetBase<0, opcode, op1, op2, op3, asm>; + +class MOPSMemorySetTagging opcode, bit op1, bit op2, bit op3, string asm> + : MOPSMemorySetBase<1, opcode, op1, op2, op3, asm>; -class MOPSMemorySetTagging opcode, bit op1, bit op2, string asm> - : MOPSMemorySetBase<1, opcode, op1, op2, asm>; +class MOPSGoMemorySetTagging opcode, bit op1, bit op2, bit op3, string asm> + : MOPSMemorySetBase<1, opcode, op1, op2, op3, asm> { + // No `Rm` operand is required, as all bits are set to 1 + let AsmString = !strconcat(asm, "\t[$Rd]!, $Rn!"); + let Inst{20-16} = 0b11111; +} multiclass MOPSMemoryCopyInsns opcode, string asm> { def "" : MOPSMemoryCopy; @@ -12674,17 +12682,27 @@ multiclass MOPSMemoryMoveInsns opcode, string asm> { } multiclass MOPSMemorySetInsns opcode, string asm> { - def "" : MOPSMemorySet; - def T : MOPSMemorySet; - def N : MOPSMemorySet; - def TN : MOPSMemorySet; + def "" : MOPSMemorySet; + def T : MOPSMemorySet; + def N : MOPSMemorySet; + def TN : MOPSMemorySet; } multiclass MOPSMemorySetTaggingInsns opcode, string asm> { - def "" : MOPSMemorySetTagging; - def T : MOPSMemorySetTagging; - def N : MOPSMemorySetTagging; - def TN : MOPSMemorySetTagging; + def "" : MOPSMemorySetTagging; + def T : MOPSMemorySetTagging; + def N : MOPSMemorySetTagging; + def TN : MOPSMemorySetTagging; +} + +//---------------------------------------------------------------------------- +// MOPS Granule Only - FEAT_MOPS_GO +//---------------------------------------------------------------------------- +multiclass MOPSGoMemorySetTaggingInsns opcode, string asm> { + def "" : MOPSGoMemorySetTagging; + def T : MOPSGoMemorySetTagging; + def N : MOPSGoMemorySetTagging; + def TN : MOPSGoMemorySetTagging; } //---------------------------------------------------------------------------- diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index cbba169f4db92..f706af605b91c 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -411,6 +411,8 @@ def HasS1POE2 : Predicate<"Subtarget->hasS1POE2()">, AssemblerPredicateWithAll<(all_of FeatureS1POE2), "poe2">; def HasTEV : Predicate<"Subtarget->hasTEV()">, AssemblerPredicateWithAll<(all_of FeatureTEV), "tev">; +def HasMOPS_GO : Predicate<"Subtarget->hasMOPS_GO()">, + AssemblerPredicateWithAll<(all_of FeatureMOPS_GO), "mops-go">; def IsLE : Predicate<"Subtarget->isLittleEndian()">; def IsBE : Predicate<"!Subtarget->isLittleEndian()">; def IsWindows : Predicate<"Subtarget->isTargetWindows()">; @@ -10867,6 +10869,15 @@ let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, maySt [], "$Rd = $Rd_wb,$Rn = $Rn_wb">, Sched<[]>; } +//----------------------------------------------------------------------------- +// MOPS Granule Only Protection (FEAT_MOPS_GO) + +let Predicates = [HasMOPS_GO, HasMTE] in { + defm SETGOP : MOPSGoMemorySetTaggingInsns<0b00, "setgop">; + defm SETGOM : MOPSGoMemorySetTaggingInsns<0b01, "setgom">; + defm SETGOE : MOPSGoMemorySetTaggingInsns<0b10, "setgoe">; +} + //----------------------------------------------------------------------------- // v8.3 Pointer Authentication late patterns diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index c19d33e51ccb2..e77aba69096c2 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -3957,6 +3957,7 @@ static const struct Extension { {"poe2", {AArch64::FeatureS1POE2}}, {"tev", {AArch64::FeatureTEV}}, {"btie", {AArch64::FeatureBTIE}}, + {"mops-go", {AArch64::FeatureMOPS_GO}}, }; static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) { @@ -6050,7 +6051,19 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc, case AArch64::MOPSSETGE: case AArch64::MOPSSETGET: case AArch64::MOPSSETGEN: - case AArch64::MOPSSETGETN: { + case AArch64::MOPSSETGETN: + case AArch64::SETGOP: + case AArch64::SETGOPT: + case AArch64::SETGOPN: + case AArch64::SETGOPTN: + case AArch64::SETGOM: + case AArch64::SETGOMT: + case AArch64::SETGOMN: + case AArch64::SETGOMTN: + case AArch64::SETGOE: + case AArch64::SETGOET: + case AArch64::SETGOEN: + case AArch64::SETGOETN: { MCRegister Xd_wb = Inst.getOperand(0).getReg(); MCRegister Xn_wb = Inst.getOperand(1).getReg(); MCRegister Xd = Inst.getOperand(2).getReg(); diff --git a/llvm/test/MC/AArch64/arm-mops-go-diagnostics.s b/llvm/test/MC/AArch64/arm-mops-go-diagnostics.s new file mode 100644 index 0000000000000..c22331b9f18e4 --- /dev/null +++ b/llvm/test/MC/AArch64/arm-mops-go-diagnostics.s @@ -0,0 +1,36 @@ +// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mops-go,+mte < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +// Operands must be different from each other + +// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same +// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same +// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same +setgop [x0]!, x0! +setgom [x0]!, x0! +setgoe [x0]!, x0! + +// SP cannot be used as argument at any position + +// CHECK-ERROR: error: invalid operand for instruction +// CHECK-ERROR: error: invalid operand for instruction +setgop [sp]!, x1! +setgop [x0]!, sp! + +// CHECK-ERROR: error: invalid operand for instruction +// CHECK-ERROR: error: invalid operand for instruction +setgom [sp]!, x1! +setgom [x0]!, sp! + +// CHECK-ERROR: error: invalid operand for instruction +// CHECK-ERROR: error: invalid operand for instruction +setgoe [sp]!, x1! +setgoe [x0]!, sp! + +// CHECK-ERROR: error: invalid operand for instruction +setgop [xzr]!, x1! + +// CHECK-ERROR: error: invalid operand for instruction +setgom [xzr]!, x1! + +// CHECK-ERROR: error: invalid operand for instruction +setgoe [xzr]!, x1! diff --git a/llvm/test/MC/AArch64/arm-mops-go.s b/llvm/test/MC/AArch64/arm-mops-go.s new file mode 100644 index 0000000000000..7520551f59eb8 --- /dev/null +++ b/llvm/test/MC/AArch64/arm-mops-go.s @@ -0,0 +1,89 @@ +// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops-go,+mte < %s \ +// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-ERROR +// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+all < %s \ +// RUN: | llvm-objdump -d --mattr=+mops-go,+mte --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST +// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+all < %s \ +// RUN: | llvm-objdump -d --mattr=-mops-go,-mte --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN +// Disassemble encoding and check the re-encoding (-show-encoding) matches. +// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops-go,+mte < %s \ +// RUN: | sed '/.text/d' | sed 's/.*encoding: //g' \ +// RUN: | llvm-mc -triple=aarch64 -mattr=+mops-go,+mte -disassemble -show-encoding \ +// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST + +//------------------------------------------------------------------------------ +// FEAT_MOPS_GO Extension instructions +//------------------------------------------------------------------------------ + +setgop [x3]!, x2! +// CHECK-INST: setgop [x3]!, x2! +// CHECK-ENCODING: [0x43,0x00,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf0043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgom [x3]!, x2! +// CHECK-INST: setgom [x3]!, x2! +// CHECK-ENCODING: [0x43,0x40,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf4043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgoe [x3]!, x2! +// CHECK-INST: setgoe [x3]!, x2! +// CHECK-ENCODING: [0x43,0x80,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf8043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgopn [x3]!, x2! +// CHECK-INST: setgopn [x3]!, x2! +// CHECK-ENCODING: [0x43,0x20,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf2043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgomn [x3]!, x2! +// CHECK-INST: setgomn [x3]!, x2! +// CHECK-ENCODING: [0x43,0x60,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf6043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgoen [x3]!, x2! +// CHECK-INST: setgoen [x3]!, x2! +// CHECK-ENCODING: [0x43,0xa0,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddfa043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgopt [x3]!, x2! +// CHECK-INST: setgopt [x3]!, x2! +// CHECK-ENCODING: [0x43,0x10,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf1043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgomt [x3]!, x2! +// CHECK-INST: setgomt [x3]!, x2! +// CHECK-ENCODING: [0x43,0x50,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf5043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgoet [x3]!, x2! +// CHECK-INST: setgoet [x3]!, x2! +// CHECK-ENCODING: [0x43,0x90,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf9043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgoptn [x3]!, x2! +// CHECK-INST: setgoptn [x3]!, x2! +// CHECK-ENCODING: [0x43,0x30,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf3043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgomtn [x3]!, x2! +// CHECK-INST: setgomtn [x3]!, x2! +// CHECK-ENCODING: [0x43,0x70,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddf7043 +// CHECK-ERROR: instruction requires: mops-go mte + +setgoetn [x3]!, x2! +// CHECK-INST: setgoetn [x3]!, x2! +// CHECK-ENCODING: [0x43,0xb0,0xdf,0x1d] +// CHECK-UNKNOWN: 1ddfb043 +// CHECK-ERROR: instruction requires: mops-go mte diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index d119b062c8e1a..5b38f006b00b0 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -1454,6 +1454,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) { AArch64::AEK_F16MM, AArch64::AEK_F16F32DOT, AArch64::AEK_F16F32MM, AArch64::AEK_POE2, AArch64::AEK_TEV, AArch64::AEK_BTIE, + AArch64::AEK_MOPS_GO, }; std::vector Features; @@ -1580,6 +1581,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) { EXPECT_TRUE(llvm::is_contained(Features, "+poe2")); EXPECT_TRUE(llvm::is_contained(Features, "+tev")); EXPECT_TRUE(llvm::is_contained(Features, "+btie")); + EXPECT_TRUE(llvm::is_contained(Features, "+mops-go")); // Assuming we listed every extension above, this should produce the same // result. @@ -1761,6 +1763,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) { {"poe2", "nopoe2", "+poe2", "-poe2"}, {"tev", "notev", "+tev", "-tev"}, {"btie", "nobtie", "+btie", "-btie"}, + {"mops-go", "nomops-go", "+mops-go", "-mops-go"}, }; for (unsigned i = 0; i < std::size(ArchExt); i++) {