[X86][MC][AsmParser] Reject H-byte regs with VEX/EVEX-encoded 8-bit RR (NDD) (llvm#160039)

### Summary
This PR resolves llvm#158585.

Author: woruyu

llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp

@@ -4018,9 +4018,14 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
       return Error(Ops[0]->getStartLoc(), "all tmm registers must be distinct");
   }
 
-  // Check that we aren't mixing AH/BH/CH/DH with REX prefix. We only need to
-  // check this with the legacy encoding, VEX/EVEX/XOP don't use REX.
-  if ((TSFlags & X86II::EncodingMask) == 0) {
+  // High 8-bit regs (AH/BH/CH/DH) are incompatible with encodings that imply
+  // extended prefixes:
+  //  * Legacy path that would emit a REX (e.g. uses r8..r15 or sil/dil/bpl/spl)
+  //  * EVEX
+  //  * REX2
+  // VEX/XOP don't use REX; they are excluded from the legacy check.
+  const unsigned Enc = TSFlags & X86II::EncodingMask;
+  if (Enc != X86II::VEX && Enc != X86II::XOP) {
     MCRegister HReg;
     bool UsesRex = TSFlags & X86II::REX_W;
     unsigned NumOps = Inst.getNumOperands();
@@ -4036,11 +4041,13 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
         UsesRex = true;
     }
 
-    if (UsesRex && HReg) {
+    if (HReg &&
+        (Enc == X86II::EVEX || ForcedOpcodePrefix == OpcodePrefix_REX2 ||
+         ForcedOpcodePrefix == OpcodePrefix_REX || UsesRex)) {
       StringRef RegName = X86IntelInstPrinter::getRegisterName(HReg);
       return Error(Ops[0]->getStartLoc(),
-                   "can't encode '" + RegName + "' in an instruction requiring "
-                   "REX prefix");
+                   "can't encode '" + RegName.str() +
+                       "' in an instruction requiring EVEX/REX2/REX prefix");
     }
   }
 

llvm/test/MC/X86/encoder-fail.s

@@ -1,16 +1,38 @@
 // RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding -x86-asm-syntax=intel %s 2>&1 | FileCheck %s --check-prefix=CHECK-INTEL
 
-// CHECK: error: can't encode 'dh' in an instruction requiring REX prefix
+// CHECK: error: can't encode 'dh' in an instruction requiring EVEX/REX2/REX prefix
 movzx %dh, %rsi
 
-// CHECK: error: can't encode 'ah' in an instruction requiring REX prefix
+// CHECK: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
 movzx %ah, %r8d
 
-// CHECK: error: can't encode 'bh' in an instruction requiring REX prefix
+// CHECK: error: can't encode 'bh' in an instruction requiring EVEX/REX2/REX prefix
 add %bh, %sil
 
-// CHECK: error: can't encode 'ch' in an instruction requiring REX prefix
+// CHECK: error: can't encode 'ch' in an instruction requiring EVEX/REX2/REX prefix
 mov %ch, (%r8)
 
-// CHECK: error: can't encode 'dh' in an instruction requiring REX prefix
+// CHECK: error: can't encode 'dh' in an instruction requiring EVEX/REX2/REX prefix
 mov %dh, (%rax,%r8)
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+add ah, ah, ah
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+and ah, byte ptr [-13426159], ah
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+ccmpa {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+ccmpae {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+sar ah, byte ptr [-13426159]
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+{rex2} add ah, al
+
+// CHECK-INTEL: error: can't encode 'ah' in an instruction requiring EVEX/REX2/REX prefix
+{rex} add ah, al