x86: simplify REX prefix parsing (#2885)

The previous implementation for handling REX prefixes was quite
complex. It required looking ahead to determine if a REX prefix
should be ignored.

This patch simplifies the implementation by instead resetting
insn->rexPrefix when a non-REX prefix is encountered.

Author: jxors

arch/X86/X86DisassemblerDecoder.c

@@ -429,48 +429,6 @@ static int readPrefixes(struct InternalInstruction *insn)
 	uint8_t nextByte;
 
 	while (isPrefix) {
-		if (insn->mode == MODE_64BIT) {
-			// eliminate consecutive redundant REX bytes in front
-			if (consumeByte(insn, &byte))
-				return -1;
-
-			if ((byte & 0xf0) == 0x40) {
-				while (true) {
-					if (lookAtByte(
-						    insn,
-						    &byte)) // out of input code
-						return -1;
-					if ((byte & 0xf0) == 0x40) {
-						// another REX prefix, but we only remember the last one
-						if (consumeByte(insn, &byte))
-							return -1;
-					} else
-						break;
-				}
-
-				// recover the last REX byte if next byte is not a legacy prefix
-				switch (byte) {
-				case 0xf2: /* REPNE/REPNZ */
-				case 0xf3: /* REP or REPE/REPZ */
-				case 0xf0: /* LOCK */
-				case 0x2e: /* CS segment override -OR- Branch not taken */
-				case 0x36: /* SS segment override -OR- Branch taken */
-				case 0x3e: /* DS segment override */
-				case 0x26: /* ES segment override */
-				case 0x64: /* FS segment override */
-				case 0x65: /* GS segment override */
-				case 0x66: /* Operand-size override */
-				case 0x67: /* Address-size override */
-					break;
-				default: /* Not a prefix byte */
-					unconsumeByte(insn);
-					break;
-				}
-			} else {
-				unconsumeByte(insn);
-			}
-		}
-
 		/* If we fail reading prefixes, just stop here and let the opcode reader deal with it */
 		if (consumeByte(insn, &byte))
 			return -1;
@@ -503,6 +461,7 @@ static int readPrefixes(struct InternalInstruction *insn)
 			// only accept the last prefix
 			setGroup0Prefix(insn, byte);
 			insn->prefix0 = byte;
+			insn->rexPrefix = 0;
 			break;
 
 		case 0x2e: /* CS segment override -OR- Branch not taken */
@@ -534,19 +493,28 @@ static int readPrefixes(struct InternalInstruction *insn)
 				// debug("Unhandled override");
 				return -1;
 			}
+			insn->rexPrefix = 0;
 			break;
 
 		case 0x66: /* Operand-size override */
 			insn->hasOpSize = true;
 			insn->prefix2 = byte;
+			insn->rexPrefix = 0;
 			break;
 
 		case 0x67: /* Address-size override */
 			insn->hasAdSize = true;
 			insn->prefix3 = byte;
+			insn->rexPrefix = 0;
 			break;
-		default: /* Not a prefix byte */
-			isPrefix = false;
+		default:
+			if (isREX(insn, byte)) {
+				/* REX prefix byte */
+				insn->rexPrefix = byte;
+			} else {
+				/* Not a prefix byte */
+				isPrefix = false;
+			}
 			break;
 		}
 	}
@@ -736,12 +704,6 @@ static int readPrefixes(struct InternalInstruction *insn)
 			// 		insn->vectorExtensionPrefix[0], insn->vectorExtensionPrefix[1],
 			// 		insn->vectorExtensionPrefix[2]);
 		}
-	} else if (isREX(insn, byte)) {
-		if (lookAtByte(insn, &nextByte))
-			return -1;
-
-		insn->rexPrefix = byte;
-		// dbgprintf(insn, "Found REX prefix 0x%hhx", byte);
 	} else
 		unconsumeByte(insn);