Fix matching of branch instructions with prefixes and sizes

Matching of branch instructions with prefixes and sizes is, to say the
least, tricky. Work through it, and add a new macro to help.

Fixes: #144
Signed-off-by: H. Peter Anvin (Intel) <[email protected]>

Author: H. Peter Anvin (Intel)

asm/assemble.c

@@ -1400,28 +1400,9 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
     ins->itemp   = temp;        /* Instruction template */
     eat = EA_SCALAR;            /* Expect a scalar EA */
 
-    /* Default operand size */
+    /* Default operand size (prefixes are handled in the byte code) */
     ins->op_size = bits != 16 ? 32 : 16;
 
-    if (bits == 64) {
-        if (ins->prefixes[PPS_ASIZE] == P_A16) {
-            nasm_warn(WARN_PREFIX_BADMODE_A16,
-                      "a64 prefix invalid in 64-bit mode");
-            ins->prefixes[PPS_ASIZE] = 0;
-        }
-    } else {
-        if (ins->prefixes[PPS_OSIZE] == P_O64) {
-            nasm_warn(WARN_PREFIX_BADMODE_O64,
-                      "o64 prefix invalid in %d-bit mode", bits);
-            ins->prefixes[PPS_OSIZE] = P_none;
-        }
-        if (ins->prefixes[PPS_ASIZE] == P_A64) {
-            nasm_warn(WARN_PREFIX_BADMODE_A64,
-                      "a64 prefix invalid in %d-bit mode", bits);
-            ins->prefixes[PPS_ASIZE] = P_none;
-        }
-    }
-
     nasm_zero(need_pfx);
 
     while (*codes) {
@@ -3061,7 +3042,7 @@ static enum match_result matches(const struct itemplate * const itemp,
 
     /* "Default" operand size (from mode and prefixes only) */
     op_size = ins->op_size;
-    if (itemp_has(itemp, IF_NWSIZE) && op_size == 32) {
+    if (bits == 64 && itemp_has(itemp, IF_NWSIZE) && op_size == 32) {
         /* If this is an nw instruction, default to 64 bits in 64-bit mode */
         op_size = bits;
     }
@@ -3101,18 +3082,23 @@ static enum match_result matches(const struct itemplate * const itemp,
 
         /* Handle implied SHORT or NEAR */
         if (unlikely(ttype & (NEAR|SHORT))) {
+            /* Treat BYTE as an alias for SHORT, ignoring size */
+            if (isize[i] == BITS8) {
+                itype[i] |= SHORT;
+                isize[i] = 0;
+            }
+            /* An explicit SHORT or BITS8 cancels NEAR; are synonyms */
+            if (itype[i] & SHORT) {
+                itype[i] &= ~NEAR;
+            }
+            /* NEAR is implicit unless otherwise specified */
+           if (!(itype[i] & (FAR|SHORT))) {
+                itype[i] |= ttype & NEAR;
+            }
             if ((ttype & (NEAR|SHORT)) == (NEAR|SHORT)) {
-                /* Only a short form exists; allow both NEAR and SHORT */
+                /* Only a short form exists; this is specially coded */
                 if (!(itype[i] & (FAR|ABS)))
                     itype[i] |= NEAR|SHORT;
-            } else if ((itype[i] & SHORT) || isize[i] == BITS8) {
-                /* An explicit SHORT or BITS8 cancel NEAR; are synonyms */
-                itype[i] &= ~NEAR;
-                if (!isize[i])
-                    isize[i] = BITS8;
-            } else if (!(itype[i] & (FAR|ABS|SHORT))) {
-                /* NEAR is implicit unless otherwise specified */
-                itype[i] |= ttype & NEAR;
             }
         }
 

asm/parser.c

@@ -964,8 +964,10 @@ insn *parse_line(char *buffer, insn *result, const int bits)
         while (i == TOKEN_SPECIAL || i == TOKEN_SIZE) {
             switch (tokval.t_integer) {
             case S_BYTE:
-                if (!setsize)   /* we want to use only the first */
+                if (!setsize) {   /* we want to use only the first */
+                    result->opt |= OPTIM_NO_Jcc_RELAX | OPTIM_NO_JMP_RELAX;
                     op->type |= BITS8;
+                }
                 setsize = 1;
                 break;
             case S_WORD:
@@ -1014,9 +1016,12 @@ insn *parse_line(char *buffer, insn *result, const int bits)
                 op->type |= FAR;
                 break;
             case S_NEAR:
+                /* This is not legacy behavior, even if it perhaps should be */
+                /* result->opt |= OPTIM_NO_Jcc_RELAX | OPTIM_NO_JMP_RELAX; */
                 op->type |= NEAR;
                 break;
             case S_SHORT:
+                result->opt |= OPTIM_NO_Jcc_RELAX | OPTIM_NO_JMP_RELAX;
                 op->type |= SHORT;
                 break;
             case S_ABS:

asm/warnings.dat

@@ -259,20 +259,11 @@ pragma-unknown [off] unknown \c{%pragma} facility or directive
     Warns about an unknown \c{%pragma} directive.
     This is not yet implemented for most cases.
 
-prefix-badmode-a64 [err] a64 prefix invalid in 16/32-bit mode
-    Warns that an \c{a64} prefix was specified in 16- or 32-bit
-    mode. If the error is demoted to a warning or suppressed, the
-    prefix is ignored by the assembler.
-
 prefix-badmode-o64 [err] o64 prefix invalid in 16/32-bit mode
     Warns that an \c{a64} prefix was specified in 16- or 32-bit
     mode. If the error is demoted to a warning or suppressed, the
-    prefix is ignored by the assembler.
-
-prefix-badmode-a16 [err] a16 prefix invalid in 64-bit mode
-    Warns that an \c{a16} prefix was specified in 64-bit mode.
-    If the error is demoted to a warning or suppressed, the
-    prefix is ignored by the assembler.
+    prefix is ignored by the assembler, but is likely to trigger
+    futher errors.
 
 prefix-bnd [on] invalid \c{BND} prefix
 =bnd

doc/changes.src

@@ -54,10 +54,9 @@ It is the production version of NASM since 2025.
 \b Hopefully fix building with OpenWatcom.
 
 \b Generate a warning, promoted to error by default, on the use of
-   \c{a64} or \c{o64} prefixes in 16- or 32-bit mode or \c{a16}
-   prefixes in 64-bit mode. Those prefixes are not encodable; if
-   demoted to a warning or suppressed the prefix is ignored, but
-   likely will generate additional, harder to debug, error messages.
+   \c{o64} prefixes in 16- or 32-bit mode. If demoted to a warning or
+   suppressed the prefix is ignored, but likely will trigger
+   subsequent, harder to debug, error messages.
 
 
 \S{cl-3.00} Version 3.00