Merge pull request #30 from ThomasWaldmann/twopass-assembly

make a 2-pass assembler to get layout/labels right

Author: ThomasWaldmann

demo.S

@@ -2,6 +2,8 @@
 
             .text
 
+            .set constant42, 42
+
 textstart:  ld r0, r1, 0    # a comment!
             st r0, r1, 0    // another comment!
             add r0, r1, r2
@@ -18,14 +20,17 @@ textstart:  ld r0, r1, 0    # a comment!
             rsh r0, r1, 42
             move r0, r1
             move r0, 42
+            move r0, textstart  # moves abs addr of textstart to r0
+            move r0, constant42
             stage_rst
             stage_inc 42
             stage_dec 23
 
-            # jumping to labels not supported yet
-            jumpr -1, 42, lt
-            jumpr +1, 23, GE
-            jump 0
+rel_b:      jumpr -1, 42, lt
+            jumpr rel_b, 42, LT
+            jumpr rel_f, 23, ge
+rel_f:      jumpr +1, 23, GE
+            jump textstart
             jump 0, eq
             jump 0, OV
             jump r0

esp32_ulp/assemble.py

@@ -7,11 +7,97 @@
 
 TEXT, DATA, BSS = 'text', 'data', 'bss'
 
+REL, ABS = 0, 1
+
+
+class SymbolTable:
+    def __init__(self, symbols, bases):
+        self._symbols = symbols
+        self._bases = bases
+        self._pass = None
+
+    def set_pass(self, _pass):
+        self._pass = _pass
+
+    def set_bases(self, bases):
+        self._bases = bases
+
+    def set_from(self, from_section, from_offset):
+        self._from_section, self._from_offset = from_section, from_offset
+
+    def get_from(self):
+        return self._from_section, self._from_offset
+
+    def set_sym(self, symbol, stype, section, value):
+        entry = (stype, section, value)
+        if symbol in self._symbols and entry != self._symbols[symbol]:
+            raise Exception('redefining symbol %s with different value %r -> %r.' % (label, self._symbols[symbol], entry))
+        self._symbols[symbol] = entry
+
+    def has_sym(self, symbol):
+        return symbol in self._symbols
+        
+    def get_sym(self, symbol):
+        try:
+            entry = self._symbols[symbol]
+        except KeyError:
+            if self._pass == 1:
+                entry = (REL, TEXT, 0)  # for a dummy, this is good enough
+            else:
+                raise
+        return entry
+
+    def dump(self):
+        for symbol, entry in self._symbols.items():
+            print(symbol, entry)
+        
+    def to_abs_addr(self, section, offset):
+        try:
+            base = self._bases[section]
+        except KeyError:
+            if self._pass == 1:
+                base = 0  # for a dummy this is good enough
+            else:
+                raise
+        return base + offset
+
+    def resolve_absolute(self, symbol):
+        if isinstance(symbol, str):
+            stype, section, value = self.get_sym(symbol)
+        elif isinstance(symbol, tuple):
+            stype, section, value = symbol
+        else:
+            raise TypeError
+        if stype == REL:
+            return self.to_abs_addr(section, value)
+        if stype == ABS:
+            return value
+        raise TypeError(stype)
+
+    def resolve_relative(self, symbol):
+        if isinstance(symbol, str):
+            sym_type, sym_section, sym_value = self.get_sym(symbol)
+        elif isinstance(symbol, tuple):
+            sym_type, sym_section, sym_value = symbol
+        else:
+            raise TypeError
+        if sym_type == REL:
+            sym_addr = self.to_abs_addr(sym_section, sym_value)
+        elif sym_type == ABS:
+            sym_addr = sym_value
+        from_addr = self.to_abs_addr(self._from_section, self._from_offset)
+        return sym_addr - from_addr
+
 
 class Assembler:
 
-    def __init__(self):
-        self.symbols = {}
+    def __init__(self, symbols=None, bases=None):
+        self.symbols = SymbolTable(symbols or {}, bases or {})
+        opcodes.symbols = self.symbols  # XXX dirty hack
+
+    def init(self, a_pass):
+        self.a_pass = a_pass
+        self.symbols.set_pass(a_pass)
         self.sections = dict(text=[], data=[])
         self.offsets = dict(text=0, data=0, bss=0)
         self.section = TEXT
@@ -78,10 +164,18 @@ def finalize_sections(self):
                 if s is not BSS:
                     self.sections[s].append(fill)
 
+    def compute_bases(self):
+        bases = {}
+        addr = 0
+        # lay out sections in this order:
+        for s in [TEXT, DATA, BSS]:  # TODO: more flexibility for custom sections
+            bases[s] = addr
+            addr += self.offsets[s] // 4  # 32bit word addresses
+        return bases
+
     def dump(self):
         print("Symbols:")
-        for label, section_offset in sorted(self.symbols.items()):
-            print(label, section_offset)
+        self.symbols.dump()
         print("%s section:" % TEXT)
         for t in self.sections[TEXT]:
             print("%08x" % int.from_bytes(t, 'little'))
@@ -108,19 +202,34 @@ def d_data(self):
     def d_bss(self):
         self.section = BSS
 
+    def fill(self, section, amount, fill_byte):
+        if fill_byte is not None and section is BSS:
+            raise ValueError('fill in bss section not allowed')
+        if section is TEXT:  # TODO: text section should be filled with NOPs
+            raise ValueError('fill/skip/align in text section not supported')
+        fill = int(fill_byte or 0).to_bytes(1, 'little') * amount
+        self.offsets[section] += len(fill)
+        if section is not BSS:
+            self.sections[section].append(fill)
+
     def d_skip(self, amount, fill=None):
-        s = self.section
         amount = int(amount)
-        if fill is not None and s is BSS:
-            raise ValueError('fill not allowed in section %s' % s)
-        if s is BSS:
-            self.append_section(amount)
-        else:
-            fill = int(fill or 0).to_bytes(1, 'little') * amount
-            self.append_section(fill)
+        self.fill(self.section, amount, fill)
 
     d_space = d_skip
 
+    def d_align(self, align=4, fill=None):
+        align = int(align)
+        offs = self.offsets[self.section]
+        mod = offs % align
+        if mod:
+            amount = align - mod
+            self.fill(self.section, amount, fill)
+
+    def d_set(self, symbol, expr):
+        value = int(expr)  # TODO: support more than just integers
+        self.symbols.set_sym(symbol, ABS, None, value)
+
     def append_data(self, wordlen, args):
         data = [int(arg).to_bytes(wordlen, 'little') for arg in args]
         self.append_section(b''.join(data))
@@ -134,12 +243,11 @@ def d_word(self, *args):
     def d_long(self, *args):
         self.append_data(4, args)
 
-    def assemble(self, lines):
+    def assembler_pass(self, lines):
         for label, opcode, args in self.parse(lines):
+            self.symbols.set_from(self.section, self.offsets[self.section] // 4)
             if label is not None:
-                if label in self.symbols:
-                    raise Exception('label %s is already defined.' % label)
-                self.symbols[label] = (self.section, self.offsets[self.section] // 4)
+                self.symbols.set_sym(label, REL, *self.symbols.get_from())
             if opcode is not None:
                 if opcode[0] == '.':
                     # assembler directive
@@ -159,3 +267,10 @@ def assemble(self, lines):
                 raise Exception('Unknown opcode or directive: %s' % opcode)
         self.finalize_sections()
 
+    def assemble(self, lines):
+        self.init(1)  # pass 1 is only to get the symbol table right
+        self.assembler_pass(lines)
+        self.symbols.set_bases(self.compute_bases())
+        self.init(2)  # now we know all symbols and bases, do the real assembler pass, pass 2
+        self.assembler_pass(lines)
+

esp32_ulp/opcodes.py

@@ -7,6 +7,9 @@
 
 from .soc import *
 
+# XXX dirty hack: use a global for the symbol table
+symbols = None
+
 # Opcodes, Sub-Opcodes, Modes, ...
 
 OPCODE_WR_REG = 1
@@ -250,7 +253,7 @@ def make_ins(layout):
 
 # assembler opcode definitions
 
-REG, IMM, COND = 0, 1, 2
+REG, IMM, COND, SYM = 0, 1, 2, 3
 ARG = namedtuple('ARG', ('type', 'value', 'raw'))
 
 
@@ -276,25 +279,44 @@ def arg_qualify(arg):
         return ARG(IMM, int(arg), arg)
     except ValueError:
         pass
-    raise TypeError('arg_qualify: unsupported arg type: %s' % arg)
+    entry = symbols.get_sym(arg)
+    return ARG(SYM, entry, arg)
 
 
 def get_reg(arg):
-    arg = arg_qualify(arg)
+    if isinstance(arg, str):
+        arg = arg_qualify(arg)
     if arg.type == REG:
         return arg.value
     raise TypeError('wanted: register, got: %s' % arg.raw)
 
 
 def get_imm(arg):
-    arg = arg_qualify(arg)
+    if isinstance(arg, str):
+        arg = arg_qualify(arg)
+    if arg.type == IMM:
+        return arg.value
+    if arg.type == SYM:
+        return symbols.resolve_absolute(arg.value)
+    raise TypeError('wanted: immediate, got: %s' % arg.raw)
+
+
+get_abs = get_imm
+
+
+def get_rel(arg):
+    if isinstance(arg, str):
+        arg = arg_qualify(arg)
     if arg.type == IMM:
         return arg.value
+    if arg.type == SYM:
+        return symbols.resolve_relative(arg.value)
     raise TypeError('wanted: immediate, got: %s' % arg.raw)
 
 
 def get_cond(arg):
-    arg = arg_qualify(arg)
+    if isinstance(arg, str):
+        arg = arg_qualify(arg)
     if arg.type == COND:
         return arg.value
     raise TypeError('wanted: condition, got: %s' % arg.raw)
@@ -429,10 +451,10 @@ def i_move(reg_dest, reg_imm_src):
         _alu_reg.sub_opcode = SUB_OPCODE_ALU_REG
         _alu_reg.opcode = OPCODE_ALU
         return _alu_reg.all
-    if src.type == IMM:
+    if src.type == IMM or src.type == SYM:
         _alu_imm.dreg = dest
         _alu_imm.sreg = 0
-        _alu_imm.imm = src.value
+        _alu_imm.imm = get_abs(src)
         _alu_imm.unused = 0
         _alu_imm.sel = ALU_SEL_MOV
         _alu_imm.sub_opcode = SUB_OPCODE_ALU_IMM
@@ -457,10 +479,10 @@ def _alu3(reg_dest, reg_src1, reg_imm_src2, alu_sel):
         _alu_reg.sub_opcode = SUB_OPCODE_ALU_REG
         _alu_reg.opcode = OPCODE_ALU
         return _alu_reg.all
-    if src2.type == IMM:
+    if src2.type == IMM or src2.type == SYM:
         _alu_imm.dreg = dest
         _alu_imm.sreg = src1
-        _alu_imm.imm = src2.value
+        _alu_imm.imm = get_abs(src2)
         _alu_imm.unused = 0
         _alu_imm.sel = alu_sel
         _alu_imm.sub_opcode = SUB_OPCODE_ALU_IMM
@@ -546,9 +568,9 @@ def i_jump(target, condition='--'):
         jump_type = BX_JUMP_TYPE_DIRECT
     else:
         raise ValueError("invalid flags condition")
-    if target.type == IMM:
+    if target.type == IMM or target.type == SYM:
         _bx.dreg = 0
-        _bx.addr = target.value
+        _bx.addr = get_abs(target)
         _bx.unused = 0
         _bx.reg = 0
         _bx.type = jump_type
@@ -568,7 +590,7 @@ def i_jump(target, condition='--'):
 
 
 def i_jumpr(offset, threshold, condition):
-    offset = get_imm(offset)
+    offset = get_rel(offset)
     threshold = get_imm(threshold)
     condition = get_cond(condition)
     if condition == 'lt':
@@ -587,7 +609,7 @@ def i_jumpr(offset, threshold, condition):
 
 
 def i_jumps(offset, threshold, condition):
-    offset = get_imm(offset)
+    offset = get_rel(offset)
     threshold = get_imm(threshold)
     condition = get_cond(condition)
     if condition == 'lt':

tests/01_compat_tests.sh

@@ -29,9 +29,9 @@ for src_file in $(ls -1 compat/*.S); do
         echo "py-esp32-ulp log:"
         cat $log_file
         echo "py-esp32-ulp output:"
-        xxd -e $ulp_file
+        xxd $ulp_file
         echo "binutils output:"
-        xxd -e $bin_file
+        xxd $bin_file
         exit 1
     else
         echo -e "\tBuild outputs match"

tests/assemble.py

@@ -1,4 +1,4 @@
-from esp32_ulp.assemble import Assembler, TEXT, DATA, BSS
+from esp32_ulp.assemble import Assembler, TEXT, DATA, BSS, REL, ABS
 
 src = """\
 
@@ -31,9 +31,10 @@ def test_parse():
 def test_assemble():
     a = Assembler()
     a.assemble(src)
-    assert {'start', 'end'} <= set(a.symbols)
-    assert a.symbols['start'] == (TEXT, 0)
-    assert a.symbols['end'] == (TEXT, 4)
+    assert a.symbols.has_sym('start')
+    assert a.symbols.has_sym('end')
+    assert a.symbols.get_sym('start') == (REL, TEXT, 0)
+    assert a.symbols.get_sym('end') == (REL, TEXT, 4)
     assert len(b''.join(a.sections[TEXT])) == 16  # 4 instructions * 4B
     assert len(a.sections[DATA]) == 0
     assert a.offsets[BSS] == 0

tests/compat/sections.S

@@ -8,8 +8,12 @@
 
             .space 4
             .space 8, 0xFF
+            .space 1
+            .align 4
+            .space 3
 
+            # a section start will be automatically 32bit-aligned:
             .bss
 
             .space 10
-
+            # a section end will be automatically 32bit-aligned