add preprocessor that can replace simple #define values in code

The preprocessor strips all comments and lines containing a define statement
but keeps the empty lines in the output (to preserve line numbering). The
output is then passed directly into the assembler.

The preprocessor does not support "function style" #define macros (i.e.
ADD(a,b) a+b) but this is not needed for expanding the constants used by
WRITE_RTC_REG(), et al.

Author: wnienhaus

esp32_ulp/__main__.py

@@ -2,6 +2,7 @@
 
 from .util import garbage_collect
 
+from .preprocess import preprocess
 from .assemble import Assembler
 from .link import make_binary
 garbage_collect('after import')
@@ -23,6 +24,7 @@ def main(fn):
     with open(fn) as f:
         src = f.read()
 
+    src = preprocess(src)
     binary = src_to_binary(src)
 
     if fn.endswith('.s') or fn.endswith('.S'):

esp32_ulp/preprocess.py

@@ -0,0 +1,57 @@
+from . import nocomment
+from .util import split_tokens
+
+
+class Preprocessor:
+    def __init__(self):
+        self._defines = {}
+
+    def parse_defines(self, content):
+        result = {}
+        for line in content.splitlines():
+            line = line.strip()
+            if not line.startswith("#define"):
+                # skip lines not containing #define
+                continue
+            line = line[8:].strip()  # remove #define
+            parts = line.split(None, 1)
+            if len(parts) != 2:
+                # skip defines without value
+                continue
+            identifier, value = parts
+            tmp = identifier.split('(', 1)
+            if len(tmp) == 2:
+                # skip parameterised defines (macros)
+                continue
+            value = "".join(nocomment.remove_comments(value)).strip()
+            result[identifier] = value
+        self._defines = result
+        return result
+
+    def expand_defines(self, line):
+        found = True
+        while found:  # do as many passed as needed, until nothing was replaced anymore
+            found = False
+            tokens = split_tokens(line)
+            line = ""
+            for t in tokens:
+                lu = self._defines.get(t, t)
+                if lu != t:
+                    found = True
+                line += lu
+
+        return line
+
+    def preprocess(self, content):
+        self.parse_defines(content)
+        lines = nocomment.remove_comments(content)
+        result = []
+        for line in lines:
+            line = self.expand_defines(line)
+            result.append(line)
+        result = "\n".join(result)
+        return result
+
+
+def preprocess(content):
+    return Preprocessor().preprocess(content)

tests/00_unit_tests.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-for file in opcodes assemble link util; do
+for file in opcodes assemble link util preprocess; do
     echo testing $file...
     micropython $file.py
 done

tests/01_compat_tests.sh

@@ -13,12 +13,14 @@ for src_file in $(ls -1 compat/*.S); do
     log_file="${src_name}.log"
     micropython -m esp32_ulp $src_file 1>$log_file   # generates $ulp_file
 
+    pre_file="${src_name}.pre"
     obj_file="${src_name}.o"
     elf_file="${src_name}.elf"
     bin_file="${src_name}.bin"
 
     echo -e "\tBuilding using binutils"
-    esp32ulp-elf-as -o $obj_file $src_file
+    gcc -E -o ${pre_file} $src_file
+    esp32ulp-elf-as -o $obj_file ${pre_file}
     esp32ulp-elf-ld -T esp32.ulp.ld -o $elf_file $obj_file
     esp32ulp-elf-objcopy -O binary $elf_file $bin_file
 

tests/compat/preprocess_simple.S

@@ -0,0 +1,7 @@
+#define GPIO 2
+#define BASE 0x100
+#define ADDR (BASE + GPIO)
+
+entry:
+  move r0, GPIO
+  move r1, ADDR

tests/preprocess.py

@@ -0,0 +1,175 @@
+from esp32_ulp.preprocess import Preprocessor
+
+tests = []
+
+
+def test(param):
+    tests.append(param)
+
+
+@test
+def test_replace_defines_should_return_empty_line_given_empty_string():
+    p = Preprocessor()
+
+    assert p.preprocess("") == ""
+
+
+@test
+def replace_defines_should_return_remove_comments():
+    p = Preprocessor()
+
+    line = "// some comment"
+    expected = ""
+    assert p.preprocess(line) == expected
+
+
+@test
+def test_parse_defines():
+    p = Preprocessor()
+
+    assert p.parse_defines("") == {}
+    assert p.parse_defines("// comment") == {}
+    assert p.parse_defines("  // comment") == {}
+    assert p.parse_defines("  /* comment */") == {}
+    assert p.parse_defines("  /* comment */ #define A 42") == {}  # #define must be the first thing on a line
+    assert p.parse_defines("#define a 1") == {"a": "1"}
+    assert p.parse_defines(" #define a 1") == {"a": "1"}
+    assert p.parse_defines("#define a 1 2") == {"a": "1 2"}
+    assert p.parse_defines("#define f(a,b) 1") == {}  # macros not supported
+    assert p.parse_defines("#define f(a, b) 1") == {}  # macros not supported
+    assert p.parse_defines("#define f (a,b) 1") == {"f": "(a,b) 1"}  # f is not a macro
+    assert p.parse_defines("#define f (a, b) 1") == {"f": "(a, b) 1"}  # f is not a macro
+    assert p.parse_defines("#define RTC_ADDR       0x12345    // start of range") == {"RTC_ADDR": "0x12345"}
+
+
+@test
+def test_parse_defines_handles_multiple_input_lines():
+    p = Preprocessor()
+
+    multi_line_1 = """\
+#define ID_WITH_UNDERSCORE something
+#define ID2 somethingelse
+"""
+    assert p.parse_defines(multi_line_1) == {"ID_WITH_UNDERSCORE": "something", "ID2": "somethingelse"}
+
+
+@test
+def test_parse_defines_does_not_understand_comments_by_current_design():
+    # comments are not understood. lines are expected to already have comments removed!
+    p = Preprocessor()
+
+    multi_line_2 = """\
+#define ID_WITH_UNDERSCORE something
+/*
+#define ID2 somethingelse
+*/
+"""
+    assert "ID2" in p.parse_defines(multi_line_2)
+
+
+@test
+def test_parse_defines_does_not_understand_line_continuations_with_backslash_by_current_design():
+    p = Preprocessor()
+
+    multi_line_3 = r"""
+    #define ID_WITH_UNDERSCORE something \
+           line2
+    """
+
+    assert p.parse_defines(multi_line_3) == {"ID_WITH_UNDERSCORE": "something \\"}
+
+
+@test
+def preprocess_should_remove_comments_and_defines_but_keep_the_lines_as_empty_lines():
+    p = Preprocessor()
+
+    lines = """\
+    // copyright
+    #define A 1
+
+    move r1, r2"""
+
+    assert p.preprocess(lines) == "\n\n\n\tmove r1, r2"
+
+
+@test
+def preprocess_should_replace_words_defined():
+    p = Preprocessor()
+
+    lines = """\
+    #define DR_REG_RTCIO_BASE 0x3ff48400
+
+    move r1, DR_REG_RTCIO_BASE"""
+
+    assert "move r1, 0x3ff48400" in p.preprocess(lines)
+
+
+@test
+def preprocess_should_replace_words_defined_multiple_times():
+    p = Preprocessor()
+
+    lines = """\
+    #define DR_REG_RTCIO_BASE 0x3ff48400
+
+    move r1, DR_REG_RTCIO_BASE  #once
+    move r2, DR_REG_RTCIO_BASE  #second time"""
+
+    assert "move r1, 0x3ff48400" in p.preprocess(lines)
+    assert "move r2, 0x3ff48400" in p.preprocess(lines)
+
+
+@test
+def preprocess_should_replace_all_defined_words():
+    p = Preprocessor()
+
+    lines = """\
+    #define DR_REG_RTCIO_BASE 0x3ff48400
+    #define SOME_OFFSET 4
+
+    move r1, DR_REG_RTCIO_BASE
+    add r2, r1, SOME_OFFSET"""
+
+    assert "move r1, 0x3ff48400" in p.preprocess(lines)
+    assert "add r2, r1, 4" in p.preprocess(lines)
+
+
+@test
+def preprocess_should_not_replace_substrings_within_identifiers():
+    p = Preprocessor()
+
+    # ie. if AAA is defined don't touch PREFIX_AAA_SUFFIX
+    lines = """\
+    #define RTCIO 4
+    move r1, DR_REG_RTCIO_BASE"""
+
+    assert "DR_REG_4_BASE" not in p.preprocess(lines)
+
+    # ie. if A and AA are defined, don't replace AA as two A's but with AA
+    lines = """\
+    #define A 4
+    #define AA 8
+    move r1, A
+    move r2, AA"""
+
+    assert "move r1, 4" in p.preprocess(lines)
+    assert "move r2, 8" in p.preprocess(lines)
+
+
+@test
+def preprocess_should_replace_defines_used_in_defines():
+    p = Preprocessor()
+
+    lines = """\
+    #define BITS (BASE << 4)
+    #define BASE 0x1234
+
+    move r1, BITS
+    move r2, BASE"""
+
+    assert "move r1, (0x1234 << 4)" in p.preprocess(lines)
+
+
+if __name__ == '__main__':
+    # run all methods marked with @test
+    for t in tests:
+        t()