Test source map support for emsymbolizer (#17568)

This adds source map tests for emsymbolizer. This also does some
refactoring in the script.

- Created `LocationInfo` class, which is used to treat location info in
  the same way across information sources (now DWARF and source map).
  Currently its `print` method prints location info in the same format
  as `llvm-dwarfdump` does.
- Removed `name` field reference from the script. It's not currently
  used anywhere, unclear what that means, and our wasm-sourcemap.py
  doesn't put anything in that field either.
- Renamed `WasmSourceMap.mapping` to `WasmSourceMap.mappings`
- Removed the warning message saying the command line and the output
  format are not finalized yet. What is? :)
- Added source map tests to `test_emsymbolizer`. Addresses are changed
  because now we build the binary with both DWARF and the source map
  enabled. In case of DWARF, we additionally check inlined info and
  function names.

Author: aheejin

emsymbolizer.py

@@ -15,10 +15,11 @@
 from collections import namedtuple
 import json
 import os
+import re
+import subprocess
 import sys
 from tools import shared
 from tools import webassembly
-from tools.shared import check_call
 
 LLVM_SYMBOLIZER = os.path.expanduser(
     shared.build_llvm_tool_path(shared.exe_suffix('llvm-symbolizer')))
@@ -28,6 +29,20 @@ class Error(BaseException):
   pass
 
 
+# Class to treat location info in a uniform way across information sources.
+class LocationInfo(object):
+  def __init__(self, source=None, line=0, column=0, func=None):
+    self.source = source
+    self.line = line
+    self.column = column
+    self.func = func
+
+  def print(self):
+    source = self.source if self.source else '??'
+    func = self.func if self.func else '??'
+    print(f'{func}\n{source}:{self.line}:{self.column}')
+
+
 def get_codesec_offset(module):
   sec = module.get_section(webassembly.SecType.CODE)
   if not sec:
@@ -46,7 +61,24 @@ def symbolize_address_dwarf(module, address):
   vma_adjust = get_codesec_offset(module)
   cmd = [LLVM_SYMBOLIZER, '-e', module.filename, f'--adjust-vma={vma_adjust}',
          str(address)]
-  check_call(cmd)
+  out = shared.run_process(cmd, stdout=subprocess.PIPE).stdout.strip()
+  out_lines = out.splitlines()
+  # Source location regex, e.g., /abc/def.c:3:5
+  SOURCE_LOC_RE = re.compile(r'(.+):(\d+):(\d+)$')
+  # llvm-dwarfdump prints two lines per location. The first line contains a
+  # function name, and the second contains a source location like
+  # '/abc/def.c:3:5'. If the function or source info is not available, it will
+  # be printed as '??', in which case we store None. If the line and column info
+  # is not available, they will be printed as 0, which we store as is.
+  for i in range(0, len(out_lines), 2):
+    func, loc_str = out_lines[i], out_lines[i + 1]
+    m = SOURCE_LOC_RE.match(loc_str)
+    source, line, column = m.group(1), m.group(2), m.group(3)
+    if func == '??':
+      func = None
+    if source == '??':
+      source = None
+    LocationInfo(source, line, column, func).print()
 
 
 def get_sourceMappingURL_section(module):
@@ -58,14 +90,12 @@ def get_sourceMappingURL_section(module):
 
 class WasmSourceMap(object):
   # This implementation is derived from emscripten's sourcemap-support.js
-  Location = namedtuple('Location',
-                        ['source', 'line', 'column', 'name'])
+  Location = namedtuple('Location', ['source', 'line', 'column'])
 
   def __init__(self):
     self.version = None
     self.sources = []
-    self.names = []
-    self.mapping = {}
+    self.mappings = {}
     self.offsets = []
 
   def parse(self, filename):
@@ -76,7 +106,6 @@ def parse(self, filename):
 
     self.version = source_map_json['version']
     self.sources = source_map_json['sources']
-    self.names = source_map_json['names']
 
     vlq_map = {}
     chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
@@ -106,7 +135,6 @@ def decodeVLQ(string):
     src = 0
     line = 1
     col = 1
-    name = 0
     for segment in source_map_json['mappings'].split(','):
       data = decodeVLQ(segment)
       info = []
@@ -121,11 +149,9 @@ def decodeVLQ(string):
       if len(data) >= 4:
         col += data[3]
         info.append(col)
-      if len(data) >= 5:
-        name += data[4]
-        info.append(name)
+      # TODO: see if we need the name, which is the next field (data[4])
 
-      self.mapping[offset] = WasmSourceMap.Location(*info)
+      self.mappings[offset] = WasmSourceMap.Location(*info)
       self.offsets.append(offset)
     self.offsets.sort()
 
@@ -144,18 +170,13 @@ def find_offset(self, offset):
 
   def lookup(self, offset):
     nearest = self.find_offset(offset)
-    assert nearest in self.mapping, 'Sourcemap has an offset with no mapping'
-    info = self.mapping[nearest]
-
-    # TODO: it's kind of icky to use Location for both the internal indexed
-    # location and external string version. Once we have more uniform output
-    # format and API for the various backends (e.g SM vs DWARF vs others), this
-    # could be improved.
-    return WasmSourceMap.Location(
+    assert nearest in self.mappings, 'Sourcemap has an offset with no mapping'
+    info = self.mappings[nearest]
+    return LocationInfo(
         self.sources[info.source] if info.source is not None else None,
         info.line,
-        info.column,
-        self.names[info.name] if info.name is not None else None)
+        info.column
+      )
 
 
 def symbolize_address_sourcemap(module, address, force_file):
@@ -175,11 +196,11 @@ def symbolize_address_sourcemap(module, address, force_file):
   sm.parse(URL)
   if shared.DEBUG:
     csoff = get_codesec_offset(module)
-    print(sm.mapping)
+    print(sm.mappings)
     # Print with section offsets to easily compare against dwarf
-    for k, v in sm.mapping.items():
+    for k, v in sm.mappings.items():
       print(f'{k-csoff:x}: {v}')
-  print(sm.lookup(address))
+  sm.lookup(address).print()
 
 
 def main(args):
@@ -228,8 +249,6 @@ def get_args():
 
 
 if __name__ == '__main__':
-  print('Warning: the command-line and output format of this tool are not '
-        'finalized yet', file=sys.stderr)
   try:
     rv = main(get_args())
   except (Error, webassembly.InvalidWasmError, OSError) as e:

test/test_other.py

@@ -8631,29 +8631,45 @@ def test(infile, source_map_added_dir=''):
     test('inner/a.cpp', 'inner')
 
   def test_emsymbolizer(self):
-    # Test DWARF output
-    self.run_process([EMCC, test_file('core/test_dwarf.c'),
-                      '-g', '-O1', '-o', 'test_dwarf.js'])
+    def check_loc_info(address, source, funcs, locs):
+      out = self.run_process(
+          [emsymbolizer, '-tcode', '-s', source, 'test_dwarf.wasm', address],
+          stdout=PIPE).stdout
+      for func in funcs:
+        self.assertIn(func, out)
+      for loc in locs:
+        self.assertIn(loc, out)
 
     # Use hard-coded addresses. This is potentially brittle, but LLVM's
     # O1 output is pretty minimal so hopefully it won't break too much?
     # Another option would be to disassemble the binary to look for certain
     # instructions or code sequences.
 
-    def get_addr(address):
-      return self.run_process(
-          [emsymbolizer, 'test_dwarf.wasm', '-tcode', address], stdout=PIPE).stdout
-
-    # Check a location in foo(), not inlined.
-    # If the output binary size changes use `wasm-objdump -d` on the binary
-    # look for the offset of the first call to `out_to_js`.
-    self.assertIn('test_dwarf.c:6:3', get_addr('0x8'))
-    # Check that both bar (inlined) and main (inlinee) are in the output,
-    # as described by the DWARF.
-    # TODO: consider also checking the function names once the output format
-    # stabilizes more
-    self.assertRegex(get_addr('0x1f').replace('\n', ''),
-                     'test_dwarf.c:13:3.*test_dwarf.c:18:3')
+    # 1. Test DWARF + source map together
+    self.run_process([EMCC, test_file('core/test_dwarf.c'),
+                      '-g', '-gsource-map', '-O1', '-o', 'test_dwarf.js'])
+    # 0x8 corresponds to out_to_js(0) within foo(), uninlined
+    # DWARF info provides function names, but source maps don't
+    check_loc_info('0x8', 'dwarf', ['foo'], ['test_dwarf.c:6:3'])
+    check_loc_info('0x8', 'sourcemap', [], ['test_dwarf.c:6:3'])
+    # 0x1f corresponds to __builtin_trap() within bar(), inlined into main()
+    # DWARF info provides inlined info, but source maps don't
+    check_loc_info('0x1f', 'dwarf', ['bar', 'main'],
+                   ['test_dwarf.c:13:3', 'test_dwarf.c:18:3'])
+    check_loc_info('0x1f', 'sourcemap', [], ['test_dwarf.c:13:3'])
+
+    # 2. Test source map only
+    self.run_process([EMCC, test_file('core/test_dwarf.c'),
+                      '-gsource-map', '-O1', '-o', 'test_dwarf.js'])
+    check_loc_info('0x8', 'sourcemap', [], ['test_dwarf.c:6:3'])
+    check_loc_info('0x1f', 'sourcemap', [], ['test_dwarf.c:13:3'])
+
+    # 3. Test DWARF only
+    self.run_process([EMCC, test_file('core/test_dwarf.c'),
+                      '-g', '-O1', '-o', 'test_dwarf.js'])
+    check_loc_info('0x8', 'dwarf', ['foo'], ['test_dwarf.c:6:3'])
+    check_loc_info('0x1f', 'dwarf', ['bar', 'main'],
+                   ['test_dwarf.c:13:3', 'test_dwarf.c:18:3'])
 
   def test_separate_dwarf(self):
     self.run_process([EMCC, test_file('hello_world.c'), '-g'])