Add wasm-emscripten-finalize flag to separate data segments into a file (#1741)

This writes the data section into a file suitable for use with emscripten's
--memory-init-file flag

Author: dschuff

scripts/test/lld.py

@@ -24,13 +24,14 @@
 def test_wasm_emscripten_finalize():
   print '\n[ checking wasm-emscripten-finalize testcases... ]\n'
 
-  extension_arg_map = {
-      '.out': [],
-      '.jscall.out': ['--emscripten-reserved-function-pointers=3'],
-  }
-
   for wast_path in files_with_pattern(options.binaryen_test, 'lld', '*.wast'):
     print '..', wast_path
+    mem_file = wast_path + '.mem'
+    extension_arg_map = {
+      '.out': [],
+      '.jscall.out': ['--emscripten-reserved-function-pointers=3'],
+      '.mem.out': ['--separate-data-segments', mem_file],
+    }
     for ext, ext_args in extension_arg_map.items():
       expected_file = wast_path + ext
       if ext != '.out' and not os.path.exists(expected_file):
@@ -44,6 +45,11 @@ def test_wasm_emscripten_finalize():
         print actual
         fail_with_error('output ' + expected_file + ' does not exist')
       fail_if_not_identical_to_file(actual, expected_file)
+      if ext == '.mem.out':
+        with open(mem_file) as mf:
+          mem = mf.read()
+          fail_if_not_identical_to_file(mem, wast_path + '.mem.mem')
+        os.remove(mem_file)
 
 
 if __name__ == '__main__':

src/support/file.h

@@ -59,6 +59,10 @@ class Output {
     return out;
   }
 
+  std::ostream& write(const char* s, std::streamsize c) {
+    return out.write(s, c);
+  }
+
  private:
   Output() = delete;
   Output(const Output &) = delete;

src/tools/wasm-emscripten-finalize.cpp

@@ -40,6 +40,7 @@ int main(int argc, const char *argv[]) {
   std::string inputSourceMapFilename;
   std::string outputSourceMapFilename;
   std::string outputSourceMapUrl;
+  std::string dataSegmentFile;
   bool emitBinary = true;
   bool debugInfo = false;
   bool legalizeJavaScriptFFI = true;
@@ -94,6 +95,9 @@ int main(int argc, const char *argv[]) {
       .add("--output-source-map-url", "-osu", "Emit specified string as source map URL",
            Options::Arguments::One,
            [&outputSourceMapUrl](Options *o, const std::string& argument) { outputSourceMapUrl = argument; })
+      .add("--separate-data-segments", "", "Separate data segments to a file",
+           Options::Arguments::One,
+           [&dataSegmentFile](Options *o, const std::string& argument) { dataSegmentFile = argument;})
       .add_positional("INFILE", Options::Arguments::One,
                       [&infile](Options *o, const std::string& argument) {
                         infile = argument;
@@ -161,6 +165,10 @@ int main(int argc, const char *argv[]) {
   generator.generateDynCallThunks();
   generator.generateJSCallThunks(numReservedFunctionPointers);
   std::string metadata = generator.generateEmscriptenMetadata(dataSize, initializerFunctions, numReservedFunctionPointers);
+  if (!dataSegmentFile.empty()) {
+    Output memInitFile(dataSegmentFile, Flags::Binary, Flags::Release);
+    generator.separateDataSegments(&memInitFile);
+  }
 
   if (options.debug) {
     std::cerr << "Module after:\n";

src/wasm-emscripten.h

@@ -19,6 +19,8 @@
 
 #include "wasm.h"
 #include "wasm-builder.h"
+#include "support/file.h"
+
 
 namespace wasm {
 
@@ -53,6 +55,8 @@ class EmscriptenGlueGenerator {
 
   void fixInvokeFunctionNames();
 
+  void separateDataSegments(Output* outfile);
+
 private:
   Module& wasm;
   Builder builder;

src/wasm/wasm-emscripten.cpp

@@ -810,4 +810,19 @@ std::string EmscriptenGlueGenerator::generateEmscriptenMetadata(
   return meta.str();
 }
 
+void EmscriptenGlueGenerator::separateDataSegments(Output* outfile) {
+  size_t lastEnd = 0;
+  for (Memory::Segment& seg : wasm.memory.segments) {
+    size_t offset = seg.offset->cast<Const>()->value.geti32();
+    size_t fill = offset - lastEnd;
+    if (fill > 0) {
+      std::vector<char> buf(fill);
+      outfile->write(buf.data(), fill);
+    }
+    outfile->write(seg.data.data(), seg.data.size());
+    lastEnd = offset + seg.data.size();
+  }
+  wasm.memory.segments.clear();
+}
+
 } // namespace wasm

test/lld/hello_world.wast.mem.mem

@@ -0,0 +1,61 @@
+(module
+ (type $0 (func (param i32) (result i32)))
+ (type $1 (func (result i32)))
+ (type $2 (func))
+ (type $FUNCSIG$ii (func (param i32) (result i32)))
+ (import "env" "puts" (func $puts (param i32) (result i32)))
+ (memory $0 2)
+ (table $0 1 1 anyfunc)
+ (global $global$0 (mut i32) (i32.const 66128))
+ (global $global$1 i32 (i32.const 66128))
+ (global $global$2 i32 (i32.const 581))
+ (export "memory" (memory $0))
+ (export "__wasm_call_ctors" (func $__wasm_call_ctors))
+ (export "main" (func $main))
+ (export "__heap_base" (global $global$1))
+ (export "__data_end" (global $global$2))
+ (export "stackSave" (func $stackSave))
+ (export "stackAlloc" (func $stackAlloc))
+ (export "stackRestore" (func $stackRestore))
+ (export "__growWasmMemory" (func $__growWasmMemory))
+ (func $main (; 1 ;) (type $1) (result i32)
+  (drop
+   (call $puts
+    (i32.const 568)
+   )
+  )
+  (i32.const 0)
+ )
+ (func $__wasm_call_ctors (; 2 ;) (type $2)
+  (nop)
+ )
+ (func $stackSave (; 3 ;) (result i32)
+  (get_global $global$0)
+ )
+ (func $stackAlloc (; 4 ;) (param $0 i32) (result i32)
+  (local $1 i32)
+  (set_global $global$0
+   (tee_local $1
+    (i32.and
+     (i32.sub
+      (get_global $global$0)
+      (get_local $0)
+     )
+     (i32.const -16)
+    )
+   )
+  )
+  (get_local $1)
+ )
+ (func $stackRestore (; 5 ;) (param $0 i32)
+  (set_global $global$0
+   (get_local $0)
+  )
+ )
+ (func $__growWasmMemory (; 6 ;) (param $newSize i32) (result i32)
+  (grow_memory
+   (get_local $newSize)
+  )
+ )
+)
+;; METADATA: { "asmConsts": {},"staticBump": 13, "initializers": ["__wasm_call_ctors"], "declares": ["puts"], "externs": [], "implementedFunctions": ["___wasm_call_ctors","_main","_stackSave","_stackAlloc","_stackRestore","___growWasmMemory"], "exports": ["memory","__wasm_call_ctors","main","__heap_base","__data_end","stackSave","stackAlloc","stackRestore","__growWasmMemory"], "invokeFuncs": [] }