Replace hacky_postprocess_around_closure_limitations.py (#16459)

juj · web-flow · commit 8f1ce84b236a · 2022-03-15T08:12:09.000-07:00
* Replace regex-based tools/hacky_postprocess_around_closure_limitations.py with an acorn AST-based unsafe optimizer.

* Add missing ArrowFunctionExpression. Rebaseline code size test.

* Fix TryStatement child traversal.

* Fix comment

* Apply review

* eslint

* Expand unsafe_optimizations in MODULARIZE builds.

* Rebaseline

* eslint

* Add pass to simplify modularize function start

* Rebaseline

* Rerun a second simple pass of Closure after main unsafe optimizations

* eslint

* Optimize comma-chained operator news. Add internal test suite to Emscripten suite.

* Rebaseline and restore code size test on Windows and mac.
diff --git a/emcc.py b/emcc.py
@@ -2857,13 +2857,13 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
 
   module_export_name_substitution()
 
-  # Run a final regex pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind
+  # Run a final optimization pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind
   # by processing steps that occurred after Closure.
-  if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0 and not settings.SINGLE_FILE:
-    # Process .js runtime file. Note that we need to handle the license text
-    # here, so that it will not confuse the hacky script.
-    js_manipulation.handle_license(final_js)
-    shared.run_process([shared.PYTHON, utils.path_from_root('tools/hacky_postprocess_around_closure_limitations.py'), final_js])
+  if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0:
+    shared.run_js_tool(utils.path_from_root('tools/unsafe_optimizations.js'), [final_js, '-o', final_js], cwd=utils.path_from_root('.'))
+    # Finally, rerun Closure compile with simple optimizations. It will be able to further minify the code. (n.b. it would not be safe
+    # to run in advanced mode)
+    final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args)
 
   # Unmangle previously mangled `import.meta` references in both main code and libraries.
   # See also: `preprocess` in parseTools.js.
@@ -3528,7 +3528,7 @@ def module_export_name_substitution():
   src = re.sub(r'{\s*[\'"]?__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__[\'"]?:\s*1\s*}', replacement, src)
   # For Node.js and other shell environments, create an unminified Module object so that
   # loading external .asm.js file that assigns to Module['asm'] works even when Closure is used.
-  if settings.MINIMAL_RUNTIME and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')):
+  if settings.MINIMAL_RUNTIME and not settings.MODULARIZE and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')):
     src = 'if(typeof Module==="undefined"){var Module={};}\n' + src
   write_file(final_js, src)
   shared.configuration.get_temp_files().note(final_js)
diff --git a/tests/code_size/hello_wasm_worker_wasm.js b/tests/code_size/hello_wasm_worker_wasm.js
@@ -1,16 +1,8 @@
-var b = Module;
-
-var c = b.$ww, e, f;
-
-e = b.mem || new WebAssembly.Memory({
+var b = Module, c = b.$ww, f, e = b.mem || new WebAssembly.Memory({
     initial: 256,
     maximum: 256,
     shared: !0
-});
-
-var g = e.buffer;
-
-var h = {}, k = 1;
+}), g = e.buffer, h = {}, k = 1, m = [], p, q;
 
 function l(a) {
     m.push(a);
@@ -22,12 +14,8 @@ function n(a) {
     d && f.get(d)(...a.x);
 }
 
-var m = [];
-
 c && (h[0] = this, addEventListener("message", l));
 
-var p, q;
-
 WebAssembly.instantiate(b.wasm, {
     a: {
         b: function(a, d) {
diff --git a/tests/code_size/hello_wasm_worker_wasm.json b/tests/code_size/hello_wasm_worker_wasm.json
@@ -1,10 +1,10 @@
 {
   "a.html": 737,
   "a.html.gz": 433,
-  "a.js": 753,
-  "a.js.gz": 468,
+  "a.js": 733,
+  "a.js.gz": 463,
   "a.wasm": 1800,
   "a.wasm.gz": 994,
-  "total": 3290,
-  "total_gz": 1895
+  "total": 3270,
+  "total_gz": 1890
 }
diff --git a/tests/code_size/hello_webgl2_wasm.json b/tests/code_size/hello_webgl2_wasm.json
@@ -1,10 +1,10 @@
 {
   "a.html": 569,
   "a.html.gz": 379,
-  "a.js": 4973,
-  "a.js.gz": 2417,
+  "a.js": 4962,
+  "a.js.gz": 2422,
   "a.wasm": 10458,
   "a.wasm.gz": 6671,
-  "total": 16000,
-  "total_gz": 9467
+  "total": 15989,
+  "total_gz": 9472
 }
diff --git a/tests/code_size/hello_webgl2_wasm2js.json b/tests/code_size/hello_webgl2_wasm2js.json
@@ -1,10 +1,10 @@
 {
   "a.html": 567,
   "a.html.gz": 379,
-  "a.js": 20046,
-  "a.js.gz": 8213,
+  "a.js": 18189,
+  "a.js.gz": 8017,
   "a.mem": 3171,
   "a.mem.gz": 2714,
-  "total": 23784,
-  "total_gz": 11306
+  "total": 21927,
+  "total_gz": 11110
 }
diff --git a/tests/code_size/hello_webgl_wasm.json b/tests/code_size/hello_webgl_wasm.json
@@ -1,10 +1,10 @@
 {
   "a.html": 569,
   "a.html.gz": 379,
-  "a.js": 4479,
-  "a.js.gz": 2250,
+  "a.js": 4447,
+  "a.js.gz": 2247,
   "a.wasm": 10458,
   "a.wasm.gz": 6671,
-  "total": 15506,
-  "total_gz": 9300
+  "total": 15474,
+  "total_gz": 9297
 }
diff --git a/tests/code_size/hello_webgl_wasm2js.json b/tests/code_size/hello_webgl_wasm2js.json
@@ -1,10 +1,10 @@
 {
   "a.html": 567,
   "a.html.gz": 379,
-  "a.js": 19531,
-  "a.js.gz": 8049,
+  "a.js": 17667,
+  "a.js.gz": 7853,
   "a.mem": 3171,
   "a.mem.gz": 2714,
-  "total": 23269,
-  "total_gz": 11142
+  "total": 21405,
+  "total_gz": 10946
 }
diff --git a/tests/code_size/hello_world_wasm.js b/tests/code_size/hello_world_wasm.js
@@ -1,24 +1,21 @@
-var d = Module;
-
-var e = new TextDecoder("utf8"), f, g, h;
+var d = Module, e = new TextDecoder("utf8"), f, g, h;
 
 WebAssembly.instantiate(d.wasm, {
     a: {
         a: function(a) {
-            var b = console, k = b.log;
+            var c = console, k = c.log;
             if (a) {
-                for (var l = a + void 0, c = a; !(c >= l) && f[c]; ) ++c;
-                a = e.decode(f.subarray(a, c));
+                for (var l = a + void 0, b = a; !(b >= l) && f[b]; ) ++b;
+                a = e.decode(f.subarray(a, b));
             } else a = "";
-            k.call(b, a);
+            k.call(c, a);
         }
     }
 }).then((function(a) {
     a = a.instance.exports;
     h = a.d;
     g = a.b;
-    var b = g.buffer;
-    f = new Uint8Array(b);
+    f = new Uint8Array(g.buffer);
     a.c();
     h();
 }));
diff --git a/tests/code_size/hello_world_wasm.json b/tests/code_size/hello_world_wasm.json
@@ -1,10 +1,10 @@
 {
   "a.html": 673,
   "a.html.gz": 431,
-  "a.js": 319,
-  "a.js.gz": 261,
+  "a.js": 308,
+  "a.js.gz": 258,
   "a.wasm": 104,
   "a.wasm.gz": 112,
-  "total": 1096,
-  "total_gz": 804
+  "total": 1085,
+  "total_gz": 801
 }
diff --git a/tests/code_size/hello_world_wasm2js.js b/tests/code_size/hello_world_wasm2js.js
@@ -1,73 +1,42 @@
-var b = Module;
-
-var c = new TextDecoder("utf8"), d, f;
-
-f = new function(a) {
+var b = Module, c = new TextDecoder("utf8"), f = new function(a) {
     this.buffer = new ArrayBuffer(65536 * a.initial);
 }({
     initial: 256,
     maximum: 256
-});
-
-var h = f.buffer;
-
-d = new Uint8Array(h);
-
-var k = {
+}), h = f.buffer, d = new Uint8Array(h), k = {
     a: function(a) {
-        var g = console, m = g.log;
+        var n = console, e = n.log;
         if (a) {
-            for (var n = a + void 0, e = a; !(e >= n) && d[e]; ) ++e;
-            a = c.decode(d.subarray(a, e));
+            for (var g = a + void 0, m = a; !(m >= g) && d[m]; ) ++m;
+            a = c.decode(d.subarray(a, m));
         } else a = "";
-        m.call(g, a);
+        e.call(n, a);
     },
     memory: f
-}, l, p = (new function() {
-    this.exports = function instantiate(v) {
-        function c(d) {
-            d.set = function(a, b) {
-                this[a] = b;
+}, p = (new function() {
+    this.exports = function(a) {
+        function n(e) {
+            e.set = function(g, m) {
+                this[g] = m;
             };
-            d.get = function(a) {
-                return this[a];
+            e.get = function(g) {
+                return this[g];
             };
-            return d;
+            return e;
         }
-        function t(u) {
-            var e = Math.imul;
-            var f = Math.fround;
-            var g = Math.abs;
-            var h = Math.clz32;
-            var i = Math.min;
-            var j = Math.max;
-            var k = Math.floor;
-            var l = Math.ceil;
-            var m = Math.trunc;
-            var n = Math.sqrt;
-            var o = u.abort;
-            var p = NaN;
-            var q = Infinity;
-            var r = u.a;
-            function x(a, b) {
-                a = a | 0;
-                b = b | 0;
-                r(1024);
-                return 0;
-            }
-            function w() {}
-            var s = c([]);
+        return function(e) {
+            var g = e.a;
             return {
-                b: w,
-                c: x,
-                d: s
+                b: function() {},
+                c: function(m, q) {
+                    g(1024);
+                    return 0;
+                },
+                d: n([])
             };
-        }
-        return t(v);
+        }(a);
     }(k);
-}).exports;
-
-l = p.c;
+}).exports, l = p.c;
 
 d.set(new Uint8Array(b.mem), 1024);
 
diff --git a/tests/code_size/hello_world_wasm2js.json b/tests/code_size/hello_world_wasm2js.json
@@ -1,10 +1,10 @@
 {
   "a.html": 671,
   "a.html.gz": 430,
-  "a.js": 870,
-  "a.js.gz": 510,
+  "a.js": 604,
+  "a.js.gz": 381,
   "a.mem": 6,
   "a.mem.gz": 32,
-  "total": 1547,
-  "total_gz": 972
+  "total": 1281,
+  "total_gz": 843
 }
diff --git a/tests/code_size/math_wasm.json b/tests/code_size/math_wasm.json
@@ -1,10 +1,10 @@
 {
   "a.html": 673,
   "a.html.gz": 431,
-  "a.js": 137,
-  "a.js.gz": 141,
+  "a.js": 119,
+  "a.js.gz": 132,
   "a.wasm": 2721,
   "a.wasm.gz": 1635,
-  "total": 3531,
-  "total_gz": 2207
+  "total": 3513,
+  "total_gz": 2198
 }
diff --git a/tests/code_size/random_printf_wasm.json b/tests/code_size/random_printf_wasm.json
@@ -1,6 +1,6 @@
 {
-  "a.html": 12977,
-  "a.html.gz": 6991,
-  "total": 12977,
-  "total_gz": 6991
+  "a.html": 12857,
+  "a.html.gz": 6952,
+  "total": 12857,
+  "total_gz": 6952
 }
diff --git a/tests/code_size/random_printf_wasm2js.json b/tests/code_size/random_printf_wasm2js.json
@@ -1,6 +1,6 @@
 {
-  "a.html": 17804,
-  "a.html.gz": 7590,
-  "total": 17804,
-  "total_gz": 7590
+  "a.html": 17372,
+  "a.html.gz": 7503,
+  "total": 17372,
+  "total_gz": 7503
 }
diff --git a/tests/test_browser.py b/tests/test_browser.py
@@ -4716,12 +4716,13 @@ def test_pthread_hello_thread(self, opts):
     for modularize in [[], ['-sMODULARIZE', '-sEXPORT_NAME=MyModule', '--shell-file', test_file('shell_that_launches_modularize.html')]]:
       self.btest_exit(test_file('pthread/hello_thread.c'), args=['-sUSE_PTHREADS'] + modularize + opts)
 
-  # Tests that a pthreads build of -s MINIMAL_RUNTIME=1 works well in different build modes
+  # Tests that a pthreads build of -s MINIMAL_RUNTIME works well in different build modes
   @parameterized({
     '': ([],),
     'modularize': (['-sMODULARIZE', '-sEXPORT_NAME=MyModule'],),
     'O3': (['-O3'],),
     'O3_modularize': (['-O3', '-sMODULARIZE', '-sEXPORT_NAME=MyModule'],),
+    'O3_modularize_MINIMAL_RUNTIME_2': (['-O3', '-sMODULARIZE', '-sEXPORT_NAME=MyModule', '-sMINIMAL_RUNTIME=2'],),
   })
   def test_minimal_runtime_hello_thread(self, opts):
     self.btest_exit(test_file('pthread/hello_thread.c'), args=['--closure=1', '-sMINIMAL_RUNTIME', '-sUSE_PTHREADS'] + opts)
@@ -5021,7 +5022,8 @@ def test_no_declare_asm_module_exports_asmjs(self):
       self.btest(test_file('declare_asm_module_exports.cpp'), '1', args=['-sDECLARE_ASM_MODULE_EXPORTS=0', '-sENVIRONMENT=web', '-O3', '--closure=1', '-sWASM=0'] + minimal_runtime)
 
   def test_no_declare_asm_module_exports_wasm_minimal_runtime(self):
-    self.btest(test_file('declare_asm_module_exports.cpp'), '1', args=['-sDECLARE_ASM_MODULE_EXPORTS=0', '-sENVIRONMENT=web', '-O3', '--closure=1', '-sMINIMAL_RUNTIME'])
+    for mode in [1, 2]:
+      self.btest(test_file('declare_asm_module_exports.cpp'), '1', args=['-sDECLARE_ASM_MODULE_EXPORTS=0', '-sENVIRONMENT=web', '-O3', '--closure=1', f'-sMINIMAL_RUNTIME={mode}'])
 
   # Tests that the different code paths in src/shell_minimal_runtime.html all work ok.
   def test_minimal_runtime_loader_shell(self):
@@ -5088,6 +5090,9 @@ def test_system(self):
   def test_wasm_worker_hello(self):
     self.btest(test_file('wasm_worker/hello_wasm_worker.c'), expected='0', args=['-sWASM_WORKERS'])
 
+  def test_wasm_worker_hello_minimal_runtime_2(self):
+    self.btest(test_file('wasm_worker/hello_wasm_worker.c'), expected='0', args=['-sWASM_WORKERS', '-sMINIMAL_RUNTIME=2'])
+
   # Tests Wasm Workers build in Wasm2JS mode.
   @also_with_minimal_runtime
   def test_wasm_worker_hello_wasm2js(self):
diff --git a/tests/test_other.py b/tests/test_other.py
@@ -9140,13 +9140,6 @@ def test(args, closure, opt):
         test(['-sWASM=0'], closure, opt)
         test(['-sWASM_ASYNC_COMPILATION=0'], closure, opt)
 
-  # TODO: Debug why the code size is different on Windows and Mac. Also, for
-  # some unknown reason (at time of writing), this test is not skipped on the
-  # Windows and Mac autorollers, despite the bot being correctly configured to
-  # skip this test in all three platforms (Linux, Mac, and Windows).
-  # The no_windows/no_mac decorators also solve that problem.
-  @no_windows("Code size is slightly different on Windows")
-  @no_mac("Code size is slightly different on Mac")
   @parameterized({
     'hello_world_wasm': ('hello_world', False, True),
     'hello_world_wasm2js': ('hello_world', True, True),
@@ -11829,3 +11822,7 @@ def test_debug_opt_warning(self):
 
   def test_clock_nanosleep(self):
     self.do_runf(test_file('other/test_clock_nanosleep.c'))
+
+  # Tests the internal test suite of tools/unsafe_optimizations.js
+  def test_unsafe_optimizations(self):
+    self.run_process(config.NODE_JS + [path_from_root('tools', 'unsafe_optimizations.js'), '--test'])
diff --git a/tools/hacky_postprocess_around_closure_limitations.py b/tools/hacky_postprocess_around_closure_limitations.py
diff --git a/tools/unsafe_optimizations.js b/tools/unsafe_optimizations.js
