Merge pull request #306 from fitzgen/wasm

Use WebAssembly to speed up SourceMapConsumer

Author: fitzgen

.gitignore

@@ -2,3 +2,6 @@
 *.log
 .idea
 node_modules/*
+build/
+package-lock.json
+bench/*.svg

.travis.yml

@@ -3,11 +3,8 @@ language: node_js
 sudo: false
 
 node_js:
-  - "0.10"
-  - "0.12"
-  - "4"
-  - "5"
-  - "6"
+  - "8"
+  - "9"
 
 cache:
   directories:

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Change Log
 
+## 0.7.0
+
+* `SourceMapConsumer` now uses WebAssembly, and is **much** faster!
+
+* **Breaking change:** `new SourceMapConsumer` now returns a `Promise` object
+  that resolves to the newly constructed `SourceMapConsumer` instance, rather
+  than returning the new instance immediately.
+
+* **Breaking change:** when you're done using a `SourceMapConsumer` instance,
+  you must call `SourceMapConsumer.prototype.destroy` on it. After calling
+  `destroy`, you must not use the instance again.
+
+* **Breaking change:** `SourceMapConsumer` used to be able to handle lines,
+  columns numbers and source and name indices up to `2^53 - 1` (aka
+  `Number.MAX_SAFE_INTEGER`). It can now only handle them up to `2^32 - 1`.
+
+* **Breaking change:** The `source-map` library now uses modern ECMAScript-isms:
+  `let`, arrow functions, `async`, etc. Use Babel to compile it down to
+  ECMAScript 5 if you need to support older JavaScript environments.
+
 ## 0.5.6
 
 * Fix for regression when people were using numbers as names in source maps. See

CONTRIBUTING.md

@@ -16,6 +16,7 @@ very appreciated.
 - [Submitting Pull Requests](#submitting-pull-requests)
 - [Running Tests](#running-tests)
 - [Writing New Tests](#writing-new-tests)
+- [Updating the `lib/mappings.wasm` WebAssembly Module](#updating-the-libmappingswasm-webassembly-module)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -51,17 +52,6 @@ This will create the following files:
 
 * `dist/source-map.js` - The plain browser build.
 
-* `dist/source-map.min.js` - The minified browser build.
-
-* `dist/source-map.min.js.map` - The source map for the minified browser build.
-
-* `dist/source-map.debug.js` - The debug browser build.
-
-* `dist/source-map.debug.js.map` - The source map for the debug browser build.
-
-* `dist/test/*` - These are the test files built for running as xpcshell unit
-  tests within mozilla-central.
-
 ## Submitting Pull Requests
 
 Make sure that tests pass locally before creating a pull request.
@@ -122,3 +112,51 @@ can use as well:
 ```js
 var util = require('./util');
 ```
+
+## Updating the `lib/mappings.wasm` WebAssembly Module
+
+Ensure that you have the Rust toolchain installed:
+
+```
+$ curl https://sh.rustup.rs -sSf | sh
+```
+
+The `wasm32-unknown-unknown` target is nightly-only at the time of writing. Use
+`rustup` to ensure you have it installed:
+
+```
+$ rustup toolchain install nightly
+$ rustup target add wasm32-unknown-unknown --toolchain nightly
+```
+
+Next, clone the Rust source used to create `lib/mappings.wasm`:
+
+```
+$ git clone https://github.com/fitzgen/source-map-mappings.git
+$ cd source-map-mappings/
+```
+
+Make sure the crate's tests pass:
+
+```
+$ cargo test
+```
+
+Build Rust crate as a `.wasm` file:
+
+```
+$ cd source-map-mappings-wasm-api/
+$ cargo build --release --target wasm32-unknown-unknown
+```
+
+The resulting `wasm` file will be located at
+`source-map-mappings-c-api/target/wasm32-unknown-unknown/release/source_map_mappings.wasm`.
+
+Finally, to minimize its size, run `wasm-gc` on it and output the minimified
+`.wasm` file into this library's `lib/mappings.wasm`:
+
+```
+$ cargo install --git https://github.com/alexcrichton/wasm-gc # If you don't already have it.
+$ wasm-gc target/wasm32-unknown-unknown/release/source_map_mappings.wasm \\
+    /path/to/mozilla/source-map/lib/mappings.wasm
+```

README.md

@@ -32,7 +32,9 @@ This is a library to generate and consume the source map format
     - [With SourceMapGenerator (low level API)](#with-sourcemapgenerator-low-level-api)
 - [API](#api)
   - [SourceMapConsumer](#sourcemapconsumer)
+    - [SourceMapConsumer.initialize(options)](#sourcemapconsumerinitializeoptions)
     - [new SourceMapConsumer(rawSourceMap)](#new-sourcemapconsumerrawsourcemap)
+    - [SourceMapConsumer.prototype.destroy()](#sourcemapconsumerprototypedestroy)
     - [SourceMapConsumer.prototype.computeColumnSpans()](#sourcemapconsumerprototypecomputecolumnspans)
     - [SourceMapConsumer.prototype.originalPositionFor(generatedPosition)](#sourcemapconsumerprototypeoriginalpositionforgeneratedposition)
     - [SourceMapConsumer.prototype.generatedPositionFor(originalPosition)](#sourcemapconsumerprototypegeneratedpositionfororiginalposition)
@@ -67,7 +69,7 @@ This is a library to generate and consume the source map format
 ### Consuming a source map
 
 ```js
-var rawSourceMap = {
+const rawSourceMap = {
   version: 3,
   file: 'min.js',
   names: ['bar', 'baz', 'n'],
@@ -76,7 +78,7 @@ var rawSourceMap = {
   mappings: 'CAAC,IAAI,IAAM,SAAUA,GAClB,OAAOC,IAAID;CCDb,IAAI,IAAM,SAAUE,GAClB,OAAOA'
 };
 
-var smc = new SourceMapConsumer(rawSourceMap);
+const smc = await new SourceMapConsumer(rawSourceMap);
 
 console.log(smc.sources);
 // [ 'http://example.com/www/js/one.js',
@@ -101,6 +103,9 @@ console.log(smc.generatedPositionFor({
 smc.eachMapping(function (m) {
   // ...
 });
+
+// Free the SourceMapConsumer's manually managed wasm data.
+smc.destroy();
 ```
 
 ### Generating a source map
@@ -182,10 +187,27 @@ const sourceMap = require("devtools/toolkit/sourcemap/source-map.js");
 
 ### SourceMapConsumer
 
-A SourceMapConsumer instance represents a parsed source map which we can query
+A `SourceMapConsumer` instance represents a parsed source map which we can query
 for information about the original file positions by giving it a file position
 in the generated source.
 
+#### SourceMapConsumer.initialize(options)
+
+When using `SourceMapConsumer` outside of node.js, for example on the Web, it
+needs to know from what URL to load `lib/mappings.wasm`. You must inform it by
+calling `initialize` before constructing any `SourceMapConsumer`s.
+
+The options object has the following properties:
+
+* `"lib/mappings.wasm"`: A `String` containing the URL of the
+  `lib/mappings.wasm` file.
+
+```js
+sourceMap.SourceMapConsumer.initialize({
+  "lib/mappings.wasm": "https://example.com/source-map/lib/mappings.wasm"
+});
+```
+
 #### new SourceMapConsumer(rawSourceMap)
 
 The only parameter is the raw source map (either as a string which can be
@@ -207,8 +229,23 @@ following attributes:
 
 * `file`: Optional. The generated filename this source map is associated with.
 
+The promise of the constructed souce map consumer is returned.
+
+When the `SourceMapConsumer` will no longer be used anymore, you must call its
+`destroy` method.
+
+```js
+const consumer = await new sourceMap.SourceMapConsumer(rawSourceMapJsonData);
+doStuffWith(consumer);
+consumer.destroy();
+```
+
+#### SourceMapConsumer.prototype.destroy()
+
+Free this source map consumer's associated wasm data that is manually-managed.
+
 ```js
-var consumer = new sourceMap.SourceMapConsumer(rawSourceMapJsonData);
+consumer.destroy();
 ```
 
 #### SourceMapConsumer.prototype.computeColumnSpans()
@@ -239,7 +276,6 @@ consumer.allGeneratedPositionsFor({ line: 2, source: "foo.coffee" })
 //   { line: 2,
 //     column: 20,
 //     lastColumn: Infinity } ]
-
 ```
 
 #### SourceMapConsumer.prototype.originalPositionFor(generatedPosition)
@@ -579,9 +615,9 @@ Creates a SourceNode from generated code and a SourceMapConsumer.
   should be relative to.
 
 ```js
-var consumer = new SourceMapConsumer(fs.readFileSync("path/to/my-file.js.map", "utf8"));
-var node = SourceNode.fromStringWithSourceMap(fs.readFileSync("path/to/my-file.js"),
-                                              consumer);
+const consumer = await new SourceMapConsumer(fs.readFileSync("path/to/my-file.js.map", "utf8"));
+onst node = SourceNode.fromStringWithSourceMap(fs.readFileSync("path/to/my-file.js"),
+                                               consumer);
 ```
 
 #### SourceNode.prototype.add(chunk)

bench/.gitignore

@@ -0,0 +1 @@
+*.csv

bench/README.md

@@ -1,15 +1,20 @@
 # Benchmarking
 
-This directory contains helpers for benchmarking (1) parsing, and (2) generating
-source maps.
+This directory contains helpers for benchmarking the `mozilla/source-map`
+library.
 
 Ensure that you have built the library, as these benchmarks rely on
 `dist/source-map.js`. See the main README.md for detais on building.
 
-## Running Within a Browser
+Run a local webserver from the root of the repository:
 
-Open `bench.html` in a browser and click on the appropriate button.
+```
+$ cd source-map/
+$ python -m SimpleHTTPServer
+```
 
-## Running with a JS Shell
+Open
+[http://localhost:8000/bench/bench.html](http://localhost:8000/bench/bench.html)
+in your browser.
 
-Run `$JS_SHELL bench-shell-bindings.js`.
+Open `bench.html` in a browser and click on the appropriate button.

bench/angular-min-source-map.js

@@ -1,9 +1,94 @@
+sourceMap.SourceMapConsumer.initialize({
+  "lib/mappings.wasm": "../lib/mappings.wasm",
+});
+
+function bindRange(labelId, updater) {
+  const label = document.getElementById(labelId);
+  const input = label.querySelector("input");
+
+  input.addEventListener("input", e => {
+    e.preventDefault();
+    updater(input.value);
+  });
+
+  updater(input.value);
+}
+
+bindRange("warm-up-iters", input => {
+  const value = parseInt(input, 10);
+  WARM_UP_ITERATIONS = value;
+});
+
+bindRange("bench-iters", input => {
+  const value = parseInt(input, 10);
+  BENCH_ITERATIONS = value;
+});
+
+const whichMap = document.getElementById("input-map");
+const multiplyBy = document.getElementById("multiply-size-by");
+
+var testSourceMap = SCALA_JS_RUNTIME_SOURCE_MAP;
+
+const updateTestSourceMap = () => {
+  const origMap = window[whichMap.value];
+  testSourceMap = JSON.parse(JSON.stringify(origMap));
+
+  const factor = parseInt(multiplyBy.value, 10);
+  if (factor === 1) {
+    return;
+  }
+
+  const mappings = new Array(factor);
+  mappings.fill(origMap.mappings);
+  testSourceMap.mappings = mappings.join(";");
+
+  for (let i = 0; i < factor; i++) {
+    testSourceMap.sources.splice(testSourceMap.sources.length, 0, ...origMap.sources);
+    testSourceMap.names.splice(testSourceMap.names.length, 0, ...origMap.names);
+  }
+};
+updateTestSourceMap();
+
+whichMap.addEventListener("input", e => {
+  e.preventDefault();
+  updateTestSourceMap();
+});
+
+multiplyBy.addEventListener("input", e => {
+  e.preventDefault();
+  updateTestSourceMap();
+});
+
+var implAndBrowser = "<unknown>";
+
+const implAndBrowserInput = document.getElementById("impl-and-browser");
+const updateImplAndBrowser = () => {
+  implAndBrowser = implAndBrowserInput.value;
+};
+implAndBrowserInput.addEventListener("input", updateImplAndBrowser);
+updateImplAndBrowser();
+
+
 // Run a benchmark when the given button is clicked and display results in the
 // given element.
-function benchOnClick(button, results, bencher) {
-  button.addEventListener("click", function (e) {
+function benchOnClick(button, results, benchName, bencher) {
+  button.addEventListener("click", async function (e) {
     e.preventDefault();
-    var stats = bencher();
+
+    const buttons = [...document.querySelectorAll("button")];
+    buttons.forEach(b => b.setAttribute("disabled", true));
+    results.innerHTML = "";
+    await new Promise(r => requestAnimationFrame(r));
+
+    var stats = await bencher();
+
+    buttons.forEach(b => b.removeAttribute("disabled"));
+
+    const csv = stats
+          .xs
+          .map(x => `"${implAndBrowser}",${testSourceMap.mappings.length},"${benchName}",${x}`)
+          .join("\n");
+
     results.innerHTML = `
       <table>
         <thead>
@@ -23,14 +108,21 @@ function benchOnClick(button, results, bencher) {
           </tr>
         </tbody>
       </table>
+      <pre style="overflow:scroll;max-height:100px; max-width:500px;outline:1px solid black">${csv}</pre>
     `;
   }, false);
 }
 
-benchOnClick(document.getElementById("bench-consumer"),
-             document.getElementById("consumer-results"),
-             benchmarkParseSourceMap);
+for (let bench of Object.keys(benchmarks)) {
+  const hr = document.createElement("hr");
+  document.body.appendChild(hr);
 
-benchOnClick(document.getElementById("bench-generator"),
-             document.getElementById("generator-results"),
-             benchmarkSerializeSourceMap);
+  const button = document.createElement("button");
+  button.innerHTML = `<h2>${bench}</h2>`;
+  document.body.appendChild(button);
+
+  const results = document.createElement("div");
+  document.body.appendChild(results);
+
+  benchOnClick(button, results, bench, benchmarks[bench]);
+}