set up utility for compiling in webworker, tweak webpack config, and re-organize VersionRange strategy

eggplantzzz · eggplantzzz · commit cdd6c2ef5429 · 2022-12-06T10:41:29.000-05:00
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -9,7 +9,8 @@
     "BUNDLE_CHAIN_FILENAME": "readonly",
     "BUNDLE_ANALYTICS_FILENAME": "readonly",
     "BUNDLE_LIBRARY_FILENAME": "readonly",
-    "BUNDLE_CONSOLE_CHILD_FILENAME": "readonly"
+    "BUNDLE_CONSOLE_CHILD_FILENAME": "readonly",
+    "WORKER_CODE_BASE64": "readonly"
   },
   "extends": [
     "plugin:react-hooks/recommended"
diff --git a/packages/compile-solidity/.gitignore b/packages/compile-solidity/.gitignore
@@ -1,4 +1,5 @@
 dist
+src/compileInWebWorker/bundledWorker.js
 node_modules
 yarn.lock
 package-lock.json
diff --git a/packages/compile-solidity/package.json b/packages/compile-solidity/package.json
@@ -46,13 +46,19 @@
     "babel-preset-env": "^1.6.1",
     "chai": "^4.2.0",
     "glob": "^7.1.6",
+    "https-browserify": "^1.0.0",
     "mocha": "10.1.0",
+    "process": "^0.11.10",
     "sinon": "^9.0.2",
+    "stream-browserify": "^3.0.0",
+    "stream-http": "^3.2.0",
     "tmp": "^0.2.1",
     "ts-node": "10.7.0",
     "ttypescript": "1.5.13",
     "typescript": "^4.7.4",
-    "typescript-transform-paths": "3.3.1"
+    "typescript-transform-paths": "3.3.1",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0"
   },
   "keywords": [
     "compile",
diff --git a/packages/compile-solidity/src/compileInWebWorker/index.ts b/packages/compile-solidity/src/compileInWebWorker/index.ts
@@ -0,0 +1,25 @@
+/* eslint-env browser */
+export async function compileInWebWorker({
+  soljson,
+  compilerInput
+}): Promise<any> {
+  // @ts-ignore
+  const dataUrl = `data:text/javascript;base64,${WORKER_CODE_BASE64}`;
+  const worker = new Worker(dataUrl);
+  return new Promise(resolve => {
+    worker.addEventListener(
+      "message",
+      function (e) {
+        const output = e.data;
+        resolve({
+          compilerOutput: output,
+          // TODO: figure out where the version information is
+          solcVersion: ""
+        });
+      },
+      false
+    );
+
+    worker.postMessage({ soljson, compilerInput });
+  });
+}
diff --git a/packages/compile-solidity/src/compileInWebWorker/worker.ts b/packages/compile-solidity/src/compileInWebWorker/worker.ts
@@ -0,0 +1,13 @@
+/* eslint-env worker */
+import solcWrap from "solc/wrapper";
+
+self.addEventListener(
+  "message",
+  e => {
+    const { soljson, compilerInput } = e.data;
+    const solc = solcWrap(eval(soljson + "; Module;"));
+    const output = JSON.parse(solc.compile(JSON.stringify(compilerInput)));
+    self.postMessage(output);
+  },
+  false
+);
diff --git a/packages/compile-solidity/src/compilerSupplier/index.ts b/packages/compile-solidity/src/compilerSupplier/index.ts
@@ -43,7 +43,7 @@ export class CompilerSupplier {
     if (cache) this.strategyOptions.cache = cache;
   }
 
-  async load() {
+  getStrategy() {
     const userSpecification = this.version;
 
     let strategy: CompilerSupplierStrategy;
@@ -53,7 +53,8 @@ export class CompilerSupplier {
     if (!userSpecification) {
       useSpecifiedLocal =
         userSpecification &&
-        (fs.existsSync(userSpecification) || path.isAbsolute(userSpecification));
+        (fs.existsSync(userSpecification) ||
+          path.isAbsolute(userSpecification));
     }
     const isValidVersionRange = semver.validRange(userSpecification);
 
@@ -66,7 +67,24 @@ export class CompilerSupplier {
     } else if (isValidVersionRange) {
       strategy = new VersionRange(this.strategyOptions);
     }
+    return {
+      strategy,
+      userSpecification
+    };
+  }
 
+  async loadSoljson() {
+    const { strategy, userSpecification } = this.getStrategy();
+    if (strategy) {
+      const soljson = await strategy.loadSoljson(userSpecification);
+      return { soljson };
+    } else {
+      throw new BadInputError(userSpecification);
+    }
+  }
+
+  async load() {
+    const { strategy, userSpecification } = this.getStrategy();
     if (strategy) {
       const solc = await strategy.load(userSpecification);
       return { solc };
diff --git a/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Docker.ts b/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Docker.ts
@@ -26,6 +26,10 @@ export class Docker {
     this.cache = new Cache();
   }
 
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Docker strategy.");
+  }
+
   async load() {
     // Set a sensible limit for maxBuffer
     // See https://github.com/nodejs/node/pull/23027
diff --git a/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Local.ts b/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Local.ts
@@ -4,6 +4,10 @@ import solcWrap from "solc/wrapper";
 import { observeListeners } from "../observeListeners";
 
 export class Local {
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Local strategy.");
+  }
+
   load(localPath: string) {
     const listeners = observeListeners();
     try {
diff --git a/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Native.ts b/packages/compile-solidity/src/compilerSupplier/loadingStrategies/Native.ts
@@ -3,6 +3,10 @@ const { normalizeSolcVersion } = require("../normalizeSolcVersion");
 const { NoVersionError } = require("../errors");
 
 export class Native {
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Native strategy.");
+  }
+
   load() {
     const versionString = this.validateAndGetSolcVersion();
     const command = "solc --standard-json";
diff --git a/packages/compile-solidity/src/compilerSupplier/loadingStrategies/VersionRange.ts b/packages/compile-solidity/src/compilerSupplier/loadingStrategies/VersionRange.ts
@@ -53,7 +53,7 @@ export class VersionRange {
     }
   }
 
-  async load(versionRange: string) {
+  async loadSoljson(versionRange: string) {
     const rangeIsSingleVersion = semver.valid(versionRange);
     if (
       rangeIsSingleVersion &&
@@ -73,6 +73,11 @@ export class VersionRange {
     }
   }
 
+  async load(versionRange: string) {
+    const soljson = await this.loadSoljson(versionRange);
+    return await this.compilerFromString(soljson);
+  }
+
   async list(index = 0) {
     if (index >= this.config.compilerRoots!.length) {
       throw new Error(
@@ -109,7 +114,7 @@ export class VersionRange {
     };
   }
 
-  compilerFromString(code: string) {
+  async compilerFromString(code: string) {
     const listeners = observeListeners();
     try {
       const soljson = requireFromString(code);
@@ -190,7 +195,7 @@ export class VersionRange {
     throw new NoVersionError(versionRange);
   }
 
-  async getAndCacheSolcByUrl(fileName: string, index: number) {
+  async getAndCacheSoljsonByUrl(fileName: string, index: number) {
     const { events, compilerRoots } = this.config;
     const url = `${compilerRoots![index].replace(/\/+$/, "")}/${fileName}`;
     events.emit("downloadCompiler:start", {
@@ -207,7 +212,7 @@ export class VersionRange {
     if (this.cache) {
       this.cache.add(response.data, fileName);
     }
-    return this.compilerFromString(response.data);
+    return response.data;
   }
 
   async getSolcFromCacheOrUrl(versionConstraint: string, index: number = 0) {
@@ -246,8 +251,9 @@ export class VersionRange {
       if (this.cache?.has(fileName)) {
         return this.getCachedSolcByFileName(fileName);
       }
-      return await this.getAndCacheSolcByUrl(fileName, index);
+      return await this.getAndCacheSoljsonByUrl(fileName, index);
     } catch (error) {
+      debug("there was an error fetching soljson -- %o", error.message);
       const attemptNumber = index + 1;
       return await this.getSolcFromCacheOrUrl(versionConstraint, attemptNumber);
     }
@@ -264,6 +270,10 @@ export class VersionRange {
     const url = `${urlRoot.replace(/\/+$/, "")}/list.json`;
     try {
       const response = await axios.get(url, { maxRedirects: 50 });
+      debug(
+        `obtained the following version list from ${url} -- %o`,
+        response.data
+      );
       return response.data;
     } catch (error) {
       events.emit("fetchSolcList:fail");
@@ -292,8 +302,13 @@ export class VersionRange {
     }
 
     const versionToUse = this.findNewestValidVersion(version, allVersions);
-
-    if (versionToUse) return allVersions.releases[versionToUse];
+    if (versionToUse) {
+      debug(
+        "the solc filename that we will use -- %o",
+        allVersions.releases[versionToUse]
+      );
+      return allVersions.releases[versionToUse];
+    }
 
     return null;
   }
diff --git a/packages/compile-solidity/src/run.ts b/packages/compile-solidity/src/run.ts
@@ -1,7 +1,7 @@
 import { zeroLinkReferences, formatLinkReferences } from "./shims";
 import debugModule from "debug";
 const debug = debugModule("compile:run");
-import OS = require("os");
+import OS from "os";
 import semver from "semver";
 import { CompilerSupplier } from "./compilerSupplier";
 import * as Common from "@truffle/compile-common";
@@ -22,6 +22,7 @@ import type {
   Targets
 } from "./types";
 import type Config from "@truffle/config";
+import { compileInWebWorker } from "./compileInWebWorker";
 
 // this function returns a Compilation - legacy/index.js and ./index.js
 // both check to make sure rawSources exist before calling this method
@@ -247,6 +248,18 @@ async function invokeCompiler({ compilerInput, options, solc }): Promise<{
     cache: options.compilers.cache
   };
 
+  // in the browser, compile in a worker and return the result
+  // @ts-ignore
+  if (window !== undefined) {
+    const supplier = new CompilerSupplier(supplierOptions);
+    const { soljson } = await supplier.loadSoljson();
+    debug(
+      "about to compile in a web worker with the following compilerInput -- %o",
+      compilerInput
+    );
+    return await compileInWebWorker({ soljson, compilerInput });
+  }
+
   if (!solc) {
     const supplier = new CompilerSupplier(supplierOptions);
     ({ solc } = await supplier.load());
diff --git a/packages/compile-solidity/tsconfig.json b/packages/compile-solidity/tsconfig.json
@@ -35,6 +35,7 @@
   ],
   "exclude": [
     "dist",
-    "node_modules"
+    "node_modules",
+    "src/compileInWebWorker/bundledWorker.js"
   ]
 }
diff --git a/packages/compile-solidity/webpack.config.js b/packages/compile-solidity/webpack.config.js
@@ -0,0 +1,35 @@
+const path = require("path");
+const webpack = require("webpack");
+
+module.exports = {
+  entry: "./src/compileInWebWorker/worker.ts",
+  mode: "development",
+  devtool: "inline-source-map",
+  target: "webworker",
+  resolve: {
+    extensions: [".tsx", ".ts", ".js", ".json"],
+    fallback: {
+      http: require.resolve("stream-http"),
+      https: require.resolve("https-browserify"),
+      stream: require.resolve("stream-browserify")
+    }
+  },
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: "ts-loader",
+        exclude: /node_modules/
+      }
+    ]
+  },
+  output: {
+    filename: "bundledWorker.js",
+    path: path.resolve(__dirname, "dist", "compileInWebWorker")
+  },
+  plugins: [
+    new webpack.ProvidePlugin({
+      process: "process/browser"
+    })
+  ]
+};
diff --git a/packages/fetch-and-compile/lib/index.ts b/packages/fetch-and-compile/lib/index.ts
@@ -1,6 +1,6 @@
 import debugModule from "debug";
 const debug = debugModule("fetch-and-compile");
-import type Config from "@truffle/config";
+import Config from "@truffle/config";
 import { SingleRecognizer } from "./recognizer";
 import { MultipleRecognizer } from "./multiple";
 import { DebugRecognizer } from "./debug";
diff --git a/packages/fetch-and-compile/webpack.config.js b/packages/fetch-and-compile/webpack.config.js
@@ -1,5 +1,6 @@
 const path = require("path");
 const webpack = require("webpack");
+const fs = require("fs");
 
 module.exports = {
   entry: "./lib/index.ts",
@@ -32,7 +33,6 @@ module.exports = {
       child_process: require.resolve("./emptyShim"),
       constants: require.resolve("./emptyShim"),
       readline: require.resolve("./emptyShim"),
-      process: require.resolve("./emptyShim"),
       os: require.resolve("os-browserify/browser"),
       path: require.resolve("path-browserify"),
       stream: require.resolve("stream-browserify"),
@@ -45,6 +45,14 @@ module.exports = {
     path: path.resolve(__dirname, "browser-dist")
   },
   plugins: [
+    new webpack.DefinePlugin({
+      WORKER_CODE_BASE64: `"${Buffer.from(
+        fs.readFileSync(
+          "../compile-solidity/dist/compileInWebWorker/bundledWorker.js",
+          { encoding: "utf8" }
+        )
+      ).toString("base64")}"`
+    }),
     new webpack.ProvidePlugin({
       process: "process/browser"
     }),
