Merge pull request #499 from antfu/feat/fetch-reader

Universal reader

Author: LingDong-

package-lock.json

@@ -68,5 +68,8 @@
     "webpack": "^4.41.4",
     "webpack-cli": "^3.3.10",
     "webpack-shell-plugin": "^0.5.0"
+  },
+  "dependencies": {
+    "xmlhttprequest": "^1.8.0"
   }
 }

package.json

@@ -9,16 +9,26 @@
       script.attributes.outputHanzi &&
       script.attributes.outputHanzi.value === "false"
     );
+    const allowHttp = !!script.attributes.allowHttp;
     let code = script.innerText;
+    let importPaths = [window.location.origin];
     if (script.src) {
       const response = await fetch(script.src);
       code = (await response.text()).toString();
+      importPaths.push(
+        script.src
+          .split("/")
+          .slice(0, -1)
+          .join("/")
+      );
     }
     execute(code, {
       scoped,
       outputHanzi,
       logCallback: isDev ? console.log : () => {},
-      resetVarCnt: false
+      resetVarCnt: false,
+      importPaths,
+      allowHttp
     });
   }
 

src/browser_runtime.js

@@ -38,6 +38,9 @@ program
     "--roman [method]",
     'Romanize identifiers. The method can be "pinyin", "baxter" or "unicode"'
   )
+  .option("--strict", "Enable static typechecking")
+  .option("--allowHttp", "Allow to import from http")
+  .option("--dir <path>", "Directory to importing from, seprates with comma(,)")
   .option("--outputHanzi", "Convert output to hanzi", true)
   .option("--log <file>", "Save log to file")
   .option("--title <title>", "Override title in rendering")
@@ -61,17 +64,21 @@ if (emptyCall || showHelp) {
 
 program.parse(process.argv);
 
-preprocess();
+(async () => {
+  preprocess();
 
-if (program.compile) {
-  output(getCompiled());
-} else if (program.render) {
-  doRender();
-} else if (program.interactive) {
-  intreactive();
-} else {
-  exec();
-}
+  if (program.compile) {
+    output(await getCompiled());
+  } else if (program.render) {
+    doRender();
+  } else if (program.interactive) {
+    await intreactive();
+  } else {
+    await exec();
+  }
+})().catch(e => {
+  console.error(e);
+});
 
 // ====== Utils ======
 
@@ -103,14 +110,35 @@ function preprocess() {
 
 function getCompiled() {
   const source = getSource();
-  return compile(program.lang, source, {
+
+  return compile(source, {
+    ...getCompileOptions()
+  });
+}
+
+function getImportPaths() {
+  const pathes = [];
+  if (program.dir) {
+    pathes.push(...program.dir.split(","));
+  }
+  pathes.push(...program.files.map(file => path.resolve(path.dirname(file))));
+  pathes.push(path.resolve("."));
+  return Array.from(new Set(pathes));
+}
+
+function getCompileOptions() {
+  return {
+    lang: program.lang,
     romanizeIdentifiers: program.roman,
+    strict: !!program.strict,
+    allowHttp: !!program.allowHttp,
+    importPaths: getImportPaths(),
     logCallback: logHandler(program.log, "a"),
     errorCallback: function(x) {
       console.error(x);
       process.exit();
     }
-  });
+  };
 }
 
 function resolvePath(x) {
@@ -125,6 +153,7 @@ function getSource() {
         : fs.readFileSync(resolvePath(x)).toString()
     )
     .join("\n");
+
   if (program.eval) scripts += `\n${program.eval}`;
 
   return scripts;
@@ -176,25 +205,25 @@ function doRender() {
   }
 }
 
-function intreactive() {
+async function intreactive() {
   if (program.lang !== "js") {
     console.error(
       `Target language "${program.lang}" is not supported for intreactive mode.`
     );
     process.exit(1);
   }
   replscope();
-  repl(getCompiled());
+  repl(await getCompiled());
 }
-function exec() {
+async function exec() {
   if (program.lang !== "js") {
     console.error(
       `Target language "${program.lang}" is not supported for direct executing. Please use --compile option instead.`
     );
     process.exit(1);
   }
 
-  evalCompiled(getCompiled(), {
+  evalCompiled(await getCompiled(), {
     outputHanzi: program.outputHanzi,
     lang: program.lang
   });

src/cli.js

@@ -1,4 +1,6 @@
-function extractMacros(txt, { lib, reader, lang }) {
+function extractMacros(txt, options = {}) {
+  const { lib, reader, lang, importPaths, requestOptions } = options;
+
   function getImports() {
     var imps = [];
     var qlvl = 0;
@@ -119,9 +121,9 @@ function extractMacros(txt, { lib, reader, lang }) {
     } else if (imports[i] in lib) {
       isrc = lib[imports[i]];
     } else {
-      isrc = reader(imports[i]);
+      isrc = reader(imports[i], importPaths, requestOptions);
     }
-    macros = macros.concat(extractMacros(isrc, { lib, reader, lang }));
+    macros = macros.concat(extractMacros(isrc, options));
   }
   return macros;
 }

src/macro.js

@@ -6,6 +6,11 @@ var version = require("./version");
 var compilers = require("./compiler/compilers");
 var { typecheck, printSignature } = require("./typecheck");
 var { expandMacros, extractMacros } = require("./macro.js");
+var { defaultImportReader } = require("./reader");
+
+const defaultTrustedHosts = [
+  "https://raw.githubusercontent.com/LingDong-/wenyan-lang/master"
+];
 
 function wy2tokens(
   txt,
@@ -213,6 +218,12 @@ function tokenRomanize(tokens, method) {
   }
 }
 
+function defaultLogCallback(x) {
+  return typeof x == "string"
+    ? console.log(x)
+    : console.dir(x, { depth: null, maxArrayLength: null });
+}
+
 function tokens2asc(
   tokens,
   assert = (msg, pos, b) => {
@@ -624,29 +635,6 @@ function pyWrapModule(name, src) {
   return `#/*___wenyan_module_${name}_start___*/\n${src}\n#/*___wenyan_module_${name}_end___*/\n`;
 }
 
-function defaultReader(x) {
-  try {
-    const fs = eval("require")("fs");
-    try {
-      return fs.readFileSync(x + ".wy").toString();
-    } catch (e) {
-      var files = fs.readdirSync("./");
-      for (var i = 0; i < files.length; i++) {
-        if (fs.lstatSync(files[i]).isDirectory()) {
-          try {
-            return fs.readFileSync(files[i] + "/" + x + ".wy").toString();
-          } catch (e) {}
-        }
-      }
-    }
-    console.log("Cannot import ", x);
-  } catch (e) {
-    console.error(
-      `Cannot import ${x}, please specify the "reader" option in compile.`
-    );
-  }
-}
-
 function compile(arg1, arg2, arg3) {
   let options = {};
   let txt = "";
@@ -663,17 +651,26 @@ function compile(arg1, arg2, arg3) {
   const {
     lang = "js",
     romanizeIdentifiers = "none",
-    resetVarCnt,
-    logCallback = x =>
-      typeof x == "string"
-        ? console.log(x)
-        : console.dir(x, { depth: null, maxArrayLength: null }),
+    resetVarCnt = true,
+    logCallback = defaultLogCallback,
     errorCallback = process.exit,
     lib = typeof STDLIB == "undefined" ? {} : STDLIB,
-    reader = defaultReader,
-    strict = false
+    reader = defaultImportReader,
+    importPaths = [],
+    strict = false,
+    allowHttp = false,
+    trustedHosts = [],
+    requestTimeout = 2000
   } = options;
 
+  trustedHosts.push(...defaultTrustedHosts);
+
+  const requestOptions = {
+    allowHttp,
+    trustedHosts,
+    requestTimeout
+  };
+
   if (resetVarCnt) idenMap = {};
   txt = (txt || "").replace(/\r\n/g, "\n");
 
@@ -703,7 +700,13 @@ function compile(arg1, arg2, arg3) {
     return 0;
   }
 
-  var macros = extractMacros(txt, { lib, reader, lang });
+  var macros = extractMacros(txt, {
+    lib,
+    reader,
+    lang,
+    importPaths,
+    requestOptions
+  });
   txt = expandMacros(txt, macros);
 
   logCallback("\n\n=== [PASS 0] EXPAND-MACROS ===");
@@ -751,19 +754,15 @@ function compile(arg1, arg2, arg3) {
     } else if (imports[i] in lib) {
       isrc = lib[imports[i]];
     } else {
-      isrc = reader(imports[i]);
+      isrc = reader(imports[i], importPaths, requestOptions);
     }
     targ =
       mwrapper(
         imports[i],
         compile(isrc, {
-          lang,
-          romanizeIdentifiers,
+          ...options,
           resetVarCnt: false,
-          strict: false,
-          logCallback,
-          errorCallback,
-          lib
+          strict: false
         })
       ) + targ;
   }

src/parser.js

@@ -0,0 +1,71 @@
+function isHostTrusted(url, trustedHosts) {
+  for (const host of trustedHosts) {
+    // FIXME: it can be bypassed by relative path resolving,
+    // for examples: https://trusted.com/a/../../hijack.com/a/
+    if (url.startsWith(host)) return true;
+  }
+  return false;
+}
+
+function isHttpURL(uri) {
+  return !!uri.match(/^https?\:\/\//);
+}
+
+function fetchTextSync(url, timeout) {
+  let XHR;
+  if (typeof window !== "undefined" && "XMLHttpRequest" in window)
+    XHR = window.XMLHttpRequest;
+  else XHR = eval("require")("xmlhttprequest").XMLHttpRequest;
+
+  var xmlHttp = new XHR();
+  xmlHttp.timeout = timeout;
+  xmlHttp.open("GET", url, false); // false for synchronous request
+  xmlHttp.send(null);
+
+  if (xmlHttp.status >= 200 && xmlHttp.status < 300)
+    return xmlHttp.responseText;
+
+  throw new URIError(xmlHttp.responseText);
+}
+
+function defaultImportReader(
+  moduleName,
+  importPaths = [],
+  requestOptions = {}
+) {
+  const {
+    allowHttp = false,
+    trustedHosts = [],
+    requestTimeout = 2000
+  } = requestOptions;
+
+  if (typeof importPaths === "string") importPaths = [importPaths];
+
+  for (dir of importPaths) {
+    const uri = dir + "/" + moduleName + ".wy";
+    if (isHttpURL(uri)) {
+      if (!allowHttp && !isHostTrusted(uri, trustedHosts)) {
+        throw new URIError(
+          `URL request "${uri}" is blocked by default for security purpose. ` +
+            `You can turn it on by specify the "allowHttp" option.`
+        );
+      }
+
+      try {
+        return fetchTextSync(uri, requestTimeout);
+      } catch (e) {}
+    } else {
+      try {
+        return eval("require")("fs").readFileSync(uri, "utf-8");
+      } catch (e) {}
+    }
+  }
+
+  throw new ReferenceError(
+    `Module "${moduleName}" is not found. Searched in ${importPaths}`
+  );
+}
+
+module.exports = {
+  defaultImportReader
+};