Fix CommonJS bundling (#1012)

* Fix CommonJS bundling

* Add bundling unit tests

Author: xenova

src/env.js

@@ -73,9 +73,20 @@ export const apis = Object.freeze({
 });
 
 const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
-const dirname__ = RUNNING_LOCALLY
-    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))
-    : './';
+
+let dirname__ = './';
+if (RUNNING_LOCALLY) {
+    // NOTE: We wrap `import.meta` in a call to `Object` to prevent Webpack from trying to bundle it in CommonJS.
+    // Although we get the warning: "Accessing import.meta directly is unsupported (only property access or destructuring is supported)",
+    // it is safe to ignore since the bundled value (`{}`) isn't used for CommonJS environments (we use __dirname instead).
+    const _import_meta_url = Object(import.meta).url;
+
+    if (_import_meta_url) {
+        dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))) // ESM
+    } else if (typeof __dirname !== 'undefined') {
+        dirname__ = path.dirname(__dirname) // CommonJS
+    }
+}
 
 // Only used for environments with access to file system
 const DEFAULT_CACHE_DIR = RUNNING_LOCALLY

tests/bundles.test.js

@@ -0,0 +1,37 @@
+import { spawnSync } from "child_process";
+
+const MODULE_NAME = "@huggingface/transformers";
+
+const CODE_BODY = `
+const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM";
+const generator = await pipeline("text-generation", model_id, { dtype: "fp32" });
+const result = await generator("hello", { max_new_tokens: 3, return_full_text: false });
+process.stdout.write(result[0].generated_text);
+`;
+
+const TARGET_OUTPUT = "erdingsAndroid Load";
+
+const wrap_async_iife = (code) => `(async function() { ${code} })();`;
+
+const check = (code, module = false) => {
+  const args = ["-e", code];
+  if (module) args.push("--input-type=module");
+  const { status, stdout, stderr } = spawnSync("node", args);
+  expect(stderr.toString()).toBe(""); // No warnings or errors are printed
+  expect(stdout.toString()).toBe(TARGET_OUTPUT); // The output should match
+  expect(status).toBe(0); // The process should exit cleanly
+};
+
+describe("Testing the bundle", () => {
+  it("ECMAScript Module (ESM)", () => {
+    check(`import { pipeline } from "${MODULE_NAME}";${CODE_BODY}`, true);
+  });
+
+  it("CommonJS (CJS) with require", () => {
+    check(`const { pipeline } = require("${MODULE_NAME}");${wrap_async_iife(CODE_BODY)}`);
+  });
+
+  it("CommonJS (CJS) with dynamic import", () => {
+    check(`${wrap_async_iife(`const { pipeline } = await import("${MODULE_NAME}");${CODE_BODY}`)}`);
+  });
+});