Support multiple backends

.gitignore

@@ -4,4 +4,4 @@ out/
 .vscode/
 env.json
 package-lock.json
-config.json
+config*.json

generate-config.js

@@ -301,7 +301,8 @@ program
       .default(["cpu", "gpu", "npu"])
   )
   .addOption(new Option("-e, --backend <backend>", "The backend to use").choices(["ort", "tflite"]).default("ort"))
-  .action(async ({ devices, browser, backend }) => {
+  .option("-o, --output <path>", "The output config file path", "config.json")
+  .action(async ({ devices, browser, backend, output }) => {
     if (process.platform === "linux" && browser === "edge_canary") {
       console.error("edge_canary is not available on linux.");
       return;
@@ -313,7 +314,7 @@ program
       devices.splice(devices.indexOf("npu"), 1);
     }
 
-    const config = { browser, ...filterSamplesWithDevices(ORIGINAL_CONFIG, devices) };
+    const config = { backend, browser, ...filterSamplesWithDevices(ORIGINAL_CONFIG, devices) };
     if (backend === "ort") {
       config.browserArgs.push("--enable-features=WebMachineLearningNeuralNetwork,WebNNOnnxRuntime");
     } else if (backend === "tflite") {
@@ -323,7 +324,7 @@ program
       );
     }
 
-    fs.writeFileSync("config.json", JSON.stringify(config, null, 2));
-    console.log(`Generated config.json for ${browser} with ${backend} backend on ${devices.join("-")}`);
+    fs.writeFileSync(output, JSON.stringify(config, null, 2));
+    console.log(`Generated ${output} for ${browser} with ${backend} backend on ${devices.join("-")}`);
   })
   .parse();

src/main.js

@@ -44,6 +44,7 @@ program
 program.action(async ({ config: configPath, filters, browserDir, userDataDir }) => {
   try {
     const config = require(path.resolve(process.cwd(), configPath));
+    console.log(`Using config file: ${configPath}`);
 
     if (filters === true) {
       console.log("Available filters:");

src/utils/report.js

@@ -132,7 +132,7 @@ async function report(results) {
   }
   const hostname = env.hostname || os.hostname();
   const reportTime = util.getTimestamp(true);
-  let subject = `[Sample Test][${results.deviceInfo.browser}] ${hostname} ${reportTime}`;
+  let subject = `[Sample Test][${results.deviceInfo.browser}][${results.deviceInfo.backend}] ${hostname} ${reportTime}`;
 
   try {
     await sendMail(subject, await renderResultsAsHTML(results), [

src/utils/util.js

@@ -334,13 +334,15 @@ async function getNPUInfo() {
 
 // get device info
 async function getDeviceInfo(config) {
-  let deviceInfo = {};
-  deviceInfo["hostname"] = config["hostname"] ? config["hostname"] : os.hostname();
-  deviceInfo.platform = os.platform();
-  deviceInfo["samplesUrl"] = config["samplesBasicUrl"];
-  deviceInfo["developerPreviewUrl"] = config["developerPreviewBasicUrl"];
-  deviceInfo["browser"] = config["browser"];
-  deviceInfo["browserArgs"] = config["browserArgs"];
+  let deviceInfo = {
+    hostname: config.hostname || os.hostname(),
+    platform: os.platform(),
+    samplesUrl: config.samplesBasicUrl,
+    developerPreviewUrl: config.developerPreviewBasicUrl,
+    backend: config.backend,
+    browser: config.browser,
+    browserArgs: config.browserArgs
+  };
   const { browserPath, userDataDir } = getBrowserPath(config);
   deviceInfo["browserPath"] = browserPath;