feat(android): aggregate reports when running more than one test (#1163)

packages/android/tests/multi-tasks/setting.test.ts

@@ -0,0 +1,80 @@
+
+import { sleep, ReportMergingTool } from '@midscene/core/utils';
+import { type TestStatus } from '@midscene/core';
+import { AndroidAgent, AndroidDevice, getConnectedDevices } from '@midscene/android';
+import { afterAll, afterEach, beforeAll, beforeEach, describe, it } from 'vitest';
+import ADB from 'appium-adb';
+
+
+describe(`Test Setting`, () => {
+    let page: AndroidDevice;
+    let adb: ADB;
+    let agent: AndroidAgent;
+    let startTime: number;
+    let itTestStatus: TestStatus = 'passed'
+    const reportMergingTool = new ReportMergingTool();
+
+    beforeAll(async () => {
+        const devices = await getConnectedDevices();
+        page = new AndroidDevice(devices[0].udid);
+        adb = await page.getAdb();
+    });
+
+    beforeEach((ctx) => {
+        startTime = performance.now()
+        agent = new AndroidAgent(page, {
+            groupName: ctx.task.name,
+        });
+    });
+
+    afterEach((ctx) => {
+        if (ctx.task.result?.state === 'pass') {
+            itTestStatus = "passed";
+        } else if (ctx.task.result?.state === 'skip') {
+            itTestStatus = "skipped";
+        } else if (ctx.task.result?.errors?.[0].message.includes("timed out")) {
+            itTestStatus = "timedOut";
+        } else {
+            itTestStatus = 'failed';
+        }
+        reportMergingTool.append({
+            reportFilePath: agent.reportFile as string,
+            reportAttributes: {
+                testId: `${ctx.task.name}`, //ID is a unique identifier used by the front end to distinguish each use case!
+                testTitle: `${ctx.task.name}`,
+                testDescription: 'desc',
+                testDuration: (Date.now() - ctx.task.result?.startTime!) | 0,
+                testStatus: itTestStatus
+            }
+        });
+    });
+
+    afterAll(() => {
+        reportMergingTool.mergeReports();
+    });
+
+    it(
+        'toggle wlan',
+        async () => {
+            await adb.shell('input keyevent KEYCODE_HOME');
+            await sleep(1000);
+            await adb.shell('am start -n com.android.settings/.Settings');
+            await sleep(1000);
+            await agent.aiAction('find and enter WLAN setting');
+            await agent.aiAction('toggle WLAN status *once*, if WLAN is off pls turn it on, otherwise turn it off.');
+        }
+    );
+
+    it(
+        'toggle bluetooth',
+        async (ctx) => {
+            const adb = await page.getAdb();
+            await adb.shell('input keyevent KEYCODE_HOME');
+            await sleep(1000);
+            await adb.shell('am start -n com.android.settings/.Settings');
+            await sleep(1000);
+            await agent.aiAction('find and enter bluetooth setting');
+            await agent.aiAction('toggle bluetooth status *once*, if bluetooth is off pls turn it on, otherwise turn it off.');
+        }
+    );
+});

packages/android/vitest.config.ts

@@ -19,6 +19,8 @@ const testFiles = (() => {
   switch (aiTestType) {
     case 'android':
       return [...aiAndroidTests];
+    case 'report-aggregation':
+      return ['tests/multi-tasks/**/*.test.ts']
     default:
       return unitTests;
   }
@@ -34,6 +36,7 @@ export default defineConfig({
     include: testFiles,
     testTimeout: 3 * 60 * 1000, // Global timeout set to 10 seconds
     dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable
+    fileParallelism: false, // disable parallel file test
   },
   define: {
     __VERSION__: `'${version}'`,

packages/core/src/agent/agent.ts

@@ -25,7 +25,6 @@ import {
   type TUserPrompt,
   type UIContext,
 } from '../index';
-
 import yaml from 'js-yaml';
 
 import {
@@ -850,9 +849,8 @@ export class Agent<
 
     const message = output
       ? undefined
-      : `Assertion failed: ${msg || (typeof assertion === 'string' ? assertion : assertion.prompt)}\nReason: ${
-          thought || executor.latestErrorTask()?.error || '(no_reason)'
-        }`;
+      : `Assertion failed: ${msg || (typeof assertion === 'string' ? assertion : assertion.prompt)}\nReason: ${thought || executor.latestErrorTask()?.error || '(no_reason)'
+      }`;
 
     if (opt?.keepRawResponse) {
       return {
@@ -983,7 +981,7 @@ export class Agent<
       param: {
         content: opt?.content || '',
       },
-      executor: async () => {},
+      executor: async () => { },
     };
     // 4. build ExecutionDump
     const executionDump: ExecutionDump = {
@@ -1009,17 +1007,17 @@ export class Agent<
     const { groupName, groupDescription, executions } = this.dump;
     const newExecutions = Array.isArray(executions)
       ? executions.map((execution: any) => {
-          const { tasks, ...restExecution } = execution;
-          let newTasks = tasks;
-          if (Array.isArray(tasks)) {
-            newTasks = tasks.map((task: any) => {
-              // only remove uiContext and log from task
-              const { uiContext, log, ...restTask } = task;
-              return restTask;
-            });
-          }
-          return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) };
-        })
+        const { tasks, ...restExecution } = execution;
+        let newTasks = tasks;
+        if (Array.isArray(tasks)) {
+          newTasks = tasks.map((task: any) => {
+            // only remove uiContext and log from task
+            const { uiContext, log, ...restTask } = task;
+            return restTask;
+          });
+        }
+        return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) };
+      })
       : [];
     return {
       groupName,
@@ -1065,7 +1063,7 @@ export class Agent<
       if (opts.cache === true) {
         throw new Error(
           'cache: true requires an explicit cache ID. Please provide:\n' +
-            'Example: cache: { id: "my-cache-id" }',
+          'Example: cache: { id: "my-cache-id" }',
         );
       }
 
@@ -1075,7 +1073,7 @@ export class Agent<
         if (!config.id) {
           throw new Error(
             'cache configuration requires an explicit id. Please provide:\n' +
-              'Example: cache: { id: "my-cache-id" }',
+            'Example: cache: { id: "my-cache-id" }',
           );
         }
         const id = config.id;

packages/core/src/types.ts

@@ -303,10 +303,10 @@ export interface BaseAgentParserOpt {
   selector?: string;
 }
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface PuppeteerParserOpt extends BaseAgentParserOpt {}
+export interface PuppeteerParserOpt extends BaseAgentParserOpt { }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface PlaywrightParserOpt extends BaseAgentParserOpt {}
+export interface PlaywrightParserOpt extends BaseAgentParserOpt { }
 
 /*
 action
@@ -376,11 +376,11 @@ export type ExecutionTask<
 > = E &
   ExecutionTaskReturn<
     E extends ExecutionTaskApply<any, any, infer TaskOutput, any>
-      ? TaskOutput
-      : unknown,
+    ? TaskOutput
+    : unknown,
     E extends ExecutionTaskApply<any, any, any, infer TaskLog>
-      ? TaskLog
-      : unknown
+    ? TaskLog
+    : unknown
   > & {
     status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
     error?: Error;
@@ -605,3 +605,16 @@ export interface AgentOpt {
   cache?: Cache;
   replanningCycleLimit?: number;
 }
+
+export type TestStatus = "passed" | "failed" | "timedOut" | "skipped" | "interrupted";
+
+export interface ReportFileWithAttributes {
+  reportFilePath: string;
+  reportAttributes: {
+    testDuration: number;
+    testStatus: TestStatus;
+    testTitle: string;
+    testId: string;
+    testDescription: string;
+  };
+}