diff --git a/packages/android/tests/multi-tasks/setting.test.ts b/packages/android/tests/multi-tasks/setting.test.ts new file mode 100644 index 000000000..70bde3694 --- /dev/null +++ b/packages/android/tests/multi-tasks/setting.test.ts @@ -0,0 +1,80 @@ + +import { sleep, ReportMergingTool } from '@midscene/core/utils'; +import { type TestStatus } from '@midscene/core'; +import { AndroidAgent, AndroidDevice, getConnectedDevices } from '@midscene/android'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, it } from 'vitest'; +import ADB from 'appium-adb'; + + +describe(`Test Setting`, () => { + let page: AndroidDevice; + let adb: ADB; + let agent: AndroidAgent; + let startTime: number; + let itTestStatus: TestStatus = 'passed' + const reportMergingTool = new ReportMergingTool(); + + beforeAll(async () => { + const devices = await getConnectedDevices(); + page = new AndroidDevice(devices[0].udid); + adb = await page.getAdb(); + }); + + beforeEach((ctx) => { + startTime = performance.now() + agent = new AndroidAgent(page, { + groupName: ctx.task.name, + }); + }); + + afterEach((ctx) => { + if (ctx.task.result?.state === 'pass') { + itTestStatus = "passed"; + } else if (ctx.task.result?.state === 'skip') { + itTestStatus = "skipped"; + } else if (ctx.task.result?.errors?.[0].message.includes("timed out")) { + itTestStatus = "timedOut"; + } else { + itTestStatus = 'failed'; + } + reportMergingTool.append({ + reportFilePath: agent.reportFile as string, + reportAttributes: { + testId: `${ctx.task.name}`, //ID is a unique identifier used by the front end to distinguish each use case! + testTitle: `${ctx.task.name}`, + testDescription: 'desc', + testDuration: (Date.now() - ctx.task.result?.startTime!) | 0, + testStatus: itTestStatus + } + }); + }); + + afterAll(() => { + reportMergingTool.mergeReports(); + }); + + it( + 'toggle wlan', + async () => { + await adb.shell('input keyevent KEYCODE_HOME'); + await sleep(1000); + await adb.shell('am start -n com.android.settings/.Settings'); + await sleep(1000); + await agent.aiAction('find and enter WLAN setting'); + await agent.aiAction('toggle WLAN status *once*, if WLAN is off pls turn it on, otherwise turn it off.'); + } + ); + + it( + 'toggle bluetooth', + async (ctx) => { + const adb = await page.getAdb(); + await adb.shell('input keyevent KEYCODE_HOME'); + await sleep(1000); + await adb.shell('am start -n com.android.settings/.Settings'); + await sleep(1000); + await agent.aiAction('find and enter bluetooth setting'); + await agent.aiAction('toggle bluetooth status *once*, if bluetooth is off pls turn it on, otherwise turn it off.'); + } + ); +}); diff --git a/packages/android/vitest.config.ts b/packages/android/vitest.config.ts index 409b13880..3ffa5ad2b 100644 --- a/packages/android/vitest.config.ts +++ b/packages/android/vitest.config.ts @@ -19,6 +19,8 @@ const testFiles = (() => { switch (aiTestType) { case 'android': return [...aiAndroidTests]; + case 'report-aggregation': + return ['tests/multi-tasks/**/*.test.ts'] default: return unitTests; } @@ -34,6 +36,7 @@ export default defineConfig({ include: testFiles, testTimeout: 3 * 60 * 1000, // Global timeout set to 10 seconds dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable + fileParallelism: false, // disable parallel file test }, define: { __VERSION__: `'${version}'`, diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index ae4040ab5..571a3ca8d 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -25,7 +25,6 @@ import { type TUserPrompt, type UIContext, } from '../index'; - import yaml from 'js-yaml'; import { @@ -850,9 +849,8 @@ export class Agent< const message = output ? undefined - : `Assertion failed: ${msg || (typeof assertion === 'string' ? assertion : assertion.prompt)}\nReason: ${ - thought || executor.latestErrorTask()?.error || '(no_reason)' - }`; + : `Assertion failed: ${msg || (typeof assertion === 'string' ? assertion : assertion.prompt)}\nReason: ${thought || executor.latestErrorTask()?.error || '(no_reason)' + }`; if (opt?.keepRawResponse) { return { @@ -983,7 +981,7 @@ export class Agent< param: { content: opt?.content || '', }, - executor: async () => {}, + executor: async () => { }, }; // 4. build ExecutionDump const executionDump: ExecutionDump = { @@ -1009,17 +1007,17 @@ export class Agent< const { groupName, groupDescription, executions } = this.dump; const newExecutions = Array.isArray(executions) ? executions.map((execution: any) => { - const { tasks, ...restExecution } = execution; - let newTasks = tasks; - if (Array.isArray(tasks)) { - newTasks = tasks.map((task: any) => { - // only remove uiContext and log from task - const { uiContext, log, ...restTask } = task; - return restTask; - }); - } - return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) }; - }) + const { tasks, ...restExecution } = execution; + let newTasks = tasks; + if (Array.isArray(tasks)) { + newTasks = tasks.map((task: any) => { + // only remove uiContext and log from task + const { uiContext, log, ...restTask } = task; + return restTask; + }); + } + return { ...restExecution, ...(newTasks ? { tasks: newTasks } : {}) }; + }) : []; return { groupName, @@ -1065,7 +1063,7 @@ export class Agent< if (opts.cache === true) { throw new Error( 'cache: true requires an explicit cache ID. Please provide:\n' + - 'Example: cache: { id: "my-cache-id" }', + 'Example: cache: { id: "my-cache-id" }', ); } @@ -1075,7 +1073,7 @@ export class Agent< if (!config.id) { throw new Error( 'cache configuration requires an explicit id. Please provide:\n' + - 'Example: cache: { id: "my-cache-id" }', + 'Example: cache: { id: "my-cache-id" }', ); } const id = config.id; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 62f7ad70f..6cbe5ead5 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -303,10 +303,10 @@ export interface BaseAgentParserOpt { selector?: string; } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PuppeteerParserOpt extends BaseAgentParserOpt {} +export interface PuppeteerParserOpt extends BaseAgentParserOpt { } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PlaywrightParserOpt extends BaseAgentParserOpt {} +export interface PlaywrightParserOpt extends BaseAgentParserOpt { } /* action @@ -376,11 +376,11 @@ export type ExecutionTask< > = E & ExecutionTaskReturn< E extends ExecutionTaskApply - ? TaskOutput - : unknown, + ? TaskOutput + : unknown, E extends ExecutionTaskApply - ? TaskLog - : unknown + ? TaskLog + : unknown > & { status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled'; error?: Error; @@ -605,3 +605,16 @@ export interface AgentOpt { cache?: Cache; replanningCycleLimit?: number; } + +export type TestStatus = "passed" | "failed" | "timedOut" | "skipped" | "interrupted"; + +export interface ReportFileWithAttributes { + reportFilePath: string; + reportAttributes: { + testDuration: number; + testStatus: TestStatus; + testTitle: string; + testId: string; + testDescription: string; + }; +} \ No newline at end of file diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index a8edb2dac..98d6b6c4f 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -16,10 +16,12 @@ import { ifInWorker, uuid, } from '@midscene/shared/utils'; -import type { Cache, Rect, ReportDumpWithAttributes } from './types'; - +import type { Cache, Rect, ReportDumpWithAttributes, ReportFileWithAttributes } from './types'; +import { getReportFileName } from './agent'; let logEnvReady = false; +export { appendFileSync } from 'fs' + export const groupedActionDumpFileExt = 'web-dump.json'; /** @@ -160,6 +162,39 @@ export function reportHTMLContent( return tpl + dumpContent; } +export function getHtmlScripts( + dumpData: ReportDumpWithAttributes, +): string { + + // if reportPath is set, it means we are in write to file mode + let dumpContent = ''; + + if (typeof dumpData === 'string') { + // do not use template string here, will cause bundle error + dumpContent = + // biome-ignore lint/style/useTemplate: + ''; + } else { + const { dumpString, attributes } = dumpData; + const attributesArr = Object.keys(attributes || {}).map((key) => { + return `${key}="${encodeURIComponent(attributes![key])}"`; + }); + + dumpContent = + // do not use template string here, will cause bundle error + // biome-ignore lint/style/useTemplate: + ''; + } + + return dumpContent; +} + export function writeDumpReport( fileName: string, dumpData: string | ReportDumpWithAttributes, @@ -200,7 +235,7 @@ export function writeDumpReport( export function writeLogFile(opts: { fileName: string; fileExt: string; - fileContent: string; + fileContent: string | ReportDumpWithAttributes; type: 'dump' | 'cache' | 'report' | 'tmp'; generateReport?: boolean; appendReport?: boolean; @@ -242,7 +277,7 @@ export function writeLogFile(opts: { if (type !== 'dump') { // do not write dump file any more - writeFileSync(filePath, fileContent); + writeFileSync(filePath, JSON.stringify(fileContent)); } if (opts?.generateReport) { @@ -371,3 +406,113 @@ export function uploadTestInfoToServer({ lastReportedRepoUrl = repoUrl; } } + + +export class ReportMergingTool { + private reportInfos: ReportFileWithAttributes[] = []; + constructor() { + + } + public append(reportInfo: ReportFileWithAttributes) { + this.reportInfos.push(reportInfo); + } + + private extractLastScriptContentFromEnd(filePath: string): string { + const INITIAL_CHUNK_SIZE = 1024 * 1024; // Initial chunk size 1MB (adjustable based on content) + const fd = fs.openSync(filePath, 'r'); + const fileSize = fs.statSync(filePath).size; + let position = fileSize; + let buffer = Buffer.alloc(INITIAL_CHUNK_SIZE); + let lastScriptContent: string | null = null; + let isInsideScript = false; // Flag indicating whether + if (isInsideScript) { + const scriptEndIdx = accumulatedContent.indexOf(''); + if (scriptEndIdx !== -1) { + // Extract complete content (from ) + const fullScriptTag = accumulatedContent.slice(0, scriptEndIdx + ''.length); + const contentStartIdx = fullScriptTag.indexOf('>') + 1; + lastScriptContent = fullScriptTag.slice(contentStartIdx, scriptEndIdx).trim(); + break; + } + } + } + + fs.closeSync(fd); + return lastScriptContent ?? ''; + } + + public mergeReports(rmOriginalReports: boolean = false): string | null { + if (this.reportInfos.length <= 1) { + console.log(`Not enough report to merge`); + return null; + } + const outputFilePath = path.resolve(`${getMidsceneRunSubDir('report')}`, `${getReportFileName('merged-report')}.html`); + console.log(`Start merging ${this.reportInfos.length} reports...\nCreating template file...`); + + try { + // Write template + fs.appendFileSync(outputFilePath, getReportTpl()); + + // Process all reports one by one + for (let i = 0; i < this.reportInfos.length; i++) { + const reportInfo = this.reportInfos[i]; + console.log(`Processing report ${i + 1}/${this.reportInfos.length}`); + + const dumpString = this.extractLastScriptContentFromEnd(reportInfo.reportFilePath); + const reportAttributes = reportInfo.reportAttributes; + + const reportHtmlStr = getHtmlScripts({ + dumpString, + attributes: { + playwright_test_duration: reportAttributes.testDuration, + playwright_test_status: reportAttributes.testStatus, + playwright_test_title: reportAttributes.testTitle, + playwright_test_id: reportAttributes.testId, + playwright_test_description: reportAttributes.testDescription, + }, + }) + '\n'; + + fs.appendFileSync(outputFilePath, reportHtmlStr); + } + + console.log(`Successfully merged new report: ${outputFilePath}`); + + // Remove original reports if needed + if (rmOriginalReports) { + for (const info of this.reportInfos) { + try { + fs.unlinkSync(info.reportFilePath); + } catch (error) { + console.error(`Error deleting report ${info.reportFilePath}:`, error); + } + } + console.log(`Removed ${this.reportInfos.length} original reports`); + } + return outputFilePath; + } catch (error) { + console.error('Error in mergeReports:', error); + throw error; + } + + } + +} \ No newline at end of file diff --git a/packages/core/tests/unit-test/utils.test.ts b/packages/core/tests/unit-test/utils.test.ts index 939475d72..7d561c3d6 100644 --- a/packages/core/tests/unit-test/utils.test.ts +++ b/packages/core/tests/unit-test/utils.test.ts @@ -24,7 +24,7 @@ import { uuid } from '@midscene/shared/utils'; import { describe, expect, it } from 'vitest'; import { z } from 'zod'; // @ts-ignore no types in es folder -import { reportHTMLContent, writeDumpReport } from '../../dist/es/utils'; // use modules from dist, otherwise we will miss the template file +import { reportHTMLContent, writeDumpReport, ReportMergingTool } from '../../dist/es/utils'; // use modules from dist, otherwise we will miss the template file import { getTmpDir, getTmpFile, @@ -176,8 +176,8 @@ describe('utils', () => { console.log( 'Memory usage before test:', `RSS: ${Math.round(startMemory.rss / 1024 / 1024)}MB, ` + - `Heap Total: ${heapTotalBefore}MB, ` + - `Heap Used: ${heapUsedBefore}MB`, + `Heap Total: ${heapTotalBefore}MB, ` + + `Heap Used: ${heapUsedBefore}MB`, ); // Store start time @@ -213,8 +213,8 @@ describe('utils', () => { console.log( 'Memory usage after test:', `RSS: ${Math.round(rssAfter)}MB, ` + - `Heap Total: ${heapTotalAfter}MB, ` + - `Heap Used: ${heapUsedAfter}MB`, + `Heap Total: ${heapTotalAfter}MB, ` + + `Heap Used: ${heapUsedAfter}MB`, ); // Check if file exists @@ -1542,3 +1542,66 @@ describe('loadActionParam and dumpActionParam integration', () => { `); }); }); + +describe('repotMergingTool', () => { + it('should merge 3 mocked reports', async () => { + const tool = new ReportMergingTool(); + let expectedContents = []; + for (let i = 0; i < 3; i++) { + // create report files + const content = `report content ${i}`; + expectedContents.push(content); + const reportPath = writeDumpReport(`report-to-merge-${i}`, { + dumpString: `report content ${i}`, + }); + // append report content and its relevant information to reportMergingTool + tool.append({ + reportFilePath: reportPath, reportAttributes: { + testDescription: `desc${i}`, + testDuration: 1, + testId: `${i}`, + testStatus: 'passed', + testTitle: `${i}` + } + }); + } + // execute merge operation + const mergedReportPath = tool.mergeReports(); + // assert merge success + const mergedReportContent = readFileSync(mergedReportPath!, 'utf-8'); + expectedContents.forEach(content => { + expect(mergedReportContent).contains(content); + }); + }); + + it('should merge 3 mocked reports, and delete original reports after that.', async () => { + const tool = new ReportMergingTool(); + let expectedContents = []; + for (let i = 0; i < 3; i++) { + const content = `report content ${i}`; + expectedContents.push(content); + const reportPath = writeDumpReport(`report-rm-to-merge-${i}`, { + dumpString: `report content ${i}, original report file deleted`, + }); + tool.append({ + reportFilePath: reportPath, reportAttributes: { + testDescription: `desc${i}`, + testDuration: 1, + testId: `${i}`, + testStatus: 'passed', + testTitle: `${i}` + } + }); + } + // assert merge success + const mergedReportPath: string = tool.mergeReports(true); + const mergedReportContent = readFileSync(mergedReportPath!, 'utf-8'); + expectedContents.forEach(content => { + expect(mergedReportContent).contains(content); + }); + // assert source report files deleted successfully + tool['reportInfos'].forEach((el: any) => { + expect(fs.existsSync(el.reportFilePath)).toBe(false); + }); + }); +}); \ No newline at end of file