feat: rename aiAction to aiAct to adapt Midscene v1.0 (#75)

yuyutaotao · web-flow · commit 9b6fcc6d91a8 · 2025-12-24T14:30:58.000+08:00
* fix: command in readme

* feat: rename aiAction as aiAct
diff --git a/android/javascript-sdk-demo/demo-run-yaml.ts b/android/javascript-sdk-demo/demo-run-yaml.ts
@@ -9,7 +9,7 @@ Promise.resolve(
 
     // 👀 init Midscene agent
     const agent = new AndroidAgent(page,{
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await page.connect();
diff --git a/android/javascript-sdk-demo/demo.ts b/android/javascript-sdk-demo/demo.ts
@@ -9,7 +9,7 @@ Promise.resolve(
 
     // 👀 init Midscene agent
     const agent = new AndroidAgent(page,{
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await page.connect();
@@ -18,7 +18,7 @@ Promise.resolve(
     await sleep(5000);
 
     // 👀 type keywords, perform a search
-    await agent.aiAction(
+    await agent.aiAct(
       'type "Headphones" in search box, click search button'
     );
 
diff --git a/android/vitest-demo/tests/setting.test.ts b/android/vitest-demo/tests/setting.test.ts
@@ -12,19 +12,19 @@ describe(
     await it('Android settings page demo for scroll', async () => {
       const devices = await getConnectedDevices();
       const agent = await agentFromAdbDevice(devices[0].udid,{
-        aiActionContext:
+        aiActContext:
           'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
       });
 
       await agent.launch('com.android.settings/.Settings');
 
 
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('open "More settings"');
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('scroll list to top');
-      await agent.aiAction('swipe down one screen');
-      await agent.aiAction('swipe up one screen');
+      await agent.aiAct('scroll list to bottom');
+      await agent.aiAct('open "More settings"');
+      await agent.aiAct('scroll list to bottom');
+      await agent.aiAct('scroll list to top');
+      await agent.aiAct('swipe down one screen');
+      await agent.aiAct('swipe up one screen');
     });
   },
   360 * 1000,
diff --git a/android/vitest-demo/tests/todo.test.ts b/android/vitest-demo/tests/todo.test.ts
@@ -17,7 +17,7 @@ describe('Test todo list', () => {
     const devices = await getConnectedDevices();
     const page = new AndroidDevice(devices[0].udid);
     agent = new AndroidAgent(page,{
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await page.connect();
@@ -28,22 +28,22 @@ describe('Test todo list', () => {
   it(
     'ai todo',
     async () => {
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study JS today' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study Rust tomorrow' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study AI the day after tomorrow' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         'move the mouse to the second item in the task list and click the delete button on the right of the second task',
       );
-      await agent.aiAction(
+      await agent.aiAct(
         'click the check button on the left of the second task',
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "click the 'completed' status button below the task list",
       );
 
diff --git a/android/yaml-scripts-demo/midscene-scripts/maps-navigation.yaml b/android/yaml-scripts-demo/midscene-scripts/maps-navigation.yaml
@@ -7,8 +7,8 @@ android:
 tasks:
   - name: travel
     flow:
-      - aiAction: open Maps app
-      - aiAction: input 'attractions' in the search bar, and click the search button
-      - aiAction: click the first search result, enter the attraction details page
-      - aiAction: click "Directions" button, enter the route planning page
-      - aiAction: click "Start" button to start navigation
+      - aiAct: open Maps app
+      - aiAct: input 'attractions' in the search bar, and click the search button
+      - aiAct: click the first search result, enter the attraction details page
+      - aiAct: click "Directions" button, enter the route planning page
+      - aiAct: click "Start" button to start navigation
diff --git a/android/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml b/android/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml
@@ -7,10 +7,10 @@ android:
 tasks:
   - name: search headphones
     flow:
-      - aiAction: open browser and navigate to ebay.com
-      - aiAction: type 'Headphones' in ebay search box, hit Enter
+      - aiAct: open browser and navigate to ebay.com
+      - aiAct: type 'Headphones' in ebay search box, hit Enter
       - sleep: 5000
-      - aiAction: scroll down the page for 800px
+      - aiAct: scroll down the page for 800px
 
   - name: extract headphones info
     flow:
diff --git a/android/yaml-scripts-demo/midscene-scripts/twitter-auto-like.yaml b/android/yaml-scripts-demo/midscene-scripts/twitter-auto-like.yaml
@@ -6,11 +6,11 @@ android:
 tasks:
   - name: like tweets
     flow:
-      - aiAction: open x app
-      - aiAction: search 'midscene ai'
-      - aiAction: click the user called 'midscene ai'
-      - aiAction: click the first tweet
-      - aiAction: click the like button
+      - aiAct: open x app
+      - aiAct: search 'midscene ai'
+      - aiAct: click the user called 'midscene ai'
+      - aiAct: click the first tweet
+      - aiAct: click the like button
 
   - name: extract tweets info
     flow:
diff --git a/bridge-mode-demo/demo-new-tab.ts b/bridge-mode-demo/demo-new-tab.ts
@@ -16,7 +16,7 @@ Promise.resolve(
     console.log("connected to a new tab !");
 
     // these are the same as normal Midscene agent
-    await agent.aiAction('type "AI 101" and hit Enter');
+    await agent.aiAct('type "AI 101" and hit Enter');
     await sleep(3000);
 
     await agent.aiAssert("there are some search results");
diff --git a/bridge-mode-demo/yaml-new-tab.yml b/bridge-mode-demo/yaml-new-tab.yml
@@ -8,7 +8,7 @@ target:
 tasks:
   - name: login
     flow:
-      - aiAction: type 'standard_user' in user name input, type 'secret_sauce' in password, click 'Login'
+      - aiAct: type 'standard_user' in user name input, type 'secret_sauce' in password, click 'Login'
 
   - name: extract items info
     flow:
diff --git a/connectivity-test/tests/connectivity.test.ts b/connectivity-test/tests/connectivity.test.ts
@@ -4,7 +4,7 @@ import OpenAI, { AzureOpenAI } from "openai";
 import { join } from "node:path";
 import { localImg2Base64 } from "@midscene/shared/img";
 import {  globalModelConfigManager } from "@midscene/shared/env";
-import { AIActionType, callAIWithObjectResponse } from "@midscene/core/ai-model";
+import { aiActType, callAIWithObjectResponse } from "@midscene/core/ai-model";
 import {
   DefaultAzureCredential,
   getBearerTokenProvider,
@@ -95,7 +95,7 @@ describe("Use Midscene wrapped OpenAI SDK", () => {
           ],
         },
       ],
-      AIActionType.EXTRACT_DATA,
+      aiActType.EXTRACT_DATA,
       globalModelConfigManager.getModelConfig('default')
     );
     console.log(result.content.content);
diff --git a/custom-interface/README.md b/custom-interface/README.md
@@ -29,7 +29,7 @@ npm install
 npm run build
 
 # run demo.ts (javascript demo)
-npx tsx demo.ts
+npx tsx demo/run.ts
 
 # run cli with yaml scripts
 npx midscene ./demo/run.yaml
diff --git a/custom-interface/demo/run.ts b/custom-interface/demo/run.ts
@@ -15,7 +15,7 @@ async function runAgentDemo() {
   );
 
   // 👀 perform action based on the action space
-  await agent.aiAction('launch the "1 password" app');
+  await agent.aiAct('launch the "1 password" app');
 }
 
 // Demo 2: launch playground to interact with the agent
diff --git a/custom-interface/demo/run.yaml b/custom-interface/demo/run.yaml
@@ -8,7 +8,7 @@ config:
   # export: 'MyDeviceClass'
 
 agent:
-  aiActionContext: 'if the user ask you to check the identity, he means tap the "facetime" app icon.'
+  aiActContext: 'if the user ask you to check the identity, he means tap the "facetime" app icon.'
 
 tasks:
   - name: assertion
@@ -22,5 +22,5 @@ tasks:
       
   - name: action
     flow:
-      - aiAction: check the identity, and launch the "1 password" app
+      - aiAct: check the identity, and launch the "1 password" app
       - aiKeyboardPress: Enter
diff --git a/ios/javascript-sdk-demo/demo-run-yaml.ts b/ios/javascript-sdk-demo/demo-run-yaml.ts
@@ -6,7 +6,7 @@ Promise.resolve(
   (async () => {
     // 👀 init Midscene agent
     const agent = await agentFromWebDriverAgent({
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await agent.launch('https://www.ebay.com');
diff --git a/ios/javascript-sdk-demo/demo.ts b/ios/javascript-sdk-demo/demo.ts
@@ -6,15 +6,15 @@ Promise.resolve(
   (async () => {
     // 👀 init Midscene agent
     const agent = await agentFromWebDriverAgent({
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await agent.launch('https://www.ebay.com');
 
     await sleep(5000);
 
     // 👀 type keywords, perform a search
-    await agent.aiAction('type "Headphones" in ebay search box, click the search button');
+    await agent.aiAct('type "Headphones" in ebay search box, click the search button');
 
     // 👀 wait for the loading
     await agent.aiWaitFor("there is at least one headphone item on page");
diff --git a/ios/vitest-demo/setting.test.ts b/ios/vitest-demo/setting.test.ts
@@ -13,19 +13,19 @@ describe(
   async () => {
     await it('iOS settings page demo for scroll', async () => {
       const agent = await agentFromWebDriverAgent({
-        aiActionContext:
+        aiActContext:
           'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
       });
 
       await agent.launch('com.apple.Preferences');
 
 
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('open "More settings"');
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('scroll list to top');
-      await agent.aiAction('swipe down one screen');
-      await agent.aiAction('swipe up one screen');
+      await agent.aiAct('scroll list to bottom');
+      await agent.aiAct('open "More settings"');
+      await agent.aiAct('scroll list to bottom');
+      await agent.aiAct('scroll list to top');
+      await agent.aiAct('swipe down one screen');
+      await agent.aiAct('swipe up one screen');
     });
   },
   360 * 1000,
diff --git a/ios/vitest-demo/todo.test.ts b/ios/vitest-demo/todo.test.ts
@@ -15,7 +15,7 @@ describe('Test todo list', () => {
 
   beforeAll(async () => {
     agent = await agentFromWebDriverAgent({
-      aiActionContext:
+      aiActContext:
         'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
     });
     await agent.launch(pageUrl);
@@ -25,22 +25,22 @@ describe('Test todo list', () => {
   it(
     'ai todo',
     async () => {
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study JS today' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study Rust tomorrow' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "type 'Study AI the day after tomorrow' in the task box input and press the Enter key",
       );
-      await agent.aiAction(
+      await agent.aiAct(
         'move the mouse to the second item in the task list and click the delete button on the right of the second task',
       );
-      await agent.aiAction(
+      await agent.aiAct(
         'click the check button on the left of the second task',
       );
-      await agent.aiAction(
+      await agent.aiAct(
         "click the 'completed' status button below the task list",
       );
 
diff --git a/ios/yaml-scripts-demo/midscene-scripts/maps-navigation.yaml b/ios/yaml-scripts-demo/midscene-scripts/maps-navigation.yaml
@@ -6,8 +6,8 @@ ios:
 tasks:
   - name: travel
     flow:
-      - aiAction: open Maps app
-      - aiAction: input 'attractions' in the search bar, and click the search button
-      - aiAction: click the first search result, enter the attraction details page
-      - aiAction: click "Directions" button, enter the route planning page
-      - aiAction: click "Start" button to start navigation
+      - aiAct: open Maps app
+      - aiAct: input 'attractions' in the search bar, and click the search button
+      - aiAct: click the first search result, enter the attraction details page
+      - aiAct: click "Directions" button, enter the route planning page
+      - aiAct: click "Start" button to start navigation
diff --git a/ios/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml b/ios/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml
@@ -6,10 +6,10 @@ ios:
 tasks:
   - name: search headphones
     flow:
-      - aiAction: open browser and navigate to ebay.com
-      - aiAction: type 'Headphones' in ebay search box, click search button
+      - aiAct: open browser and navigate to ebay.com
+      - aiAct: type 'Headphones' in ebay search box, click search button
       - sleep: 5000
-      - aiAction: scroll down the page for 800px
+      - aiAct: scroll down the page for 800px
 
   - name: extract headphones info
     flow:
diff --git a/ios/yaml-scripts-demo/midscene-scripts/twitter-auto-like.yaml b/ios/yaml-scripts-demo/midscene-scripts/twitter-auto-like.yaml
@@ -5,11 +5,11 @@ ios:
 tasks:
   - name: like tweets
     flow:
-      - aiAction: open x app
-      - aiAction: search 'midscene ai'
-      - aiAction: click the user called 'midscene ai'
-      - aiAction: click the first tweet
-      - aiAction: click the like button
+      - aiAct: open x app
+      - aiAct: search 'midscene ai'
+      - aiAct: click the user called 'midscene ai'
+      - aiAct: click the first tweet
+      - aiAct: click the like button
 
   - name: extract tweets info
     flow:
diff --git a/playwright-demo/demo.ts b/playwright-demo/demo.ts
@@ -22,7 +22,7 @@ Promise.resolve(
     const agent = new PlaywrightAgent(page);
 
     // 👀 type keywords, perform a search
-    await agent.aiAction('type "Headphones" in search box, hit Enter');
+    await agent.aiAct('type "Headphones" in search box, hit Enter');
 
     // 👀 wait for the loading
     await agent.aiWaitFor("there is at least one headphone item on page");
diff --git a/puppeteer-demo/demo.ts b/puppeteer-demo/demo.ts
@@ -25,7 +25,7 @@ Promise.resolve(
     const agent = new PuppeteerAgent(page);
 
     // 👀 type keywords, perform a search
-    await agent.aiAction('type "Headphones" in search box, hit Enter');
+    await agent.aiAct('type "Headphones" in search box, hit Enter');
 
     // 👀 wait for the loading
     await agent.aiWaitFor("there is at least one headphone item on page");
diff --git a/puppeteer-with-vitest-demo/tests/demo.test.ts b/puppeteer-with-vitest-demo/tests/demo.test.ts
@@ -27,22 +27,22 @@ describe("Test todo list", () => {
   });
 
   it("ai todo", async () => {
-    await agent.aiAction(
+    await agent.aiAct(
       "type 'Study JS today' in the task box input and press the Enter key"
     );
-    await agent.aiAction(
+    await agent.aiAct(
       "type 'Study Rust tomorrow' in the task box input and press the Enter key"
     );
-    await agent.aiAction(
+    await agent.aiAct(
       "type 'Study AI the day after tomorrow' in the task box input and press the Enter key"
     );
-    await agent.aiAction(
+    await agent.aiAct(
       "move the mouse to the second item in the task list and click the delete button on the right of the second task"
     );
-    await agent.aiAction(
+    await agent.aiAct(
       "click the check button on the left of the second task"
     );
-    await agent.aiAction(
+    await agent.aiAct(
       "click the 'completed' status button below the task list"
     );
 
diff --git a/remote-playwright-demo/demo.ts b/remote-playwright-demo/demo.ts
@@ -43,7 +43,7 @@ Promise.resolve(
 
     // 👀 type keywords, perform a search
     console.log('🔍 Searching for Headphones...');
-    await agent.aiAction('type "Headphones" in search box, hit Enter');
+    await agent.aiAct('type "Headphones" in search box, hit Enter');
 
     // 👀 wait for the loading
     await agent.aiWaitFor("there is at least one headphone item on page");
diff --git a/remote-puppeteer-demo/demo.ts b/remote-puppeteer-demo/demo.ts
diff --git a/yaml-scripts-demo/midscene-scripts/local-static-server.yml b/yaml-scripts-demo/midscene-scripts/local-static-server.yml
diff --git a/yaml-scripts-demo/midscene-scripts/sauce-demo.yaml b/yaml-scripts-demo/midscene-scripts/sauce-demo.yaml
diff --git a/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml b/yaml-scripts-demo/midscene-scripts/search-headphone-on-ebay.yaml
