Skip to content

Commit 372e32f

Browse files
quanruquanru
authored
feat: add @midscene/harmony demo examples (#85)
Add HarmonyOS device automation demo examples using @midscene/harmony: - JavaScript SDK demo: eBay search with aiAct, aiQuery, aiAssert, etc. - Vitest demo: todo list test and settings page scroll test
1 parent da49791 commit 372e32f

File tree

9 files changed

+287
-0
lines changed

9 files changed

+287
-0
lines changed

‎harmony/javascript-sdk-demo/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules/
2+
midscene_run/

‎harmony/javascript-sdk-demo/README.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# HarmonyOS demo (JavaScript SDK)
2+
3+
This is a demo to show how to use `@midscene/harmony` JavaScript SDK to control HarmonyOS devices.
4+
5+
## Steps
6+
7+
### Preparation
8+
9+
Create `.env` file
10+
11+
```shell
12+
# Replace with your own API key
13+
MIDSCENE_MODEL_BASE_URL="https://.../compatible-mode/v1"
14+
MIDSCENE_MODEL_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
15+
MIDSCENE_MODEL_NAME="qwen3-vl-plus"
16+
MIDSCENE_MODEL_FAMILY="qwen3-vl"
17+
```
18+
19+
Connect a HarmonyOS device with [HDC](https://developer.huawei.com/consumer/en/doc/harmonyos-guides-V5/ide-hdc-V5)
20+
21+
Refer to this document if you want to use other models like Qwen: https://midscenejs.com/model-strategy.html
22+
23+
### Install
24+
25+
```bash
26+
npm install
27+
```
28+
29+
### Run
30+
31+
```bash
32+
npm test
33+
```
34+
35+
## Reference
36+
37+
https://midscenejs.com/harmony-api-reference

‎harmony/javascript-sdk-demo/demo.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import {
2+
HarmonyAgent,
3+
HarmonyDevice,
4+
getConnectedDevices,
5+
} from '@midscene/harmony';
6+
import 'dotenv/config';
7+
8+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
9+
10+
Promise.resolve(
11+
(async () => {
12+
const devices = await getConnectedDevices();
13+
const device = new HarmonyDevice(devices[0].deviceId);
14+
15+
// 👀 init Midscene agent
16+
const agent = new HarmonyAgent(device);
17+
await device.connect();
18+
await agent.aiAct('open Browser and go to https://www.ebay.com');
19+
20+
await sleep(5000);
21+
22+
// 👀 type keywords, perform a search
23+
await agent.aiAct('type "Headphones" in search box, click search button');
24+
25+
// 👀 wait for the loading
26+
await agent.aiWaitFor('there is at least one headphone item on page');
27+
28+
// 👀 understand the page content, find the items
29+
const items = await agent.aiQuery(
30+
'{itemTitle: string, price: Number}[], find item in list and corresponding price',
31+
);
32+
console.log('headphones in stock', items);
33+
34+
const isMoreThan1000 = await agent.aiBoolean(
35+
'Is the price of the headphones more than 1000?',
36+
);
37+
console.log('isMoreThan1000', isMoreThan1000);
38+
39+
const price = await agent.aiNumber(
40+
'What is the price of the first headphone?',
41+
);
42+
console.log('price', price);
43+
44+
const name = await agent.aiString(
45+
'What is the name of the first headphone?',
46+
);
47+
console.log('name', name);
48+
49+
const location = await agent.aiLocate(
50+
'What is the location of the first headphone?',
51+
);
52+
console.log('location', location);
53+
54+
// 👀 assert by AI
55+
await agent.aiAssert('There is a category filter on the left');
56+
})(),
57+
);

‎harmony/javascript-sdk-demo/package.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "harmony-demo",
3+
"private": true,
4+
"version": "1.0.0",
5+
"description": "> quick start",
6+
"main": "index.js",
7+
"type": "module",
8+
"scripts": {
9+
"test": "tsx demo.ts"
10+
},
11+
"author": "",
12+
"license": "MIT",
13+
"devDependencies": {
14+
"@midscene/harmony": "latest",
15+
"dotenv": "^16.4.5",
16+
"tsx": "4.20.1"
17+
}
18+
}

‎harmony/vitest-demo/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules/
2+
midscene_run/

‎harmony/vitest-demo/README.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# HarmonyOS demo (vitest)
2+
3+
This is a demo to show how to use HDC to control HarmonyOS devices for automation tasks.
4+
5+
## Steps
6+
7+
### Preparation
8+
9+
Create `.env` file
10+
11+
```shell
12+
# Replace with your own API key
13+
MIDSCENE_MODEL_BASE_URL="https://.../compatible-mode/v1"
14+
MIDSCENE_MODEL_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
15+
MIDSCENE_MODEL_NAME="qwen3-vl-plus"
16+
MIDSCENE_MODEL_FAMILY="qwen3-vl"
17+
```
18+
19+
Connect a HarmonyOS device with [HDC](https://developer.huawei.com/consumer/en/doc/harmonyos-guides-V5/ide-hdc-V5)
20+
21+
Refer to this document if you want to use other models like Qwen: https://midscenejs.com/model-strategy.html
22+
23+
### Install
24+
25+
```bash
26+
npm install
27+
```
28+
29+
### Run
30+
31+
case1: Settings page scroll demo
32+
33+
```bash
34+
npm run test -- setting.test.ts
35+
```
36+
37+
case2: Todo app demo
38+
39+
```bash
40+
npm run test -- todo.test.ts
41+
```
42+
43+
## Reference
44+
45+
https://midscenejs.com/harmony-api-reference

‎harmony/vitest-demo/package.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "harmony-with-vitest-demo",
3+
"private": true,
4+
"version": "1.0.0",
5+
"main": "index.js",
6+
"type": "module",
7+
"scripts": {
8+
"test": "vitest --run"
9+
},
10+
"author": "",
11+
"license": "MIT",
12+
"devDependencies": {
13+
"@midscene/harmony": "latest",
14+
"@types/node": "^18.0.0",
15+
"dotenv": "^16.4.5",
16+
"vitest": "^2.1.8"
17+
}
18+
}

‎harmony/vitest-demo/tests/setting.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import { agentFromHdcDevice, getConnectedDevices } from '@midscene/harmony';
2+
import { describe, it, vi } from 'vitest';
3+
import 'dotenv/config';
4+
5+
vi.setConfig({
6+
testTimeout: 90 * 1000,
7+
});
8+
9+
describe(
10+
'harmony integration',
11+
async () => {
12+
await it('HarmonyOS settings page demo for scroll', async () => {
13+
const devices = await getConnectedDevices();
14+
const agent = await agentFromHdcDevice(devices[0].deviceId);
15+
16+
await agent.launch('Settings');
17+
18+
await agent.aiAct('scroll list to bottom');
19+
await agent.aiAct('open "Bluetooth" settings');
20+
await agent.aiAct('scroll list to bottom');
21+
await agent.aiAct('scroll list to top');
22+
await agent.aiAct('swipe down one screen');
23+
await agent.aiAct('swipe up one screen');
24+
});
25+
},
26+
360 * 1000,
27+
);

‎harmony/vitest-demo/tests/todo.test.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import {
2+
HarmonyAgent,
3+
HarmonyDevice,
4+
getConnectedDevices,
5+
} from '@midscene/harmony';
6+
import { beforeAll, describe, expect, it, vi } from 'vitest';
7+
import 'dotenv/config';
8+
9+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
10+
11+
vi.setConfig({
12+
testTimeout: 240 * 1000,
13+
hookTimeout: 240 * 1000,
14+
});
15+
16+
const pageUrl = 'https://todomvc.com/examples/react/dist/';
17+
18+
describe('Test todo list', () => {
19+
let agent: HarmonyAgent;
20+
21+
beforeAll(async () => {
22+
const devices = await getConnectedDevices();
23+
const device = new HarmonyDevice(devices[0].deviceId);
24+
agent = new HarmonyAgent(device);
25+
await device.connect();
26+
await agent.aiAct(`open Browser and go to ${pageUrl}`);
27+
await sleep(3000);
28+
});
29+
30+
it(
31+
'ai todo',
32+
async () => {
33+
await agent.aiAct(
34+
"type 'Study JS today' in the task box input and press the Enter key",
35+
);
36+
await agent.aiAct(
37+
"type 'Study Rust tomorrow' in the task box input and press the Enter key",
38+
);
39+
await agent.aiAct(
40+
"type 'Study AI the day after tomorrow' in the task box input and press the Enter key",
41+
);
42+
await agent.aiAct(
43+
'move the mouse to the second item in the task list and click the delete button on the right of the second task',
44+
);
45+
await agent.aiAct(
46+
'click the check button on the left of the second task',
47+
);
48+
await agent.aiAct(
49+
"click the 'completed' status button below the task list",
50+
);
51+
52+
const list = await agent.aiQuery('string[], the complete task list');
53+
expect(list.length).toEqual(1);
54+
55+
await agent.aiAssert(
56+
'Near the bottom of the list, there is a tip shows "1 item left".',
57+
);
58+
59+
const name = await agent.aiString(
60+
'What is the name of the first todo?',
61+
);
62+
console.log('name', name);
63+
64+
const todoCount = await agent.aiNumber(
65+
'How many todos are there in the list?',
66+
);
67+
console.log('todoCount', todoCount);
68+
69+
const isAllCompleted = await agent.aiBoolean(
70+
'Is all todos completed?',
71+
);
72+
console.log('isAllCompleted', isAllCompleted);
73+
74+
const location = await agent.aiLocate(
75+
'What is the location of the first todo?',
76+
);
77+
console.log('location', location);
78+
},
79+
720 * 1000,
80+
);
81+
});

0 commit comments

Comments
 (0)