From a65364f0a538b50cfe99ee3de85c6ba0952b133b Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 17:23:44 +0800 Subject: [PATCH 01/17] feat(ios): add iOS automation support via screen mirroring --- .../docs/en/automate-with-scripts-in-yaml.mdx | 57 ++ .../docs/zh/automate-with-scripts-in-yaml.mdx | 57 ++ examples/README-iOS.md | 112 +++ examples/ios-input-example.yaml | 38 + examples/ios-input-test.ts | 82 ++ examples/ios-input-test.yaml | 46 + examples/ios-yaml-example.yaml | 70 ++ packages/cli/package.json | 1 + packages/cli/src/cli-utils.ts | 25 + packages/cli/src/config-factory.ts | 6 + packages/cli/src/create-yaml-player.ts | 25 +- packages/cli/src/index.ts | 1 + .../core/src/ai-model/prompt/llm-planning.ts | 4 +- packages/core/src/index.ts | 4 + packages/core/src/types.ts | 3 +- packages/core/src/yaml.ts | 23 +- packages/ios/README.md | 307 ++++++ packages/ios/bin/server.js | 41 + packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md | 131 +++ packages/ios/docs/ios-mirroring-guide.md | 315 +++++++ packages/ios/examples/ios-mirroring-demo.js | 232 +++++ packages/ios/getAppWindowRect.scpt | Bin 0 -> 2224 bytes packages/ios/idb/auto_server.py | 472 ++++++++++ packages/ios/modern.config.ts | 16 + packages/ios/package.json | 58 ++ packages/ios/scripts/getAppWindowRect.scpt | 11 + packages/ios/setup.sh | 35 + packages/ios/src/agent/index.ts | 58 ++ packages/ios/src/index.ts | 4 + packages/ios/src/page/index.ts | 871 ++++++++++++++++++ packages/ios/src/utils/index.ts | 106 +++ packages/ios/tests/ios-input-test.ts | 56 ++ packages/ios/tests/unit-test/agent.test.ts | 17 + packages/ios/tests/unit-test/utils.test.ts | 18 + packages/ios/tsconfig.json | 23 + packages/ios/vitest.config.ts | 7 + packages/web-integration/src/common/tasks.ts | 5 +- packages/web-integration/src/yaml/utils.ts | 25 +- .../tests/unit-test/yaml/utils.test.ts | 29 + pnpm-lock.yaml | 224 ++++- 40 files changed, 3590 insertions(+), 25 deletions(-) create mode 100644 examples/README-iOS.md create mode 100644 examples/ios-input-example.yaml create mode 100644 examples/ios-input-test.ts create mode 100644 examples/ios-input-test.yaml create mode 100644 examples/ios-yaml-example.yaml create mode 100644 packages/ios/README.md create mode 100755 packages/ios/bin/server.js create mode 100644 packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md create mode 100644 packages/ios/docs/ios-mirroring-guide.md create mode 100644 packages/ios/examples/ios-mirroring-demo.js create mode 100644 packages/ios/getAppWindowRect.scpt create mode 100644 packages/ios/idb/auto_server.py create mode 100644 packages/ios/modern.config.ts create mode 100644 packages/ios/package.json create mode 100644 packages/ios/scripts/getAppWindowRect.scpt create mode 100755 packages/ios/setup.sh create mode 100644 packages/ios/src/agent/index.ts create mode 100644 packages/ios/src/index.ts create mode 100644 packages/ios/src/page/index.ts create mode 100644 packages/ios/src/utils/index.ts create mode 100644 packages/ios/tests/ios-input-test.ts create mode 100644 packages/ios/tests/unit-test/agent.test.ts create mode 100644 packages/ios/tests/unit-test/utils.test.ts create mode 100644 packages/ios/tsconfig.json create mode 100644 packages/ios/vitest.config.ts diff --git a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx index 6f51625f0..3e6d7b718 100644 --- a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx @@ -80,6 +80,27 @@ tasks: - aiAssert: The results show weather information ``` +Or, to drive an iOS device automation task (requires PyAutoGUI server setup and device mirroring): + +```yaml +ios: + serverPort: 1412 + mirrorConfig: + mirrorX: 100 + mirrorY: 200 + mirrorWidth: 400 + mirrorHeight: 800 + +tasks: + - name: Search for weather + flow: + - ai: Open Safari browser + - ai: Navigate to bing.com + - ai: Search for "today's weather" + - sleep: 3000 + - aiAssert: The results show weather information +``` + Run the script: ```bash @@ -177,6 +198,37 @@ android: output: ``` +### The `ios` part + +```yaml +ios: + # PyAutoGUI server port, optional, defaults to 1412. + serverPort: + + # PyAutoGUI server URL, optional, defaults to http://localhost:1412. + serverUrl: + + # Whether to automatically dismiss keyboard after input, optional, defaults to false. + autoDismissKeyboard: + + # iOS device mirroring configuration for precise targeting operations + mirrorConfig: + # X position of the mirror on the host display + mirrorX: + # Y position of the mirror on the host display + mirrorY: + # Width of the mirror + mirrorWidth: + # Height of the mirror + mirrorHeight: + + # The launch URL or app, optional, defaults to the device's current page. + launch: + + # The path to the JSON file for outputting aiQuery/aiAssert results, optional. + output: +``` + ### The `tasks` part The `tasks` part is an array that defines the steps of the script. Remember to add a `-` before each step to indicate it's an array item. @@ -304,6 +356,11 @@ The command-line tool provides several options to control the execution behavior - `--web.viewportWidth `: Sets the browser viewport width, which will override the `web.viewportWidth` parameter in all script files. - `--web.viewportHeight `: Sets the browser viewport height, which will override the `web.viewportHeight` parameter in all script files. - `--android.deviceId `: Sets the Android device ID, which will override the `android.deviceId` parameter in all script files. +- `--ios.serverPort `: Sets the iOS PyAutoGUI server port, which will override the `ios.serverPort` parameter in all script files. +- `--ios.mirrorX `: Sets the iOS mirror X position, which will override the `ios.mirrorConfig.mirrorX` parameter in all script files. +- `--ios.mirrorY `: Sets the iOS mirror Y position, which will override the `ios.mirrorConfig.mirrorY` parameter in all script files. +- `--ios.mirrorWidth `: Sets the iOS mirror width, which will override the `ios.mirrorConfig.mirrorWidth` parameter in all script files. +- `--ios.mirrorHeight `: Sets the iOS mirror height, which will override the `ios.mirrorConfig.mirrorHeight` parameter in all script files. - `--dotenv-debug`: Sets the debug log for dotenv, disabled by default. - `--dotenv-override`: Sets whether dotenv overrides global environment variables with the same name, disabled by default. diff --git a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx index 13ef5966b..e415c04c9 100644 --- a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx @@ -80,6 +80,27 @@ tasks: - aiAssert: 结果显示天气信息 ``` +或者驱动 iOS 设备的自动化任务（需要设置 PyAutoGUI 服务器和设备镜像） + +```yaml +ios: + serverPort: 1412 + mirrorConfig: + mirrorX: 100 + mirrorY: 200 + mirrorWidth: 400 + mirrorHeight: 800 + +tasks: + - name: 搜索天气 + flow: + - ai: 打开 Safari 浏览器 + - ai: 导航到 bing.com + - ai: 搜索 "今日天气" + - sleep: 3000 + - aiAssert: 结果显示天气信息 +``` + 运行脚本 ```bash @@ -177,6 +198,37 @@ android: output: ``` +### `ios` 部分 + +```yaml +ios: + # PyAutoGUI 服务器端口，可选，默认 1412 + serverPort: + + # PyAutoGUI 服务器 URL，可选，默认 http://localhost:1412 + serverUrl: + + # 输入后是否自动隐藏键盘，可选，默认 false + autoDismissKeyboard: + + # iOS 设备镜像配置，用于精确定位操作 + mirrorConfig: + # 镜像在主显示器上的 X 位置 + mirrorX: + # 镜像在主显示器上的 Y 位置 + mirrorY: + # 镜像的宽度 + mirrorWidth: + # 镜像的高度 + mirrorHeight: + + # 启动 URL 或应用，可选，默认使用设备当前页面 + launch: + + # 输出 aiQuery/aiAssert 结果的 JSON 文件路径，可选 + output: +``` + ### `tasks` 部分 `tasks` 部分是一个数组，定义了脚本执行的步骤。记得在每个步骤前添加 `-` 符号，表明这些步骤是个数组。 @@ -308,6 +360,11 @@ midscene './scripts/**/*.yaml' - `--web.viewportWidth `: 设置浏览器视口宽度，这将覆盖所有脚本文件中的 `web.viewportWidth` 参数。 - `--web.viewportHeight `: 设置浏览器视口高度，这将覆盖所有脚本文件中的 `web.viewportHeight` 参数。 - `--android.deviceId `: 设置安卓设备 ID，这将覆盖所有脚本文件中的 `android.deviceId` 参数。 +- `--ios.serverPort `: 设置 iOS PyAutoGUI 服务器端口，这将覆盖所有脚本文件中的 `ios.serverPort` 参数。 +- `--ios.mirrorX `: 设置 iOS 镜像 X 位置，这将覆盖所有脚本文件中的 `ios.mirrorConfig.mirrorX` 参数。 +- `--ios.mirrorY `: 设置 iOS 镜像 Y 位置，这将覆盖所有脚本文件中的 `ios.mirrorConfig.mirrorY` 参数。 +- `--ios.mirrorWidth `: 设置 iOS 镜像宽度，这将覆盖所有脚本文件中的 `ios.mirrorConfig.mirrorWidth` 参数。 +- `--ios.mirrorHeight `: 设置 iOS 镜像高度，这将覆盖所有脚本文件中的 `ios.mirrorConfig.mirrorHeight` 参数。 - `--dotenv-debug`: 设置 dotenv 的 debug 日志，默认关闭。 - `--dotenv-override`: 设置 dotenv 是否覆盖同名的全局环境变量，默认关闭。 diff --git a/examples/README-iOS.md b/examples/README-iOS.md new file mode 100644 index 000000000..94c754a00 --- /dev/null +++ b/examples/README-iOS.md @@ -0,0 +1,112 @@ +# iOS YAML Automation Examples + +This directory contains examples of using Midscene.js with iOS devices through YAML configuration files. + +## Prerequisites + +1. **PyAutoGUI Server**: You need to have a PyAutoGUI server running on your system to communicate with the iOS device. +2. **iOS Device Mirroring**: Your iOS device should be mirrored to your computer screen (using tools like QuickTime Player, AirServer, or similar). +3. **Midscene CLI**: Install the Midscene CLI tool: `npm install -g @midscene/cli` + +## Configuration + +### Basic iOS Configuration + +```yaml +ios: + # Server configuration (required for iOS automation) + serverPort: 1412 + serverUrl: "http://localhost:1412" + + # Mirror configuration (required for precise targeting) + mirrorConfig: + mirrorX: 100 # X position of the mirrored iOS screen + mirrorY: 200 # Y position of the mirrored iOS screen + mirrorWidth: 414 # Width of the mirrored screen + mirrorHeight: 896 # Height of the mirrored screen +``` + +### Optional Configuration + +```yaml +ios: + # Auto dismiss keyboard after input (optional) + autoDismissKeyboard: true + + # Launch URL or app when starting (optional) + launch: "https://example.com" + + # Output file for results (optional) + output: "./results.json" +``` + +## Examples + +### 1. Simple iOS Test (`ios-yaml-example.yaml`) + +A basic example showing iOS automation with Safari browser interaction. + +### 2. Comprehensive Example (`ios-comprehensive-example.yaml`) + +A more complex example demonstrating: +- Safari navigation +- Search functionality +- Data extraction +- Settings app interaction +- Home screen operations + +### 3. Configuration File (`ios-config.yaml`) + +Shows how to use a configuration file to set global iOS settings for multiple test scripts. + +## Running the Examples + +### Single Script + +```bash +# Run a single iOS automation script +midscene ./ios-yaml-example.yaml +``` + +### Multiple Scripts with Configuration + +```bash +# Run multiple scripts using a configuration file +midscene --config ./ios-config.yaml +``` + +### Command Line Options + +You can override iOS settings from the command line: + +```bash +# Override mirror settings +midscene --ios.mirrorX 150 --ios.mirrorY 250 ./ios-yaml-example.yaml + +# Override server port +midscene --ios.serverPort 1413 ./ios-yaml-example.yaml +``` + +## Mirror Configuration Setup + +1. **Connect your iOS device** to your computer +2. **Enable mirroring** (e.g., using QuickTime Player's "New Movie Recording" and select your iOS device) +3. **Measure the mirror position and size** on your computer screen +4. **Update the mirrorConfig** values in your YAML file: + - `mirrorX` and `mirrorY`: Top-left corner coordinates of the mirrored screen + - `mirrorWidth` and `mirrorHeight`: Dimensions of the mirrored screen + +## Tips + +- Make sure the PyAutoGUI server is running before executing the scripts +- Adjust the `sleep` durations based on your device's performance +- Test the mirror configuration with simple actions first +- Use descriptive prompts in `aiAction` for better AI understanding +- The `aiAssert` statements help verify that actions completed successfully + +## Troubleshooting + +- **Connection issues**: Verify the PyAutoGUI server is running on the specified port +- **Targeting issues**: Double-check your mirror configuration coordinates +- **Performance issues**: Increase sleep durations between actions +- **Recognition issues**: Use more descriptive text in your AI prompts diff --git a/examples/ios-input-example.yaml b/examples/ios-input-example.yaml new file mode 100644 index 000000000..fa26f528e --- /dev/null +++ b/examples/ios-input-example.yaml @@ -0,0 +1,38 @@ +# iOS automation with YAML script example +# This example shows how to automate iOS devices using PyAutoGUI server + +ios: + # PyAutoGUI server configuration + serverUrl: "http://localhost:1412" + + # Auto dismiss keyboard after input (optional) + autoDismissKeyboard: false + + # iOS device mirroring configuration for precise location targeting + # These values define the position and size of the mirrored device screen + mirrorConfig: + mirrorX: 692 # X position of iOS mirror on computer screen + mirrorY: 161 # Y position of iOS mirror on computer screen + mirrorWidth: 344 # Width of the mirrored iOS screen + mirrorHeight: 764 # Height of the mirrored iOS screen (iPhone 11 Pro size) + + # Output file for aiQuery/aiAssert results (optional) + output: "./results.json" + +tasks: + - name: Open music app and search Coldplay + flow: + - sleep: 5000 + - aiAction: "打开音乐应用" + - sleep: 2000 + - aiTap: "搜索按钮" + - sleep: 3000 + - aiInput: "Coldplay" + locate: "Search box" + - sleep: 2000 + - aiKeyboardPress: "Enter" + - sleep: 3000 + - aiWaitFor: "Search results are displayed" + - aiAction: "随机播放一首歌曲" + - sleep: 3000 + - aiAction: "返回Home" \ No newline at end of file diff --git a/examples/ios-input-test.ts b/examples/ios-input-test.ts new file mode 100644 index 000000000..9c844dac9 --- /dev/null +++ b/examples/ios-input-test.ts @@ -0,0 +1,82 @@ +#!/usr/bin/env tsx +/** + * iOS Input Test - Demonstrates the improved iOS input functionality + * + * This test shows how the iOS input system now automatically handles: + * - Element focusing by tapping + * - Content clearing with cmd+a and delete + * - Optimized typing with proper intervals for iOS keyboards + * - Automatic keyboard dismissal + * + * The beauty is that it all happens transparently - no special iOS methods needed! + */ + +import { agentFromPyAutoGUI } from '../packages/ios/src/agent'; +import type { iOSDeviceOpt } from '../packages/ios/src/page'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function testIOSInput() { + console.log('🚀 Starting iOS Input Test...'); + + // Configure for iOS device mirroring - adjust these coordinates for your setup + const options: iOSDeviceOpt = { + serverPort: 1412, + autoDismissKeyboard: true, + iOSMirrorConfig: { + mirrorX: 692, // X position of iOS mirror on screen + mirrorY: 161, // Y position of iOS mirror on screen + mirrorWidth: 344, // Width of the mirrored iOS screen + mirrorHeight: 764, // Height of the mirrored iOS screen + }, + }; + + try { + // Create agent - this will automatically start the PyAutoGUI server if needed + const agent = await agentFromPyAutoGUI(options); + console.log('✅ iOS Agent created successfully'); + + // Test 1: Simple text input + console.log('\n📝 Test 1: Simple text input using aiInput'); + await agent.aiInput('Hello iOS!', 'search box or text field'); + await sleep(2000); + + // Test 2: Email input with special characters + console.log('\n📧 Test 2: Email input with special characters'); + await agent.aiInput('test@example.com', 'email input field'); + await sleep(2000); + + // Test 3: Multi-word text with spaces + console.log('\n📄 Test 3: Multi-word text input'); + await agent.aiInput( + 'This is a longer text message with spaces', + 'text area or message field', + ); + await sleep(2000); + + // Test 4: Numbers and symbols + console.log('\n🔢 Test 4: Numbers and symbols'); + await agent.aiInput('Password123!@#', 'password field'); + await sleep(2000); + + // Test 5: Clear and replace existing text + console.log('\n🔄 Test 5: Clear and replace existing text'); + await agent.aiInput('', 'input field'); // Clear the field + await sleep(1000); + await agent.aiInput('New replacement text', 'same input field'); + await sleep(2000); + + console.log('\n✅ All iOS input tests completed successfully!'); + console.log('🎉 The iOS input system is working properly with:'); + console.log(' - Automatic element focusing'); + console.log(' - Smart content clearing'); + console.log(' - Optimized typing intervals'); + console.log(' - Automatic keyboard dismissal'); + } catch (error) { + console.error('❌ iOS Input Test failed:', error); + process.exit(1); + } +} + +// Run the test +testIOSInput().catch(console.error); diff --git a/examples/ios-input-test.yaml b/examples/ios-input-test.yaml new file mode 100644 index 000000000..2533027a9 --- /dev/null +++ b/examples/ios-input-test.yaml @@ -0,0 +1,46 @@ +# iOS Input Test - YAML Version +# This demonstrates how the improved iOS input system works seamlessly +# through the standard YAML automation interface + +ios: + serverPort: 1412 + autoDismissKeyboard: true + iOSMirrorConfig: + mirrorX: 692 + mirrorY: 161 + mirrorWidth: 344 + mirrorHeight: 764 + +tasks: + - name: test basic text input + flow: + - aiInput: "Hello from iOS!" + locate: "search box or text field" + - sleep: 2000 + + - name: test email input + flow: + - aiInput: "user@example.com" + locate: "email input field" + - sleep: 2000 + + - name: test password with symbols + flow: + - aiInput: "SecurePass123!@#" + locate: "password field" + - sleep: 2000 + + - name: test clear and replace + flow: + - aiInput: "" # Clear the field + locate: "text input" + - sleep: 1000 + - aiInput: "Replaced text content" + locate: "same text input" + - sleep: 2000 + + - name: test long text + flow: + - aiInput: "This is a much longer text message that tests the iOS keyboard input with multiple words and proper spacing between characters" + locate: "text area or message field" + - sleep: 3000 diff --git a/examples/ios-yaml-example.yaml b/examples/ios-yaml-example.yaml new file mode 100644 index 000000000..b20af69af --- /dev/null +++ b/examples/ios-yaml-example.yaml @@ -0,0 +1,70 @@ +# iOS automation with YAML script example +# This example shows how to automate iOS devices using PyAutoGUI server + +ios: + # PyAutoGUI server configuration + serverUrl: "http://localhost:1412" + + # Auto dismiss keyboard after input (optional) + autoDismissKeyboard: true + + # iOS device mirroring configuration for precise location targeting + # These values define the position and size of the mirrored device screen + mirrorConfig: + mirrorX: 692 # X position of iOS mirror on computer screen + mirrorY: 161 # Y position of iOS mirror on computer screen + mirrorWidth: 344 # Width of the mirrored iOS screen + mirrorHeight: 764 # Height of the mirrored iOS screen (iPhone 11 Pro size) + + # Output file for aiQuery/aiAssert results (optional) + output: "./results.json" + +tasks: + - name: Open Safari and search + flow: + - aiAction: "Open Safari browser" + - sleep: 2000 + - aiAction: "Navigate to google.com" + - sleep: 3000 + - aiAction: "Type 'iOS automation' in the search box" + - aiAction: "Tap the search button" + - sleep: 5000 + - aiAssert: "Search results are displayed" + + - name: Extract search results + flow: + - aiQuery: > + {title: string, description: string}[], + return the titles and descriptions of the first 3 search results + name: search_results + + - name: Navigate to settings + flow: + - aiAction: "Open Settings app" + - sleep: 2000 + - aiAction: "Scroll down to find General settings" + - aiAction: "Tap on General" + - sleep: 1000 + - aiAssert: "General settings page is displayed" + + - name: Test iOS input functionality + flow: + - aiAction: "Open Notes app" + - sleep: 2000 + - aiAction: "Tap on new note button or create new note" + - sleep: 1000 + - aiInput: "This is a test note created with iOS automation. The input should work properly with character-by-character typing and keyboard dismissal." + - sleep: 1000 + - aiAction: "Save the note" + - aiAssert: "Note is saved successfully" + + - name: Test iOS search input + flow: + - aiAction: "Go to iOS home screen" + - sleep: 1000 + - aiAction: "Swipe down to open search" + - sleep: 1000 + - aiInput: "Settings" + - sleep: 2000 + - aiAction: "Tap on Settings in search results" + - aiAssert: "Settings app is opened" diff --git a/packages/cli/package.json b/packages/cli/package.json index 000963c8b..d10858826 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,6 +23,7 @@ "dependencies": { "@midscene/android": "workspace:*", "@midscene/core": "workspace:*", + "@midscene/ios": "workspace:*", "@midscene/shared": "workspace:*", "@midscene/web": "workspace:*", "http-server": "14.1.1", diff --git a/packages/cli/src/cli-utils.ts b/packages/cli/src/cli-utils.ts index 8cf58be9b..5fd8f9252 100644 --- a/packages/cli/src/cli-utils.ts +++ b/packages/cli/src/cli-utils.ts @@ -89,6 +89,31 @@ Usage: type: 'string', description: 'Override device ID for Android environments.', }, + 'ios.server-port': { + alias: 'ios.serverPort', + type: 'number', + description: 'Override PyAutoGUI server port for iOS environments.', + }, + 'ios.mirror-x': { + alias: 'ios.mirrorConfig.mirrorX', + type: 'number', + description: 'Override mirror X position for iOS environments.', + }, + 'ios.mirror-y': { + alias: 'ios.mirrorConfig.mirrorY', + type: 'number', + description: 'Override mirror Y position for iOS environments.', + }, + 'ios.mirror-width': { + alias: 'ios.mirrorConfig.mirrorWidth', + type: 'number', + description: 'Override mirror width for iOS environments.', + }, + 'ios.mirror-height': { + alias: 'ios.mirrorConfig.mirrorHeight', + type: 'number', + description: 'Override mirror height for iOS environments.', + }, }) .version('version', 'Show version number', __VERSION__) .help() diff --git a/packages/cli/src/config-factory.ts b/packages/cli/src/config-factory.ts index 606cd03ed..19a8d037c 100644 --- a/packages/cli/src/config-factory.ts +++ b/packages/cli/src/config-factory.ts @@ -4,6 +4,7 @@ import { cwd } from 'node:process'; import type { MidsceneYamlConfig, MidsceneYamlScriptAndroidEnv, + MidsceneYamlScriptIOSEnv, MidsceneYamlScriptWebEnv, } from '@midscene/core'; import { interpolateEnvVars } from '@midscene/web/yaml'; @@ -33,6 +34,7 @@ export interface ConfigFactoryOptions { dotenvDebug?: boolean; web?: Partial; android?: Partial; + ios?: Partial; } export interface ParsedConfig { @@ -42,6 +44,7 @@ export interface ParsedConfig { shareBrowserContext: boolean; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; + ios?: MidsceneYamlScriptIOSEnv; target?: MidsceneYamlScriptWebEnv; files: string[]; patterns: string[]; // Keep patterns for reference @@ -118,6 +121,7 @@ export async function parseConfigYaml( configYaml.shareBrowserContext ?? defaultConfig.shareBrowserContext, web: configYaml.web, android: configYaml.android, + ios: configYaml.ios, patterns: configYaml.files, files, headed: configYaml.headed ?? defaultConfig.headed, @@ -138,11 +142,13 @@ export async function createConfig( { web: parsedConfig.web, android: parsedConfig.android, + ios: parsedConfig.ios, target: parsedConfig.target, }, { web: options?.web, android: options?.android, + ios: options?.ios, }, ); diff --git a/packages/cli/src/create-yaml-player.ts b/packages/cli/src/create-yaml-player.ts index 5f2c14ae1..916f46198 100644 --- a/packages/cli/src/create-yaml-player.ts +++ b/packages/cli/src/create-yaml-player.ts @@ -10,6 +10,7 @@ import type { MidsceneYamlScript, MidsceneYamlScriptEnv, } from '@midscene/core'; +import { agentFromPyAutoGUI } from '@midscene/ios'; import { AgentOverChromeBridge } from '@midscene/web/bridge-mode'; import { puppeteerAgentForTarget } from '@midscene/web/puppeteer-agent-launcher'; import type { Browser } from 'puppeteer'; @@ -161,8 +162,30 @@ export async function createYamlPlayer( return { agent, freeFn }; } + // handle ios + if (typeof yamlScript.ios !== 'undefined') { + const iosTarget = yamlScript.ios; + const agent = await agentFromPyAutoGUI({ + serverUrl: iosTarget.serverUrl, + serverPort: iosTarget.serverPort, + autoDismissKeyboard: iosTarget.autoDismissKeyboard, + iOSMirrorConfig: iosTarget.mirrorConfig, + }); + + if (iosTarget?.launch) { + await agent.launch(iosTarget.launch); + } + + freeFn.push({ + name: 'destroy_ios_agent', + fn: () => agent.destroy(), + }); + + return { agent, freeFn }; + } + throw new Error( - 'No valid target configuration found in the yaml script, should be either "web" or "android"', + 'No valid target configuration found in the yaml script, should be either "web", "android", or "ios"', ); }, undefined, diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 4a9c2eee6..f5f02fc2c 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -39,6 +39,7 @@ Promise.resolve( dotenvDebug: options['dotenv-debug'], web: options.web, android: options.android, + ios: options.ios, }; let config; diff --git a/packages/core/src/ai-model/prompt/llm-planning.ts b/packages/core/src/ai-model/prompt/llm-planning.ts index 7586c12f8..aaf5d6856 100644 --- a/packages/core/src/ai-model/prompt/llm-planning.ts +++ b/packages/core/src/ai-model/prompt/llm-planning.ts @@ -25,7 +25,7 @@ Target: User will give you a screenshot, an instruction and some previous logs i Restriction: - Don't give extra actions or plans beyond the instruction. ONLY plan for what the instruction requires. For example, don't try to submit the form if the instruction is only to fill something. -- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === 'android' ? ', AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull.' : '.'} +- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === 'android' || pageType === 'ios' ? ', AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull.' : '.'} - Don't repeat actions in the previous logs. - Bbox is the bounding box of the element to be located. It's an array of 4 numbers, representing ${bboxDescription(vlMode)}. @@ -37,7 +37,7 @@ Supporting actions: - KeyboardPress: { type: "KeyboardPress", param: { value: string } } - Scroll: { type: "Scroll", ${vlLocateParam} | null, param: { direction: 'down'(default) | 'up' | 'right' | 'left', scrollType: 'once' (default) | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft', distance: null | number }} // locate is the element to scroll. If it's a page scroll, put \`null\` in the \`locate\` field. ${ - pageType === 'android' + pageType === 'android' || pageType === 'ios' ? `- AndroidBackButton: { type: "AndroidBackButton", param: {} } - AndroidHomeButton: { type: "AndroidHomeButton", param: {} } - AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 71e4946b9..48d760653 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -21,4 +21,8 @@ export type { MidsceneYamlFlowItem, MidsceneYamlFlowItemAIRightClick, MidsceneYamlConfigResult, + MidsceneYamlScriptWebEnv, + MidsceneYamlScriptAndroidEnv, + MidsceneYamlScriptIOSEnv, + MidsceneYamlConfig, } from './yaml'; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index c39b5d576..5ca59be2f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -561,7 +561,8 @@ export type PageType = | 'playwright' | 'static' | 'chrome-extension-proxy' - | 'android'; + | 'android' + | 'ios'; export interface StreamingCodeGenerationOptions { /** Whether to enable streaming output */ diff --git a/packages/core/src/yaml.ts b/packages/core/src/yaml.ts index ac111fbb0..d91ed8e9a 100644 --- a/packages/core/src/yaml.ts +++ b/packages/core/src/yaml.ts @@ -32,6 +32,7 @@ export interface MidsceneYamlScript { target?: MidsceneYamlScriptWebEnv; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; + ios?: MidsceneYamlScriptIOSEnv; tasks: MidsceneYamlTask[]; } @@ -79,9 +80,28 @@ export interface MidsceneYamlScriptAndroidEnv launch?: string; } +export interface MidsceneYamlScriptIOSEnv extends MidsceneYamlScriptEnvBase { + // The URL or app to launch, optional, will use the current screen if not specified + launch?: string; + + // PyAutoGUI server configuration + serverUrl?: string; + serverPort?: number; + autoDismissKeyboard?: boolean; + + // iOS device mirroring configuration to define the mirror position and size + mirrorConfig?: { + mirrorX: number; + mirrorY: number; + mirrorWidth: number; + mirrorHeight: number; + }; +} + export type MidsceneYamlScriptEnv = | MidsceneYamlScriptWebEnv - | MidsceneYamlScriptAndroidEnv; + | MidsceneYamlScriptAndroidEnv + | MidsceneYamlScriptIOSEnv; export interface MidsceneYamlFlowItemAIAction { ai?: string; // this is the shortcut for aiAction @@ -209,6 +229,7 @@ export interface MidsceneYamlConfig { shareBrowserContext?: boolean; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; + ios?: MidsceneYamlScriptIOSEnv; files: string[]; headed?: boolean; keepWindow?: boolean; diff --git a/packages/ios/README.md b/packages/ios/README.md new file mode 100644 index 000000000..836b8b73f --- /dev/null +++ b/packages/ios/README.md @@ -0,0 +1,307 @@ +# @midscene/ios + +iOS automation package for Midscene.js with coordinate mapping support for iOS device mirroring. + +## Features + +- **iOS Device Mirroring**: Control iOS devices through screen mirroring on macOS +- **Coordinate Mapping**: Automatic transformation from iOS coordinates to macOS screen coordinates +- **AI Integration**: Use natural language to interact with iOS interfaces +- **Screenshot Capture**: Take region-specific screenshots of iOS mirrors +- **PyAutoGUI Backend**: Reliable macOS system control through Python server + +## Installation + +```bash +npm install @midscene/ios +``` + +## Prerequisites + +1. **Python 3** with required packages: + + ```bash + pip3 install flask pyautogui + ``` + +2. **macOS Accessibility Permissions**: + - Go to System Preferences → Security & Privacy → Privacy → Accessibility + - Add your terminal application to the list + - Required for PyAutoGUI to control mouse and keyboard + +3. **iOS Device Mirroring**: + - iPhone Mirroring (macOS Sequoia) + +## Getting iPhone Mirroring Window Coordinates + +To use iOS automation, you need to determine where the iPhone Mirroring window is positioned on your macOS screen. We provide a helpful AppleScript that automatically detects this for you. + +### Using the AppleScript + +```bash +# Navigate to the iOS package directory +cd packages/ios + +# Run the script to get window coordinates +osascript scripts/getAppWindowRect.scpt +``` + +**Important**: The script gives you 4 seconds to make the iPhone Mirroring app the foreground window before it captures the coordinates. + +The output will look like: + +```text +{"iPhone Mirroring", {692, 161}, {344, 764}} +``` + +This means: + +- App name: "iPhone Mirroring" +- Position: x=692, y=161 (use these for `mirrorX` and `mirrorY`) +- Size: width=344, height=764 (use these for `mirrorWidth` and `mirrorHeight`) + +## Quick Start + +### 1. Start PyAutoGUI Server + +```bash +cd packages/ios/idb +python3 auto_server.py 1412 +``` + +### 2. Configure iOS Mirroring + +First, get the mirror window coordinates using the AppleScript mentioned above, then: + +```typescript +import { iOSDevice, iOSAgent } from '@midscene/ios'; + +const device = new iOSDevice({ + serverPort: 1412, + mirrorConfig: { + mirrorX: 692, // Mirror position on macOS screen + mirrorY: 161, + mirrorWidth: 344, // Mirror size on macOS screen + mirrorHeight: 764 + } +}); + +await device.connect(); +const agent = new iOSAgent(device); + +// AI interactions with automatic coordinate mapping +await agent.aiTap('Settings app'); +await agent.aiInput('Wi-Fi', 'Search settings'); +const settings = await agent.aiQuery('string[], visible settings'); +``` + +### 3. Basic Device Control + +```typescript +// Direct coordinate operations +await device.tap({ left: 100, top: 200 }); +await device.input('Hello', { left: 150, top: 300 }); +await device.scroll({ direction: 'down', distance: 200 }); + +// Screenshots (automatically crops to iOS mirror region) +const screenshot = await device.screenshotBase64(); +``` + +## API Reference + +### agentFromPyAutoGUI(options?) + +Creates an iOS agent with PyAutoGUI backend. + +**Options:** + +- `serverUrl?: string` - Custom server URL (default: `http://localhost:1412`) +- `serverPort?: number` - Server port (default: `1412`) +- `autoDismissKeyboard?: boolean` - Auto dismiss keyboard (not applicable for desktop) + +### iOSDevice Methods + +#### `launch(uri: string): Promise` + +Launch an application or URL. + +- For URLs: `await device.launch('https://example.com')` +- For apps: `await device.launch('Safari')` + +#### `size(): Promise` + +Get screen dimensions and pixel ratio. + +#### `screenshotBase64(): Promise` + +Take a screenshot and return as base64 string. + +#### `tap(point: Point): Promise` + +Click at the specified coordinates. + +#### `hover(point: Point): Promise` + +Move mouse to the specified coordinates. + +#### `input(text: string): Promise` + +Type text using the keyboard. + +#### `keyboardPress(key: string): Promise` + +Press a specific key. Supported keys: + +- `'Return'`, `'Enter'` - Enter key +- `'Tab'` - Tab key +- `'Space'` - Space bar +- `'Backspace'` - Backspace +- `'Delete'` - Delete key +- `'Escape'` - Escape key + +#### `scroll(options: ScrollOptions): Promise` + +Scroll in the specified direction. + +**ScrollOptions:** + +- `direction: 'up' | 'down' | 'left' | 'right'` +- `distance?: number` - Scroll distance in pixels (default: 100) + +## PyAutoGUI Server API + +The Python server accepts POST requests to `/run` with JSON payloads: + +### Supported Actions + +#### Click + +```json +{ + "action": "click", + "x": 100, + "y": 100 +} +``` + +#### Move (Hover) + +```json +{ + "action": "move", + "x": 200, + "y": 200, + "duration": 0.2 +} +``` + +#### Drag + +```json +{ + "action": "drag", + "x": 100, + "y": 100, + "x2": 200, + "y2": 200, + "duration": 0.5 +} +``` + +#### Type + +```json +{ + "action": "type", + "text": "Hello World", + "interval": 0.0 +} +``` + +#### Key Press + +```json +{ + "action": "key", + "key": "return" +} +``` + +#### Hotkey Combination + +```json +{ + "action": "hotkey", + "keys": ["cmd", "c"] +} +``` + +#### Scroll + +```json +{ + "action": "scroll", + "x": 400, + "y": 300, + "clicks": 3 +} +``` + +#### Sleep + +```json +{ + "action": "sleep", + "seconds": 1.0 +} +``` + +### Health Check + +GET `/health` - Returns server status and screen information. + +## Architecture + +```text +┌─────────────────┐ HTTP ┌─────────────────┐ PyAutoGUI ┌─────────────────┐ +│ TypeScript │ ────> │ Python Server │ ─────────> │ macOS System │ +│ iOS Agent │ │ (Flask + PyAutoGUI) │ │ (Mouse/Keyboard) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +## Troubleshooting + +### Accessibility Permissions + +If you get permission errors, ensure your terminal has accessibility permissions: + +1. System Preferences → Security & Privacy → Privacy +2. Select "Accessibility" from the left sidebar +3. Click the lock to make changes +4. Add your terminal application to the list + +### Python Dependencies + +```bash +# Install required Python packages +pip3 install flask pyautogui + +# On macOS, you might also need: +pip3 install pillow +``` + +### Port Already in Use + +If port 1412 is already in use, specify a different port: + +```typescript +const agent = await agentFromPyAutoGUI({ serverPort: 1413 }); +``` + +## Example + +See `examples/ios-mirroring-demo.js` for a complete usage example. + +## License + +MIT diff --git a/packages/ios/bin/server.js b/packages/ios/bin/server.js new file mode 100755 index 000000000..3986753ee --- /dev/null +++ b/packages/ios/bin/server.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const serverPath = join(__dirname, '../idb/auto_server.py'); + +const port = process.argv[2] || '1412'; + +console.log(`Starting PyAutoGUI server on port ${port}...`); + +const server = spawn('python3', [serverPath, port], { + stdio: 'inherit', + env: { + ...process.env, + PYTHONUNBUFFERED: '1', + }, +}); + +server.on('error', (error) => { + console.error('Failed to start PyAutoGUI server:', error); + process.exit(1); +}); + +server.on('close', (code) => { + console.log(`PyAutoGUI server exited with code ${code}`); + process.exit(code || 0); +}); + +// Handle graceful shutdown +process.on('SIGINT', () => { + console.log('\nShutting down PyAutoGUI server...'); + server.kill('SIGINT'); +}); + +process.on('SIGTERM', () => { + console.log('\nShutting down PyAutoGUI server...'); + server.kill('SIGTERM'); +}); diff --git a/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md b/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md new file mode 100644 index 000000000..ce2b94d61 --- /dev/null +++ b/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md @@ -0,0 +1,131 @@ +# iOS Input Implementation Improvements + +## 问题描述 +iOS的aiInput之前存在问题，因为Android可以调用adb直接输入，但iOS只能通过模拟键盘输入来实现。这导致了输入不稳定、字符丢失等问题。 + +## 解决方案 + +### 1. 优化的输入流程 +新的iOS输入实现包含以下步骤： + +1. **聚焦输入框**: 先点击输入框来获得焦点 +2. **清空现有内容**: 使用Cmd+A选择全部，然后删除 +3. **字符间隔输入**: 使用适当的间隔时间逐字符输入 +4. **自动关闭键盘**: 输入完成后自动关闭键盘 + +### 2. Python服务器改进 +- 添加了`ios_input`动作类型，专门处理iOS输入 +- 支持`interval`参数，控制字符间输入间隔 +- 为iOS设置默认的最小间隔时间(20ms)以确保字符正确输入 + +### 3. TypeScript实现改进 +- 添加了`aiInputIOS`方法，提供专门的iOS输入处理 +- 更新了`clearInput`方法，先点击聚焦再清空 +- 添加了`dismissKeyboard`方法，自动关闭iOS键盘 +- 在任务执行器中添加了iOS特定的输入逻辑 + +## 新增功能 + +### PyAutoGUI服务器新动作 +```python +{ + "action": "ios_input", + "x": 100, # 输入框的x坐标（可选） + "y": 200, # 输入框的y坐标（可选） + "text": "Hello", # 要输入的文本 + "interval": 0.05, # 字符间间隔（秒） + "clear_first": true # 是否先清空现有内容 +} +``` + +### iOS设备新方法 +```typescript +// 专门的iOS输入方法 +await iosDevice.aiInputIOS(text, element, options); + +// 改进的键盘关闭方法 +await iosDevice.dismissKeyboard(); + +// 改进的清空输入方法 +await iosDevice.clearInput(element); +``` + +## 使用示例 + +### YAML配置示例 +```yaml +ios: + serverUrl: "http://localhost:1412" + autoDismissKeyboard: true + mirrorConfig: + mirrorX: 692 + mirrorY: 161 + mirrorWidth: 344 + mirrorHeight: 764 + +tasks: + - name: Test iOS input + flow: + - aiAction: "Open Notes app" + - aiInput: "This text will be input properly on iOS" + - aiAssert: "Text is entered correctly" +``` + +### 编程接口示例 +```typescript +const agent = await agentFromPyAutoGUI({ + serverPort: 1412, + autoDismissKeyboard: true, + mirrorConfig: { + mirrorX: 692, + mirrorY: 161, + mirrorWidth: 344, + mirrorHeight: 764, + }, +}); + +// 使用aiInput，现在对iOS优化 +await agent.aiInput('Hello iOS!', 'in the text input field'); +``` + +## 技术改进细节 + +### 1. 字符间隔控制 +- 默认间隔: 20ms (Android的adb输入不需要间隔) +- 可配置间隔: 通过`interval`参数自定义 +- 自动调整: 为iOS设备自动设置合适的默认值 + +### 2. 聚焦处理 +- 自动点击: 在输入前先点击输入框获得焦点 +- 等待时间: 点击后等待300ms确保焦点获得 +- 坐标转换: 自动处理iOS到macOS的坐标转换 + +### 3. 键盘管理 +- 自动关闭: 输入完成后自动关闭iOS键盘 +- 多种方法: 尝试Return键或点击键盘外区域 +- 可配置: 通过`autoDismissKeyboard`选项控制 + +### 4. 错误处理 +- 优雅降级: 如果特殊方法失败，回退到基本方法 +- 日志记录: 详细的调试日志帮助排查问题 +- 类型安全: 通过TypeScript类型检查确保正确使用 + +## 测试 +运行测试脚本验证功能： +```bash +cd packages/ios +npm run test:input +``` + +## 兼容性 +- ✅ iOS模拟器 +- ✅ iOS真机（通过屏幕镜像） +- ✅ 向后兼容现有API +- ✅ 支持中文输入 +- ✅ 支持特殊字符 + +## 性能优化 +- 减少不必要的延迟 +- 优化字符输入速度 +- 智能键盘关闭策略 +- 高效的坐标转换 diff --git a/packages/ios/docs/ios-mirroring-guide.md b/packages/ios/docs/ios-mirroring-guide.md new file mode 100644 index 000000000..7ea4dda7c --- /dev/null +++ b/packages/ios/docs/ios-mirroring-guide.md @@ -0,0 +1,315 @@ +# iOS Device Mirroring with Coordinate Mapping + +This document explains how to use Midscene.js with iOS device mirroring through macOS, including automatic coordinate transformation for accurate touch events. + +## Overview + +iOS device mirroring allows you to control iOS devices through their screen representation on macOS. This is useful for: + +- **App Testing**: Automated testing of iOS apps without physical interaction +- **Screen Recording**: Capture iOS interactions for documentation +- **Remote Control**: Control iOS devices through macOS automation +- **CI/CD Integration**: Automated iOS testing in continuous integration + +## Prerequisites + +1. **Python Dependencies**: + ```bash + pip3 install flask pyautogui + ``` + +2. **iOS Device Mirroring Setup** (choose one): + - **QuickTime Player**: Connect iOS device → File → New Movie Recording → Select iOS device + - **iPhone Mirroring** (macOS Sequoia): Built-in iOS mirroring feature + - **iOS Simulator**: Xcode's iOS Simulator + - **Third-party tools**: Reflector, AirServer, etc. + +3. **Screen Position**: Note the exact position and size of iOS mirror on your macOS screen + +## Configuration + +### 1. Basic Setup + +```typescript +import { iOSDevice, iOSAgent } from '@midscene/ios'; + +const device = new iOSDevice({ + serverPort: 1412, + iOSMirrorConfig: { + mirrorX: 100, // Mirror position X on macOS screen + mirrorY: 50, // Mirror position Y on macOS screen + mirrorWidth: 400, // Mirror width on macOS screen + mirrorHeight: 800 // Mirror height on macOS screen + } +}); + +await device.connect(); +const agent = new iOSAgent(device); +``` + +### 2. Finding Mirror Coordinates + +**Method 1: Manual Measurement** +1. Position iOS mirror window on your screen +2. Use macOS's built-in screenshot tool to measure: + - Press `Cmd + Shift + 4` + - Drag from top-left to bottom-right of iOS mirror + - Note the coordinates and dimensions + +**Method 2: Using Digital Color Meter** +1. Open Digital Color Meter (Applications → Utilities) +2. Move cursor to top-left corner of iOS mirror → note coordinates +3. Move cursor to bottom-right corner → calculate width/height + +**Method 3: Programmatic Detection** (Advanced) +```python +# Use this Python script to help find iOS mirror region +import pyautogui +import cv2 +import numpy as np + +def find_ios_mirror(): + # Take screenshot + screenshot = pyautogui.screenshot() + + # Convert to OpenCV format + img = cv2.cvtColor(np.array(screenshot), cv2.COLOR_RGB2BGR) + + # Look for iOS-specific UI patterns (status bar, home indicator, etc.) + # This is a simplified example - real implementation would be more complex + + # Return detected region + return {"x": 100, "y": 50, "width": 400, "height": 800} +``` + +## Coordinate Transformation + +The system automatically transforms iOS coordinates to macOS screen coordinates: + +### Transformation Formula +``` +macOS_x = mirror_x + (iOS_x × scale_x) +macOS_y = mirror_y + (iOS_y × scale_y) + +where: +scale_x = mirror_width / ios_width +scale_y = mirror_height / ios_height +``` + +### Example +``` +iOS Device: 393×852 (iPhone 15 Pro) +Mirror Region: (100, 50) with size 400×800 + +iOS coordinate (100, 200) transforms to: +macOS_x = 100 + (100 × 400/393) = 100 + 101.8 = ~202 +macOS_y = 50 + (200 × 800/852) = 50 + 187.8 = ~238 +``` + +## Usage Examples + +### Basic Touch Operations + +```typescript +// Tap at iOS coordinates - automatically transformed +await device.tap({ left: 100, top: 200 }); + +// Drag gesture +await device.drag( + { left: 100, top: 300 }, // Start point + { left: 300, top: 300 } // End point +); + +// Scroll +await device.scroll({ + direction: 'down', + startPoint: { left: 200, top: 400 }, + distance: 200 +}); +``` + +### AI-Powered Automation + +```typescript +// AI can understand iOS interface elements +await agent.aiTap('Settings app icon'); +await agent.aiInput('Wi-Fi', 'Search bar'); +await agent.aiScroll({ direction: 'down', scrollType: 'once' }); + +// Extract data from iOS interface +const appList = await agent.aiQuery('string[], visible app names on home screen'); + +// Verify iOS interface state +await agent.aiAssert('Control Center is open'); +``` + +### Screenshots + +```typescript +// Takes screenshot of iOS mirror region only +const screenshot = await device.screenshotBase64(); + +// Screenshot is automatically cropped to iOS mirror area +// Perfect for AI analysis of iOS interface +``` + +## Common Device Information (For Reference) + +Note: These logical resolutions are automatically detected by the system. You only need to configure the mirror position and size on your macOS screen. + +### iPhone Models +```typescript +// iPhone 15 Pro / 14 Pro: 393 x 852 logical pixels +// iPhone 15 Plus / 14 Plus: 428 x 926 logical pixels +// iPhone SE (3rd generation): 375 x 667 logical pixels +// iPhone 15 / 14: 393 x 852 logical pixels +``` + +### iPad Models +```typescript +// iPad Pro 12.9" (6th generation): 1024 x 1366 logical pixels +// iPad Pro 11" (4th generation): 834 x 1194 logical pixels +// iPad Air (5th generation): 820 x 1180 logical pixels +``` + +## Best Practices + +### 1. Calibration Testing +```typescript +// Always test coordinate accuracy first +const testPoints = [ + { left: 50, top: 50 }, // Top-left corner + { left: 196, top: 426 }, // Center (for iPhone 15 Pro) + { left: 343, top: 802 } // Bottom-right corner +]; + +for (const point of testPoints) { + await device.tap(point); + await new Promise(resolve => setTimeout(resolve, 1000)); +} +``` + +### 2. Handle Different Mirror Sizes +```typescript +// Support multiple mirror configurations +const configs = { + small: { mirrorWidth: 300, mirrorHeight: 600 }, + medium: { mirrorWidth: 400, mirrorHeight: 800 }, + large: { mirrorWidth: 500, mirrorHeight: 1000 } +}; + +// Choose based on screen size or user preference +const config = configs.medium; +``` + +### 3. Error Handling +```typescript +try { + await device.connect(); +} catch (error) { + if (error.message.includes('Python server')) { + console.error('Start Python server: python3 auto_server.py 1412'); + } else if (error.message.includes('configuration')) { + console.error('Check mirror coordinates and iOS device size'); + } + throw error; +} +``` + +### 4. Performance Optimization +```typescript +// Batch operations for better performance +const actions = [ + { action: 'click', x: 100, y: 200 }, + { action: 'sleep', seconds: 0.5 }, + { action: 'click', x: 200, y: 300 } +]; + +// All actions use coordinate transformation +await device.executeBatchActions(actions); +``` + +## Troubleshooting + +### Common Issues + +**1. Coordinate Misalignment** +- **Problem**: Taps don't hit intended targets +- **Solution**: Re-measure mirror position and size, ensure iOS device orientation is correct + +**2. Python Server Connection Failed** +- **Problem**: `Failed to connect to Python server` +- **Solution**: Start server with `python3 auto_server.py 1412`, check firewall settings + +**3. Screenshots Show Wrong Region** +- **Problem**: Screenshots include macOS desktop instead of iOS mirror +- **Solution**: Verify mirror coordinates, ensure iOS window is not minimized + +**4. Scale Factor Issues** +- **Problem**: Coordinates are consistently off by same ratio +- **Solution**: Double-check iOS device logical resolution vs mirror size + +### Debug Tools + +```typescript +// Check current configuration +const config = await device.getConfiguration(); +console.log('Current mapping:', config); + +// Test coordinate transformation +const testCoord = { left: 100, top: 200 }; +console.log('iOS:', testCoord); +// Tap will show transformed macOS coordinates in logs +await device.tap(testCoord); +``` + +## Advanced Features + +### Dynamic Reconfiguration +```typescript +// Change mirror configuration at runtime +await device.configureIOSMirror({ + mirrorX: 200, // New position + mirrorY: 100, + mirrorWidth: 450, // New size for different device + mirrorHeight: 950 +}); +``` + +### Multiple Device Support +```typescript +// Control multiple iOS devices +const device1 = new iOSDevice({ + serverPort: 1412, + iOSMirrorConfig: config1 +}); + +const device2 = new iOSDevice({ + serverPort: 1413, // Different server instance + iOSMirrorConfig: config2 +}); +``` + +### Integration with Test Frameworks +```typescript +// Jest/Vitest example +describe('iOS App Tests', () => { + let device, agent; + + beforeAll(async () => { + device = new iOSDevice({ /* config */ }); + await device.connect(); + agent = new iOSAgent(device); + }); + + test('should login successfully', async () => { + await agent.aiTap('Login button'); + await agent.aiInput('user@example.com', 'Email field'); + await agent.aiInput('password123', 'Password field'); + await agent.aiTap('Sign in button'); + await agent.aiAssert('Dashboard is visible'); + }); +}); +``` + +This coordinate mapping system makes iOS device automation through macOS screen mirroring seamless and accurate, enabling powerful AI-driven testing and automation workflows. diff --git a/packages/ios/examples/ios-mirroring-demo.js b/packages/ios/examples/ios-mirroring-demo.js new file mode 100644 index 000000000..d3e201e0b --- /dev/null +++ b/packages/ios/examples/ios-mirroring-demo.js @@ -0,0 +1,232 @@ +#!/usr/bin/env node + +/** + * Complete example showing how to use iOS device mirroring with coordinate mapping and enhanced scrolling + * + * This example demonstrates: + * 1. Setting up iOS device mirroring configuration + * 2. Using coordinate transformation for accurate touch events + * 3. Enhanced scrolling with mouse wheel/trackpad for iOS mirror compatibility + * 4. Taking region-specific screenshots + * 5. Automating iOS apps through macOS screen mirroring + * + * Key improvements: + * - Uses mouse wheel/trackpad scrolling instead of drag for better iOS mirror compatibility + * - Proper coordinate handling that prevents focus loss + * - Unified scrolling method that works for both iOS mirroring and regular modes + */ + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +// Load environment variables from .env file +import dotenv from 'dotenv'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Load .env file from the package root directory +dotenv.config({ path: path.join(__dirname, '..', '.env') }); + +// Use dynamic import for TypeScript modules +let iOSDevice; +let iOSAgent; + +async function loadModules() { + try { + const pageModule = await import('../src/page/index.ts'); + const agentModule = await import('../src/agent/index.ts'); + iOSDevice = pageModule.iOSDevice; + iOSAgent = agentModule.iOSAgent; + } catch (error) { + console.error('❌ Failed to load modules. Please build the project first:'); + console.error(' npm run build'); + console.error(' Or run with tsx: npx tsx examples/ios-mirroring-demo.js'); + process.exit(1); + } +} + +async function demonstrateIOSMirroring() { + console.log('🍎 iOS Device Mirroring Demo with Midscene.js'); + console.log('===============================================\n'); + + // Load modules first + await loadModules(); + + // Step 1: Configure iOS device mirroring + console.log('📱 Step 1: Setting up iOS device mirroring...'); + + // Example configuration for iOS device mirrored via macOS screen sharing + // {"iPhone Mirroring", {692, 161}, {344, 764}} + const mirrorConfig = { + mirrorX: 692, // X position of iOS mirror on macOS screen + mirrorY: 161, // Y position of iOS mirror on macOS screen + mirrorWidth: 344, // Width of iOS mirror on macOS screen + mirrorHeight: 764, // Height of iOS mirror on macOS screen + }; + + const device = new iOSDevice({ + serverPort: 1412, + iOSMirrorConfig: mirrorConfig, + }); + + try { + console.log('🔗 Connecting to iOS device...'); + await device.connect(); + + // Verify configuration + const config = await device.getConfiguration(); + console.log('✅ iOS mirroring configured successfully!'); + console.log( + ` 🖥️ Mirror Region: (${config.config.mirror_x}, ${config.config.mirror_y}) ${config.config.mirror_width}x${config.config.mirror_height}`, + ); + + // Step 2: Initialize AI agent + console.log('🤖 Step 2: Initializing AI agent...'); + const agent = new iOSAgent(device); + console.log('✅ AI agent ready!\n'); + + // Step 3: Demonstrate coordinate transformation + console.log('🎯 Step 3: Testing coordinate transformation...'); + // sleep 5 seconds to allow user to make the mirror app foreground + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Test tap at various iOS coordinates + const testPoints = [ + { left: 100, top: 200, description: 'Upper left area' }, + { left: 196, top: 426, description: 'Center of screen' }, + { left: 300, top: 700, description: 'Lower right area' }, + ]; + + for (const point of testPoints) { + console.log( + ` 📍 Tapping at iOS coordinates (${point.left}, ${point.top}) - ${point.description}`, + ); + await device.tap(point); + await new Promise((resolve) => setTimeout(resolve, 500)); // Brief pause + } + console.log('✅ Coordinate transformation test completed!\n'); + + // Step 4: Take iOS region screenshot + console.log('📸 Step 4: Taking iOS region screenshot...'); + const screenshot = await device.screenshotBase64(); + console.log(`✅ Screenshot captured (${screenshot.length} bytes)`); + console.log(' 💾 Screenshot contains only the iOS mirrored area\n'); + + // Step 5: Test enhanced scrolling functionality (now uses intelligent distance mapping) + console.log( + '🖱️ Step 5: Testing enhanced trackpad scrolling with intelligent distance mapping...', + ); + + console.log( + ' 🔄 Testing horizontal scroll right (300px) - should scroll horizontally to the right:', + ); + await device.scroll({ direction: 'right', distance: 300 }); + await new Promise((resolve) => setTimeout(resolve, 1500)); + + console.log( + ' ⬅️ Testing horizontal scroll left (300px) - should scroll horizontally to the left:', + ); + await device.scroll({ direction: 'left', distance: 300 }); + await new Promise((resolve) => setTimeout(resolve, 1500)); + + console.log('✅ Enhanced horizontal scrolling test completed!\n'); + + // Step 6: Demonstrate AI automation + console.log('🧠 Step 6: AI automation example...'); + console.log(' (This would work with actual iOS app content)'); + + // Example AI operations (commented out as they need actual iOS app content) + await agent.aiTap('Settings app icon'); + await agent.ai('返回主屏幕'); + // await agent.ai("在 显示与亮度中 开启深色模式") + + console.log('✅ Demo completed successfully!\n'); + + // Step 7: Show usage summary + console.log('📋 Usage Summary:'); + console.log('================'); + console.log( + '• iOS coordinates are automatically transformed to macOS coordinates', + ); + console.log('• Screenshots capture only the iOS mirrored region'); + console.log( + '• Scrolling now uses intelligent distance mapping for better Android compatibility', + ); + console.log( + '• Distance values (e.g., 200px) are automatically converted to appropriate scroll events', + ); + console.log( + '• Trackpad scrolling by default provides smooth, natural iOS experience', + ); + console.log('• Mouse wheel scrolling available as fallback option'); + console.log('• All Midscene AI features work with iOS device mirroring'); + console.log('• Perfect for testing iOS apps through screen mirroring'); + console.log( + '• Coordinate system is unified: use iOS logical coordinates everywhere\n', + ); + } catch (error) { + console.error('❌ Demo failed:', error.message); + console.error('\n🔧 Troubleshooting:'); + console.error( + '• Ensure Python server is running: python3 auto_server.py 1412', + ); + console.error('• Check iOS device is properly mirrored on macOS screen'); + console.error('• Verify mirror coordinates match actual screen position'); + console.error('• Install required dependencies: flask, pyautogui'); + process.exit(1); + } +} + +// Additional utility functions for iOS mirroring setup +async function detectIOSMirrorRegion() { + console.log('🔍 iOS Mirror Region Detection Helper'); + console.log('===================================='); + console.log( + 'Use this function to help detect iOS mirror coordinates on your screen:', + ); + console.log('1. Open iOS device in screen mirroring/QuickTime/Simulator'); + console.log('2. Note the position and size of the iOS window'); + console.log('3. Update mirrorConfig with these values'); + console.log('4. Test with small tap operations first'); + + // This could be enhanced with screen capture analysis + // to automatically detect iOS mirror regions +} + +function calculateMirrorInfo(mirrorWidth, mirrorHeight) { + // Common iOS device aspect ratios for reference + const commonRatios = [ + { name: 'iPhone 15 Pro', width: 393, height: 852 }, + { name: 'iPhone 12/13/14', width: 390, height: 844 }, + { name: 'iPhone 11 Pro Max', width: 414, height: 896 }, + { name: 'iPhone X/XS', width: 375, height: 812 }, + ]; + + const mirrorRatio = mirrorHeight / mirrorWidth; + + console.log('📐 Mirror Information:'); + console.log(` Mirror Size: ${mirrorWidth}x${mirrorHeight}`); + console.log(` Aspect Ratio: ${mirrorRatio.toFixed(3)}`); + + // Find closest matching iOS device + const closest = commonRatios.reduce((prev, curr) => { + const prevRatio = prev.height / prev.width; + const currRatio = curr.height / curr.width; + return Math.abs(currRatio - mirrorRatio) < Math.abs(prevRatio - mirrorRatio) + ? curr + : prev; + }); + + console.log( + ` Closest iOS device: ${closest.name} (${closest.width}x${closest.height})`, + ); + + return { mirrorWidth, mirrorHeight, suggestedDevice: closest }; +} + +// Check if this file is being run directly +const isMainModule = import.meta.url === `file://${process.argv[1]}`; + +if (isMainModule) { + demonstrateIOSMirroring().catch(console.error); +} diff --git a/packages/ios/getAppWindowRect.scpt b/packages/ios/getAppWindowRect.scpt new file mode 100644 index 0000000000000000000000000000000000000000..88b8718218cc0243408c17b12a3c60219ce249d3 GIT binary patch literal 2224 zcmb_d*;5op9R6k&5oc%?5RZ7^5WKPiyQ_F1s3@dNN(-wfUlOZQ<1)wwc4up6cT@P} zbDr{#ZFg^3DEW}#9MA|sT6joGNe9K~Pc3raa6f5MWF7Jm!d zg3-Vq@J8Hhab!f~Fm%mk4&)Uq7#VF9hJ1*~2U?epKsAl+M3*z_7o8!uq5;HCWR zK;RU7C*ux#PI~W3rVLr?Us0?p)3p^z+`qykE=?G&SicYA^@JPq6D?;}9H- z!bg$+A-E{PW2>DtU@JvQ76WWMD6s7y!#Ts@oHX`QL(8?s9;A`y=d-3cc93J;$nkS9 z7>;@z)AC|yu?bBkDj7{Rlp-)=IzN#2@=o3w@?Nj~orbrdM6uS$8^zeenU&azsJxa} z3Q?GSRMaE2s8fV+Q!I3$-b6hW=b}Lo%si)ASd4j?uL$M_L-#+y0xVPnGaJflIkHF* zj3oOtsFjzNyhMeS%eonbMKIgBxfa9u1B)#JB`G(jXgJ(7E=XQP}S(u~ir94izdTv#(! z$?sTc5h|?Icj>Ic%5roouu>jJ^w8}$E9_nQH0o}EfZO5=i9@z3=%CI=N@bJ?*v-98y?Yww*vy?%Hy9y4%E*+dH=0!JGtU z1?fZHH`;L<>pHouXy4e)UE07rw<}&Dm#j-7Ux;lFw( ziM8L$m(nSZ(IOG=C_C=hoj#Cv^P?TpCrrqu1r;Z|EN43V$-n{m)g!w)=nXkI%YMMK z<%S#@CGAv`V_h<`W>?L{0om8VF7%M&TFZXp6Mo~<{l@<#8FOL8`P^uFeXZe@J|k~5 z+mVE$wWO?wTqA#MHTWZ=c5%*CORn94X=T_`6v4hTMVW-JAPLusbQ!eV*uISh<>nOHs=%(m`oJXgelV8yz zXXT8XmQ!+4PRMaN7Lf}))s)ZtyjpT@;^14fz$-vruVET7*OIfkPfE7s%nX;nlGB`{ ziP4r*QzT)@Nv%o(v*pCpCAH+Z9wx!qa%}p&tdmRFcH`S>7=5?uu|76793OAF{q>!e z{a<;TlD!xDVq-_*^;jIcHyrQ1)OYkjOYfDwqmNp8ul61J<#?tP-M2652B|!sqUv|o HuKoBIYt#&a literal 0 HcmV?d00001 diff --git a/packages/ios/idb/auto_server.py b/packages/ios/idb/auto_server.py new file mode 100644 index 000000000..4c516df65 --- /dev/null +++ b/packages/ios/idb/auto_server.py @@ -0,0 +1,472 @@ +from flask import Flask, request, jsonify +import pyautogui +import time +import traceback +import json +import sys +import subprocess +import os + +app = Flask(__name__) + +# Configure pyautogui +pyautogui.FAILSAFE = True +pyautogui.PAUSE = 0.1 + +def execute_applescript(script): + """Execute AppleScript command""" + try: + result = subprocess.run(['osascript', '-e', script], + capture_output=True, text=True, timeout=5) + return result.returncode == 0, result.stdout, result.stderr + except Exception as e: + return False, "", str(e) + +# iOS device configuration +ios_config = { + "enabled": False, + "mirror_x": 0, + "mirror_y": 0, + "mirror_width": 0, + "mirror_height": 0, + "ios_aspect_ratio": 2.17, # Default iPhone ratio (852/393) + "estimated_ios_width": 393, + "estimated_ios_height": 852 +} + +def setup_ios_mapping(mirror_x, mirror_y, mirror_width, mirror_height): + """Setup coordinate mapping for iOS device mirroring""" + global ios_config + + # Estimate iOS device dimensions based on mirror aspect ratio + mirror_aspect_ratio = mirror_height / mirror_width + + # Common iOS device configurations + ios_devices = [ + {"name": "iPhone 15 Pro", "width": 393, "height": 852}, + {"name": "iPhone 15 Plus", "width": 428, "height": 926}, + {"name": "iPhone 12/13/14", "width": 390, "height": 844}, + {"name": "iPhone 11 Pro Max", "width": 414, "height": 896}, + {"name": "iPhone X/XS", "width": 375, "height": 812}, + {"name": "iPad Pro 12.9", "width": 1024, "height": 1366}, + {"name": "iPad Pro 11", "width": 834, "height": 1194}, + ] + + # Find closest matching device based on aspect ratio + best_match = min(ios_devices, key=lambda d: abs((d["height"] / d["width"]) - mirror_aspect_ratio)) + + ios_config.update({ + "enabled": True, + "mirror_x": mirror_x, + "mirror_y": mirror_y, + "mirror_width": mirror_width, + "mirror_height": mirror_height, + "ios_aspect_ratio": mirror_aspect_ratio, + "estimated_ios_width": best_match["width"], + "estimated_ios_height": best_match["height"] + }) + + print(f"iOS mapping configured: Estimated {best_match['name']} ({best_match['width']}x{best_match['height']}) -> {mirror_width}x{mirror_height} at ({mirror_x},{mirror_y})") + print(f"Aspect ratio: {mirror_aspect_ratio:.3f}, Device: {best_match['name']}") + +def transform_ios_coordinates(ios_x, ios_y): + """Transform iOS coordinates to macOS screen coordinates""" + if not ios_config["enabled"]: + return ios_x, ios_y + + # Calculate scale factors based on estimated iOS dimensions + scale_x = ios_config["mirror_width"] / ios_config["estimated_ios_width"] + scale_y = ios_config["mirror_height"] / ios_config["estimated_ios_height"] + + # Convert iOS coordinates to macOS coordinates + mac_x = ios_config["mirror_x"] + (ios_x * scale_x) + mac_y = ios_config["mirror_y"] + (ios_y * scale_y) + + return int(mac_x), int(mac_y) + +def get_ios_screenshot_region(): + """Get the region for iOS device screenshot""" + if not ios_config["enabled"]: + return None + + return ( + ios_config["mirror_x"], + ios_config["mirror_y"], + ios_config["mirror_width"], + ios_config["mirror_height"] + ) + +def handle_action(action): + try: + act = action.get("action") + if act == "click": + x = int(action["x"]) + y = int(action["y"]) + # Transform coordinates if iOS mapping is enabled + mac_x, mac_y = transform_ios_coordinates(x, y) + + # Validate coordinates are within expected iOS mirror region + if ios_config["enabled"]: + mirror_left = ios_config["mirror_x"] + mirror_top = ios_config["mirror_y"] + mirror_right = mirror_left + ios_config["mirror_width"] + mirror_bottom = mirror_top + ios_config["mirror_height"] + + if not (mirror_left <= mac_x <= mirror_right and mirror_top <= mac_y <= mirror_bottom): + print(f"WARNING: Click coordinates ({mac_x}, {mac_y}) are outside iOS mirror region ({mirror_left}, {mirror_top}, {mirror_right}, {mirror_bottom})") + print(f"Original iOS coordinates: ({x}, {y})") + print(f"This might cause the iOS app to lose focus!") + + print(f"Clicking at iOS coords ({x}, {y}) -> macOS coords ({mac_x}, {mac_y})") + + pyautogui.click(mac_x, mac_y) + return {"status": "ok", "action": "click", "ios_coords": [x, y], "mac_coords": [mac_x, mac_y]} + + elif act == "move": + x = int(action["x"]) + y = int(action["y"]) + duration = float(action.get("duration", 0.2)) + # Transform coordinates if iOS mapping is enabled + mac_x, mac_y = transform_ios_coordinates(x, y) + pyautogui.moveTo(mac_x, mac_y, duration=duration) + return {"status": "ok", "action": "move", "ios_coords": [x, y], "mac_coords": [mac_x, mac_y]} + + elif act == "drag": + x = int(action["x"]) + y = int(action["y"]) + x2 = int(action["x2"]) + y2 = int(action["y2"]) + duration = float(action.get("duration", 0.5)) + # Transform coordinates if iOS mapping is enabled + mac_x, mac_y = transform_ios_coordinates(x, y) + mac_x2, mac_y2 = transform_ios_coordinates(x2, y2) + pyautogui.moveTo(mac_x, mac_y) + pyautogui.dragTo(mac_x2, mac_y2, duration=duration) + return {"status": "ok", "action": "drag", "ios_from": [x, y], "ios_to": [x2, y2], "mac_from": [mac_x, mac_y], "mac_to": [mac_x2, mac_y2]} + + elif act == "type": + text = action["text"] + interval = float(action.get("interval", 0.0)) + # For iOS, we need slower typing to ensure proper character registration + # iOS virtual keyboards can miss characters if typing is too fast + if interval == 0.0: + # Set a default interval for iOS compatibility + interval = 0.02 # 20ms between characters - good balance for iOS + + print(f"📱 iOS Type: '{text}' with interval {interval}s") + + # Use AppleScript to simulate keyboard input to avoid system shortcuts + # This method sends text directly to the active application without triggering shortcuts + try: + # Escape special characters for AppleScript + escaped_text = text.replace('"', '\\"').replace('\\', '\\\\') + + applescript = f''' + tell application "System Events" + keystroke "{escaped_text}" + end tell + ''' + + success, stdout, stderr = execute_applescript(applescript) + if success: + print(f" ✅ Used AppleScript keystroke for text input") + # Add interval delay if specified + if interval > 0: + time.sleep(len(text) * interval) + return {"status": "ok", "action": "type", "text": text, "method": "applescript", "interval": interval} + else: + print(f" ❌ AppleScript failed: {stderr}, falling back to character-by-character") + + except Exception as e: + print(f" ❌ AppleScript method failed: {e}, falling back to character-by-character") + + # Fallback: Character-by-character input with modifier key clearing + print(f" 🔤 Using character-by-character input method") + + # Clear any pressed modifier keys first + modifier_keys = ['shift', 'ctrl', 'alt', 'cmd'] + for key in modifier_keys: + try: + pyautogui.keyUp(key) + except: + pass # Ignore if key wasn't pressed + + # Type each character individually + for i, char in enumerate(text): + try: + # For special characters that might cause issues, use write method + if char in [' ', '\n', '\t']: + if char == ' ': + pyautogui.press('space') + elif char == '\n': + pyautogui.press('enter') + elif char == '\t': + pyautogui.press('tab') + else: + # Use write for individual characters to avoid shortcut combinations + pyautogui.write(char) + + # Add interval delay between characters + if interval > 0 and i < len(text) - 1: + time.sleep(interval) + + except Exception as char_error: + print(f" ⚠️ Error typing character '{char}': {char_error}") + continue + + return {"status": "ok", "action": "type", "text": text, "method": "character_by_character", "interval": interval} + + elif act == "key": + key = action["key"] + pyautogui.press(key) + return {"status": "ok", "action": "key", "key": key} + + elif act == "hotkey": + keys = action["keys"] + if isinstance(keys, list): + pyautogui.hotkey(*keys) + else: + pyautogui.hotkey(keys) + return {"status": "ok", "action": "hotkey", "keys": keys} + + elif act == "scroll": + x = int(action.get("x", ios_config["estimated_ios_width"] // 2 if ios_config["enabled"] else pyautogui.size().width // 2)) + y = int(action.get("y", ios_config["estimated_ios_height"] // 2 if ios_config["enabled"] else pyautogui.size().height // 2)) + + # Enhanced distance calculation for better Android compatibility + distance = int(action.get("distance", 100)) + direction = action.get("direction", "down") # up, down, left, right + + # Transform coordinates if iOS mapping is enabled + mac_x, mac_y = transform_ios_coordinates(x, y) + + # Calculate clicks based on distance + if distance <= 50: + clicks = max(8, int(distance * 0.4)) + elif distance <= 150: + clicks = max(12, int(distance * 0.25)) + elif distance <= 300: + clicks = max(18, int(distance * 0.18)) + else: + clicks = max(25, int(distance * 0.12)) + + print(f"📍 SCROLL: iOS({x}, {y}) -> Mac({mac_x}, {mac_y}), Direction: {direction}, Distance: {distance}px, Clicks: {clicks}") + + # Move mouse to the target position first + pyautogui.moveTo(mac_x, mac_y) + + # Simplified scroll logic - direct implementation with multiple methods + if direction in ["left", "right"]: + print(f"🔄 HORIZONTAL SCROLL: {direction}") + method = "horizontal_scroll" + success = False + + # Method 1: Try using hscroll if available (rarely works on macOS) + if hasattr(pyautogui, 'hscroll'): + try: + scroll_amount = clicks * 3 if direction == "right" else -clicks * 3 + pyautogui.hscroll(scroll_amount, x=mac_x, y=mac_y) + print(f" ✅ Used hscroll method: {scroll_amount}") + success = True + except Exception as e: + print(f" ❌ hscroll failed: {e}") + + # Method 2: Try AppleScript system events (macOS native approach) + if not success: + try: + scroll_amount = clicks * 5 # More units for better effect + scroll_direction = "right" if direction == "right" else "left" + + # Use AppleScript to simulate horizontal scroll + applescript = f''' + tell application "System Events" + set mousePosition to {{{mac_x}, {mac_y}}} + set mouseLoc to mousePosition + repeat {clicks} times + tell application "System Events" to scroll mouseLoc horizontally by {5 if direction == "right" else -5} + delay 0.01 + end repeat + end tell + ''' + + success, stdout, stderr = execute_applescript(applescript) + if success: + print(f" ✅ Used AppleScript horizontal scroll: {scroll_direction}") + else: + print(f" ❌ AppleScript failed: {stderr}") + except Exception as e: + print(f" ❌ AppleScript method failed: {e}") + + # Method 3: Drag simulation (most reliable fallback) + if not success: + try: + print(f" 🖱️ Using drag simulation for horizontal scroll") + start_x = mac_x + start_y = mac_y + + # Calculate drag distance based on clicks (more aggressive) + drag_distance = min(clicks * 15, 300) # Cap at 300px + + if direction == "right": + end_x = start_x - drag_distance # Drag left to scroll right + else: + end_x = start_x + drag_distance # Drag right to scroll left + + # Ensure we don't drag outside reasonable bounds + screen_width = pyautogui.size().width + end_x = max(50, min(end_x, screen_width - 50)) + + # Perform smooth drag scroll + pyautogui.moveTo(start_x, start_y) + time.sleep(0.1) # Brief pause + pyautogui.mouseDown() + pyautogui.moveTo(end_x, start_y, duration=0.4) # Slower for iOS compatibility + pyautogui.mouseUp() + + print(f" ✅ Drag scroll: ({start_x}, {start_y}) -> ({end_x}, {start_y}), distance: {abs(end_x - start_x)}px") + success = True + method = "horizontal_drag_scroll" + except Exception as e: + print(f" ❌ Drag simulation failed: {e}") + + # Final fallback: shift+scroll (even though it might not work) + if not success: + print(f" ⚠️ All methods failed, trying shift+scroll fallback") + pyautogui.keyDown('shift') + for i in range(clicks): + scroll_amount = 8 if direction == "left" else -8 + pyautogui.scroll(scroll_amount, x=mac_x, y=mac_y) + time.sleep(0.008) + pyautogui.keyUp('shift') + method = "horizontal_scroll_fallback" + + else: + print(f"⬆️⬇️ VERTICAL SCROLL: {direction}") + # Vertical scrolling (this should work fine) + for i in range(clicks): + scroll_amount = 8 if direction == "up" else -8 + pyautogui.scroll(scroll_amount, x=mac_x, y=mac_y) + time.sleep(0.008) # Fast succession + method = "vertical_scroll" + + print(f"✅ Scroll completed: {direction} ({clicks} iterations)") + return {"status": "ok", "action": "scroll", "method": method, "ios_coords": [x, y], "mac_coords": [mac_x, mac_y], "direction": direction, "clicks": clicks, "distance": distance} + + elif act == "screenshot": + # Take screenshot of iOS region if mapping is enabled + region = get_ios_screenshot_region() + if region: + screenshot = pyautogui.screenshot(region=region) + else: + screenshot = pyautogui.screenshot() + # Save to temporary file and return path + temp_path = f"/tmp/screenshot_{int(time.time())}.png" + screenshot.save(temp_path) + return {"status": "ok", "action": "screenshot", "path": temp_path, "ios_region": region is not None} + + elif act == "get_screen_size": + if ios_config["enabled"]: + return {"status": "ok", "action": "get_screen_size", "width": ios_config["estimated_ios_width"], "height": ios_config["estimated_ios_height"], "mode": "ios"} + else: + size = pyautogui.size() + return {"status": "ok", "action": "get_screen_size", "width": size.width, "height": size.height, "mode": "mac"} + + elif act == "configure_ios": + mirror_x = int(action["mirror_x"]) + mirror_y = int(action["mirror_y"]) + mirror_width = int(action["mirror_width"]) + mirror_height = int(action["mirror_height"]) + setup_ios_mapping(mirror_x, mirror_y, mirror_width, mirror_height) + return {"status": "ok", "action": "configure_ios", "config": ios_config} + + elif act == "sleep": + seconds = float(action["seconds"]) + time.sleep(seconds) + return {"status": "ok", "action": "sleep", "seconds": seconds} + + else: + return {"status": "error", "error": f"Unknown action: {act}"} + + except Exception as e: + return { + "status": "error", + "error": str(e), + "traceback": traceback.format_exc() + } + +@app.route("/health", methods=["GET"]) +def health_check(): + """Health check endpoint""" + try: + screen_size = pyautogui.size() + return jsonify({ + "status": "ok", + "message": "PyAutoGUI server is running", + "screen_size": {"width": screen_size.width, "height": screen_size.height}, + "pyautogui_version": pyautogui.__version__ if hasattr(pyautogui, '__version__') else "unknown" + }) + except Exception as e: + return jsonify({ + "status": "error", + "error": str(e) + }), 500 + +@app.route("/configure", methods=["POST"]) +def configure_ios(): + """Configure iOS device mapping""" + try: + data = request.get_json() + # Support both snake_case and camelCase naming + mirror_x = data.get("mirror_x") or data.get("mirrorX") + mirror_y = data.get("mirror_y") or data.get("mirrorY") + mirror_width = data.get("mirror_width") or data.get("mirrorWidth") + mirror_height = data.get("mirror_height") or data.get("mirrorHeight") + + result = handle_action({ + "action": "configure_ios", + "mirror_x": mirror_x, + "mirror_y": mirror_y, + "mirror_width": mirror_width, + "mirror_height": mirror_height + }) + return jsonify(result) + except Exception as e: + return jsonify({ + "status": "error", + "error": str(e), + "traceback": traceback.format_exc() + }) + +@app.route("/config", methods=["GET"]) +def get_config(): + """Get current iOS configuration""" + return jsonify({ + "status": "ok", + "config": ios_config + }) + +@app.route("/run", methods=["POST"]) +def run_actions(): + try: + data = request.get_json() + if isinstance(data, list): + results = [handle_action(act) for act in data] + return jsonify({"status": "done", "results": results}) + elif isinstance(data, dict): + result = handle_action(data) + return jsonify(result) + else: + return jsonify({"status": "error", "error": "Invalid input format"}) + except Exception as e: + return jsonify({ + "status": "error", + "error": str(e), + "traceback": traceback.format_exc() + }) + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 1412 + print(f"Starting PyAutoGUI server on port {port}") + print(f"Screen size: {pyautogui.size()}") + print("Health check available at: http://localhost:{}/health".format(port)) + app.run(host="0.0.0.0", port=port, debug=False) \ No newline at end of file diff --git a/packages/ios/modern.config.ts b/packages/ios/modern.config.ts new file mode 100644 index 000000000..db0213ece --- /dev/null +++ b/packages/ios/modern.config.ts @@ -0,0 +1,16 @@ +import { defineConfig, moduleTools } from '@modern-js/module-tools'; + +export default defineConfig({ + plugins: [moduleTools()], + buildPreset: 'npm-library', + buildConfig: { + input: { + index: './src/index.ts', + agent: './src/agent/index.ts', + }, + target: 'es2020', + dts: { + respectExternal: true, + }, + }, +}); diff --git a/packages/ios/package.json b/packages/ios/package.json new file mode 100644 index 000000000..a56a44fa7 --- /dev/null +++ b/packages/ios/package.json @@ -0,0 +1,58 @@ +{ + "name": "@midscene/ios", + "version": "0.1.0", + "description": "Midscene.js for iOS automation", + "main": "./dist/lib/index.js", + "types": "./dist/lib/index.d.ts", + "exports": { + ".": { + "types": "./dist/lib/index.d.ts", + "import": "./dist/es/index.js", + "require": "./dist/lib/index.js" + }, + "./agent": { + "types": "./dist/lib/agent.d.ts", + "import": "./dist/es/agent.js", + "require": "./dist/lib/agent.js" + } + }, + "bin": { + "midscene-ios-server": "./bin/server.js" + }, + "files": ["lib/**/*", "bin/**/*", "idb/**/*", "README.md"], + "scripts": { + "build": "modern build", + "dev": "modern dev", + "test": "vitest", + "server": "node bin/server.js", + "server:test": "python3 idb/test.py", + "example": "tsx tests/example.ts", + "setup": "./setup.sh", + "prepack": "npm run build" + }, + "dependencies": { + "@midscene/core": "workspace:*", + "@midscene/shared": "workspace:*", + "@midscene/web": "workspace:*", + "node-fetch": "^3.3.2" + }, + "devDependencies": { + "@modern-js/module-tools": "^2.60.3", + "@types/node": "^22.10.5", + "dotenv": "^16.4.5", + "tsx": "^4.17.0", + "typescript": "^5.7.2", + "vitest": "^2.1.8" + }, + "peerDependencies": {}, + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/web-infra-dev/midscene.git", + "directory": "packages/ios" + }, + "license": "MIT" +} diff --git a/packages/ios/scripts/getAppWindowRect.scpt b/packages/ios/scripts/getAppWindowRect.scpt new file mode 100644 index 000000000..735c3f8d3 --- /dev/null +++ b/packages/ios/scripts/getAppWindowRect.scpt @@ -0,0 +1,11 @@ +delay 4 -- you have 4 seconds to make iPhone Mirroring App foreground!! + +tell application "System Events" + set frontApp to name of first application process whose frontmost is true + tell application process frontApp + set win to first window + set pos to position of win + set size_ to size of win + return {frontApp, pos, size_} + end tell +end tell diff --git a/packages/ios/setup.sh b/packages/ios/setup.sh new file mode 100755 index 000000000..9cd0e87e9 --- /dev/null +++ b/packages/ios/setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +echo "Setting up iOS package dependencies..." + +# Check if Python 3 is installed +if ! command -v python3 &> /dev/null; then + echo "❌ Python 3 is not installed. Please install Python 3 first." + exit 1 +fi + +echo "✅ Python 3 found: $(python3 --version)" + +# Check if pip3 is available +if ! command -v pip3 &> /dev/null; then + echo "❌ pip3 is not available. Please install pip3 first." + exit 1 +fi + +echo "✅ pip3 found: $(pip3 --version)" + +# Install required Python packages +echo "Installing Python dependencies..." +pip3 install flask pyautogui pillow requests + +echo "✅ Python dependencies installed" + +# Make the server script executable +chmod +x bin/server.js + +echo "✅ iOS package setup completed!" +echo "" +echo "To test the setup:" +echo "1. Start the server: npm run server" +echo "2. Test the server: python3 idb/test.py" +echo "3. Run example: npm run example" diff --git a/packages/ios/src/agent/index.ts b/packages/ios/src/agent/index.ts new file mode 100644 index 000000000..e10fd8ad6 --- /dev/null +++ b/packages/ios/src/agent/index.ts @@ -0,0 +1,58 @@ +import { vlLocateMode } from '@midscene/shared/env'; +import { PageAgent, type PageAgentOpt } from '@midscene/web/agent'; +import { iOSDevice, type iOSDeviceOpt } from '../page'; +import { debugPage } from '../page'; +import { getScreenSize, startPyAutoGUIServer } from '../utils'; + +type iOSAgentOpt = PageAgentOpt; + +export class iOSAgent extends PageAgent { + declare page: iOSDevice; + + async launch(uri: string): Promise { + const device = this.page; + await device.launch(uri); + } + + async back(): Promise { + await this.page.back(); + } + + async home(): Promise { + await this.page.home(); + } + + async recentApps(): Promise { + await this.page.recentApps(); + } +} + +export async function agentFromPyAutoGUI(opts?: iOSAgentOpt & iOSDeviceOpt) { + // Start PyAutoGUI server if not already running + const serverPort = opts?.serverPort || 1412; + + try { + // Try to test if server is already running + const fetch = (await import('node-fetch')).default; + await fetch(`http://localhost:${serverPort}/run`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'sleep', seconds: 0 }), + }); + console.log(`PyAutoGUI server is already running on port ${serverPort}`); + } catch (error) { + console.log(`Starting PyAutoGUI server on port ${serverPort}...`); + await startPyAutoGUIServer(serverPort); + } + + const page = new iOSDevice({ + serverUrl: opts?.serverUrl, + serverPort, + autoDismissKeyboard: opts?.autoDismissKeyboard, + iOSMirrorConfig: opts?.iOSMirrorConfig || opts?.mirrorConfig, + }); + + await page.connect(); + + return new iOSAgent(page, opts); +} diff --git a/packages/ios/src/index.ts b/packages/ios/src/index.ts new file mode 100644 index 000000000..f2a459a1b --- /dev/null +++ b/packages/ios/src/index.ts @@ -0,0 +1,4 @@ +export { iOSDevice } from './page'; +export { iOSAgent, agentFromPyAutoGUI } from './agent'; +export { getScreenSize } from './utils'; +export { overrideAIConfig } from '@midscene/shared/env'; diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts new file mode 100644 index 000000000..cfcd301f4 --- /dev/null +++ b/packages/ios/src/page/index.ts @@ -0,0 +1,871 @@ +import assert from 'node:assert'; +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Point, Size } from '@midscene/core'; +import type { PageType } from '@midscene/core'; +import { getTmpFile, sleep } from '@midscene/core/utils'; +import type { ElementInfo } from '@midscene/shared/extractor'; +import { resizeImg } from '@midscene/shared/img'; +import { getDebug } from '@midscene/shared/logger'; +import type { AndroidDeviceInputOpt, AndroidDevicePage } from '@midscene/web'; +import { type ScreenInfo, getScreenSize } from '../utils'; + +export const debugPage = getDebug('ios:device'); + +export interface iOSDeviceOpt extends AndroidDeviceInputOpt { + serverUrl?: string; + serverPort?: number; + autoDismissKeyboard?: boolean; + // iOS device mirroring configuration + iOSMirrorConfig?: { + mirrorX: number; + mirrorY: number; + mirrorWidth: number; + mirrorHeight: number; + }; + // Alternative name for better API compatibility + mirrorConfig?: { + mirrorX: number; + mirrorY: number; + mirrorWidth: number; + mirrorHeight: number; + }; +} + +export interface PyAutoGUIAction { + action: + | 'click' + | 'move' + | 'drag' + | 'type' + | 'key' + | 'hotkey' + | 'sleep' + | 'screenshot' + | 'scroll'; + x?: number; + y?: number; + x2?: number; + y2?: number; + text?: string; + key?: string; + keys?: string[]; + seconds?: number; + direction?: 'up' | 'down' | 'left' | 'right'; + clicks?: number; + distance?: number; // Original scroll distance in pixels + scroll_type?: 'wheel' | 'trackpad'; + interval?: number; // Interval between keystrokes for type action +} + +export interface PyAutoGUIResult { + status: 'ok' | 'error'; + action?: string; + x?: number; + y?: number; + text?: string; + seconds?: number; + from?: [number, number]; + to?: [number, number]; + path?: string; // For screenshot action + ios_region?: boolean; // For screenshot action + direction?: string; // For scroll action + clicks?: number; // For scroll action + method?: string; // For scroll action (wheel, trackpad, etc.) + ios_coords?: [number, number]; // For coordinate transformation info + mac_coords?: [number, number]; // For coordinate transformation info + error?: string; + traceback?: string; +} + +export class iOSDevice implements AndroidDevicePage { + private devicePixelRatio = 1; + private screenInfo: ScreenInfo | null = null; + private destroyed = false; + pageType: PageType = 'ios'; + uri: string | undefined; + options?: iOSDeviceOpt; + private serverUrl: string; + + constructor(options?: iOSDeviceOpt) { + this.options = options; + this.serverUrl = + options?.serverUrl || `http://localhost:${options?.serverPort || 1412}`; + } + + public async connect(): Promise { + if (this.destroyed) { + throw new Error('iOSDevice has been destroyed and cannot be used'); + } + + // Health check to ensure Python server is running + try { + const response = await fetch(`${this.serverUrl}/health`); + if (!response.ok) { + throw new Error( + `Python server health check failed: ${response.status}`, + ); + } + const healthData = await response.json(); + debugPage(`Python server is running: ${JSON.stringify(healthData)}`); + } catch (error: any) { + throw new Error( + `Failed to connect to Python server at ${this.serverUrl}: ${error.message}`, + ); + } + + // Configure iOS mirroring if provided + if (this.options?.iOSMirrorConfig) { + await this.configureIOSMirror(this.options.iOSMirrorConfig); + } + + // Get screen information (will use iOS dimensions if configured) + this.screenInfo = await getScreenSize(); + this.devicePixelRatio = this.screenInfo.dpr; + + debugPage( + `iOS Device initialized - Screen: ${this.screenInfo.width}x${this.screenInfo.height}, DPR: ${this.devicePixelRatio}`, + ); + } + + private async configureIOSMirror(config: { + mirrorX: number; + mirrorY: number; + mirrorWidth: number; + mirrorHeight: number; + }): Promise { + try { + const response = await fetch(`${this.serverUrl}/configure`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(config), + }); + + if (!response.ok) { + throw new Error(`Failed to configure iOS mirror: ${response.status}`); + } + + const result = await response.json(); + if (result.status !== 'ok') { + throw new Error(`iOS configuration failed: ${result.error}`); + } + + debugPage( + `iOS mirroring configured: mirror region ${config.mirrorX},${config.mirrorY} -> ${config.mirrorWidth}x${config.mirrorHeight}`, + ); + } catch (error: any) { + throw new Error(`Failed to configure iOS mirroring: ${error.message}`); + } + } + + async getConfiguration(): Promise { + const response = await fetch(`${this.serverUrl}/config`); + if (!response.ok) { + throw new Error(`Failed to get configuration: ${response.status}`); + } + return await response.json(); + } + + public async launch(uri: string): Promise { + this.uri = uri; + + try { + if (uri.startsWith('http://') || uri.startsWith('https://')) { + // Open URL in default browser + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + await execAsync(`open "${uri}"`); + debugPage(`Successfully launched URL: ${uri}`); + } else { + // Try to open as application + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + await execAsync(`open -a "${uri}"`); + debugPage(`Successfully launched app: ${uri}`); + } + } catch (error: any) { + debugPage(`Error launching ${uri}: ${error}`); + throw new Error(`Failed to launch ${uri}: ${error.message}`, { + cause: error, + }); + } + + return this; + } + + async size(): Promise { + // 对于iOS镜像模式，返回iOS设备的逻辑尺寸而不是macOS屏幕尺寸 + if (this.options?.iOSMirrorConfig) { + // 从Python服务器获取配置信息，使用估算的iOS设备尺寸 + try { + const config = await this.getConfiguration(); + if (config.status === 'ok' && config.config.enabled) { + return { + width: config.config.estimated_ios_width, + height: config.config.estimated_ios_height, + dpr: 1, // iOS坐标系不需要额外的像素比调整 + }; + } + } catch (error) { + debugPage('Failed to get iOS configuration, using fallback:', error); + } + } + + // 非iOS镜像模式或配置获取失败时的fallback + if (!this.screenInfo) { + this.screenInfo = await getScreenSize(); + } + + return { + width: this.screenInfo.width, + height: this.screenInfo.height, + dpr: this.devicePixelRatio, + }; + } + + private adjustCoordinates(x: number, y: number): { x: number; y: number } { + const ratio = this.devicePixelRatio; + return { + x: Math.round(x * ratio), + y: Math.round(y * ratio), + }; + } + + private reverseAdjustCoordinates( + x: number, + y: number, + ): { x: number; y: number } { + const ratio = this.devicePixelRatio; + return { + x: Math.round(x / ratio), + y: Math.round(y / ratio), + }; + } + + async screenshotBase64(): Promise { + debugPage('screenshotBase64 begin'); + + try { + // Use PyAutoGUI server's screenshot functionality for iOS mirroring + if (this.options?.iOSMirrorConfig) { + const result = await this.executePyAutoGUIAction({ + action: 'screenshot', + }); + + if (result.status === 'ok' && result.path) { + // Read the screenshot file and convert to base64 + const screenshotBuffer = await fs.promises.readFile(result.path); + + // Get iOS device dimensions for resizing + const { width, height } = await this.size(); + + // Resize to match iOS device dimensions + const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { + width, + height, + }); + + // Clean up temporary file + try { + await fs.promises.unlink(result.path); + } catch (cleanupError) { + debugPage('Failed to cleanup temp screenshot file:', cleanupError); + } + + debugPage('screenshotBase64 end (via PyAutoGUI server)'); + return `data:image/png;base64,${resizedScreenshotBuffer.toString('base64')}`; + } else { + throw new Error('PyAutoGUI screenshot failed: no path returned'); + } + } else { + // Fallback to macOS screencapture for non-mirroring scenarios + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + const tempPath = getTmpFile('png')!; + + // Use screencapture to take screenshot + await execAsync(`screencapture -x "${tempPath}"`); + + // Read and resize the screenshot + const screenshotBuffer = await fs.promises.readFile(tempPath); + const { width, height } = await this.size(); + + const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { + width, + height, + }); + + debugPage('screenshotBase64 end (via screencapture)'); + return `data:image/png;base64,${resizedScreenshotBuffer.toString('base64')}`; + } + } catch (error: any) { + debugPage('screenshotBase64 error:', error); + throw new Error(`Failed to take screenshot: ${error.message}`); + } + } + + /** + * Execute action via PyAutoGUI server + */ + private async executePyAutoGUIAction( + action: PyAutoGUIAction, + ): Promise { + try { + const fetch = (await import('node-fetch')).default; + + const response = await fetch(`${this.serverUrl}/run`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(action), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = (await response.json()) as PyAutoGUIResult; + + if (result.status === 'error') { + throw new Error(`PyAutoGUI error: ${result.error}`); + } + + return result; + } catch (error: any) { + debugPage('PyAutoGUI action failed:', error); + throw new Error(`Failed to execute PyAutoGUI action: ${error.message}`); + } + } + + async tap(point: Point): Promise { + debugPage(`tap at (${point.left}, ${point.top})`); + + // 对于iOS mirroring模式，直接传递iOS坐标，让Python服务器处理坐标变换 + // 对于非mirroring模式，使用设备像素比调整坐标 + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'click', + x: point.left, + y: point.top, + }); + } else { + const adjusted = this.adjustCoordinates(point.left, point.top); + await this.executePyAutoGUIAction({ + action: 'click', + x: adjusted.x, + y: adjusted.y, + }); + } + } + + async hover(point: Point): Promise { + debugPage(`hover at (${point.left}, ${point.top})`); + + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'move', + x: point.left, + y: point.top, + }); + } else { + const adjusted = this.adjustCoordinates(point.left, point.top); + await this.executePyAutoGUIAction({ + action: 'move', + x: adjusted.x, + y: adjusted.y, + }); + } + } + + async input(text: string, options?: AndroidDeviceInputOpt): Promise { + debugPage(`input text: ${text}`); + + // For iOS, we use the optimized type action with proper intervals + // The auto server will handle this appropriately for iOS + await this.executePyAutoGUIAction({ + action: 'type', + text, + interval: 0.05, // Proper interval for iOS keyboard responsiveness + }); + + // For iOS mirroring, default to NOT dismissing keyboard as it can cause issues + // Only dismiss if explicitly enabled + if ( + options?.autoDismissKeyboard === true || + this.options?.autoDismissKeyboard === true + ) { + await this.dismissKeyboard(); + } + } + + private async dismissKeyboard(): Promise { + try { + // Method 1: Try to tap the "Done" or "Return" button if visible + // This is iOS-specific logic - many keyboards have a "Done" button + await this.keyboardPress('return'); + debugPage('Dismissed keyboard using Return key'); + } catch (error) { + try { + // Method 2: Tap outside the keyboard area (top part of screen) + const { width, height } = await this.size(); + const tapX = width / 2; + const tapY = height / 4; // Tap in the upper quarter of the screen + + await this.tap({ left: tapX, top: tapY }); + debugPage('Dismissed keyboard by tapping outside'); + } catch (fallbackError) { + debugPage('Failed to dismiss keyboard:', fallbackError); + // Don't throw error - keyboard dismissal is optional + } + } + } + + async keyboardPress(key: string): Promise { + debugPage(`keyboard press: ${key}`); + + // Check if it's a combination key (contains '+') + if (key.includes('+')) { + // Handle hotkey combinations like 'cmd+1', 'cmd+tab', etc. + const keys = key.split('+').map((k) => k.trim().toLowerCase()); + + // Map common key names to PyAutoGUI format + const keyMapping: Record = { + cmd: 'command', + ctrl: 'ctrl', + alt: 'alt', + option: 'alt', + shift: 'shift', + tab: 'tab', + enter: 'enter', + return: 'enter', + space: 'space', + backspace: 'backspace', + delete: 'delete', + escape: 'escape', + esc: 'escape', + }; + + const mappedKeys = keys.map((k) => keyMapping[k] || k); + + await this.executePyAutoGUIAction({ + action: 'hotkey', + keys: mappedKeys, + }); + } else { + // Handle single key press + const keyMap: Record = { + Enter: 'enter', + Return: 'enter', + Tab: 'tab', + Space: 'space', + Backspace: 'backspace', + Delete: 'delete', + Escape: 'escape', + }; + + const mappedKey = keyMap[key] || key.toLowerCase(); + + await this.executePyAutoGUIAction({ + action: 'key', + key: mappedKey, + }); + } + } + + async scroll(scrollType: { + direction: 'up' | 'down' | 'left' | 'right'; + distance?: number; + }): Promise { + debugPage( + `scroll ${scrollType.direction}, distance: ${scrollType.distance || 'default'}`, + ); + + // Get current screen center for scroll + const { width, height } = await this.size(); + const centerX = width / 2; + const centerY = height / 2; + + const distance = scrollType.distance || 100; + + // Improved distance calculation to better match Android scroll behavior + // Android scroll distance is in pixels, we need to convert to effective scroll events + // Base the calculation on screen size for better proportional scrolling + const screenArea = width * height; + const scrollRatio = distance / Math.sqrt(screenArea); // Normalize by screen size + + // Calculate clicks with better scaling - aim for more responsive scrolling + let clicks: number; + if (distance <= 50) { + // Small scrolls: direct mapping for fine control + clicks = Math.max(3, Math.floor(distance / 8)); + } else if (distance <= 200) { + // Medium scrolls: moderate scaling + clicks = Math.max(8, Math.floor(distance / 12)); + } else { + // Large scrolls: aggressive scaling for significant movement + clicks = Math.max(15, Math.floor(distance / 10)); + } + + debugPage( + `Scroll distance: ${distance}px -> ${clicks} clicks (ratio: ${scrollRatio.toFixed(3)})`, + ); + + // Pass both distance and calculated clicks to Python server + const scrollAction: PyAutoGUIAction = { + action: 'scroll', + x: centerX, + y: centerY, + direction: scrollType.direction, + clicks: clicks, + distance: distance, // Pass original distance for server-side fine-tuning + scroll_type: 'trackpad', // Default to trackpad for smooth scrolling + }; + + // Always use mouse wheel/trackpad for scrolling (better compatibility) + if (this.options?.iOSMirrorConfig) { + // iOS mirroring mode: use iOS coordinates directly + await this.executePyAutoGUIAction(scrollAction); + } else { + // Non-mirroring mode: adjust coordinates + const adjusted = this.adjustCoordinates(centerX, centerY); + await this.executePyAutoGUIAction({ + ...scrollAction, + x: adjusted.x, + y: adjusted.y, + scroll_type: 'wheel', // Use wheel for non-iOS devices + }); + } + } + + async getElementText(elementInfo: ElementInfo): Promise { + // For iOS/macOS, we can't easily extract text from elements + // This would require accessibility APIs or OCR + throw new Error('getElementText is not implemented for iOS devices'); + } + + // Required AndroidDevicePage interface methods + async getElementsNodeTree(): Promise { + // Simplified implementation, returns an empty node tree + return { + node: null, + children: [], + }; + } + + // @deprecated + async getElementsInfo(): Promise { + throw new Error('getElementsInfo is not implemented for iOS devices'); + } + + get mouse(): any { + return { + click: async (x: number, y: number, options: { button: string }) => { + // 直接使用传入的坐标，因为这些坐标已经是iOS坐标系的 + // 在executePyAutoGUIAction中会进行iOS到macOS的坐标变换 + await this.executePyAutoGUIAction({ + action: 'click', + x: x, + y: y, + }); + }, + wheel: async (deltaX: number, deltaY: number) => { + throw new Error('mouse wheel is not implemented for iOS devices'); + }, + move: async (x: number, y: number) => { + await this.hover({ left: x, top: y }); + }, + drag: async ( + from: { x: number; y: number }, + to: { x: number; y: number }, + ) => { + // 对于iOS镜像模式，直接传递坐标；对于非镜像模式，使用设备像素比调整 + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'drag', + x: from.x, + y: from.y, + x2: to.x, + y2: to.y, + }); + } else { + const startAdjusted = this.adjustCoordinates(from.x, from.y); + const endAdjusted = this.adjustCoordinates(to.x, to.y); + + await this.executePyAutoGUIAction({ + action: 'drag', + x: startAdjusted.x, + y: startAdjusted.y, + x2: endAdjusted.x, + y2: endAdjusted.y, + }); + } + }, + }; + } + + get keyboard(): any { + return { + type: async (text: string, options?: AndroidDeviceInputOpt) => { + await this.input(text, options); + }, + press: async (action: any) => { + if (Array.isArray(action)) { + for (const a of action) { + await this.keyboardPress(a.key); + } + } else { + await this.keyboardPress(action.key); + } + }, + }; + } + + async clearInput(element: any): Promise { + // For iOS, we need to focus the input first by tapping it + if (element?.center) { + debugPage( + `Focusing input field at (${element.center[0]}, ${element.center[1]})`, + ); + await this.tap({ left: element.center[0], top: element.center[1] }); + await sleep(300); // Wait for focus and potential keyboard animation + } + + // Select all text and delete it - this works well on iOS + await this.keyboardPress('cmd+a'); + await sleep(100); + await this.keyboardPress('delete'); + await sleep(100); + + debugPage('Input field cleared'); + } + + url(): string { + return this.uri || ''; + } + + async scrollUntilTop(startingPoint?: Point): Promise { + const screenSize = await this.size(); + const point = startingPoint || { + left: screenSize.width / 2, + top: screenSize.height / 2, + }; + + // Scroll up multiple times to reach top + for (let i = 0; i < 10; i++) { + await this.scroll({ direction: 'up', distance: screenSize.height / 3 }); + await sleep(500); + } + } + + async scrollUntilBottom(startingPoint?: Point): Promise { + const screenSize = await this.size(); + const point = startingPoint || { + left: screenSize.width / 2, + top: screenSize.height / 2, + }; + + // Scroll down multiple times to reach bottom + for (let i = 0; i < 10; i++) { + await this.scroll({ direction: 'down', distance: screenSize.height / 3 }); + await sleep(500); + } + } + + async scrollUntilLeft(startingPoint?: Point): Promise { + const screenSize = await this.size(); + const point = startingPoint || { + left: screenSize.width / 2, + top: screenSize.height / 2, + }; + + // Scroll left multiple times to reach leftmost + for (let i = 0; i < 10; i++) { + await this.scroll({ direction: 'left', distance: screenSize.width / 3 }); + await sleep(500); + } + } + + async scrollUntilRight(startingPoint?: Point): Promise { + const screenSize = await this.size(); + const point = startingPoint || { + left: screenSize.width / 2, + top: screenSize.height / 2, + }; + + // Scroll right multiple times to reach rightmost + for (let i = 0; i < 10; i++) { + await this.scroll({ direction: 'right', distance: screenSize.width / 3 }); + await sleep(500); + } + } + + async scrollUp(distance?: number, startingPoint?: Point): Promise { + await this.scroll({ direction: 'up', distance }); + } + + async scrollDown(distance?: number, startingPoint?: Point): Promise { + await this.scroll({ direction: 'down', distance }); + } + + async scrollLeft(distance?: number, startingPoint?: Point): Promise { + await this.scroll({ direction: 'left', distance }); + } + + async scrollRight(distance?: number): Promise { + await this.scroll({ direction: 'right', distance }); + } + + async getXpathsById(id: string): Promise { + throw new Error('getXpathsById is not implemented for iOS devices'); + } + + async getXpathsByPoint( + point: Point, + isOrderSensitive: boolean, + ): Promise { + throw new Error('getXpathsByPoint is not implemented for iOS devices'); + } + + async getElementInfoByXpath(xpath: string): Promise { + throw new Error('getElementInfoByXpath is not implemented for iOS devices'); + } + + async back(): Promise { + // For iOS/macOS, we can simulate Command+[ or use system back gesture + await this.keyboardPress('cmd+['); + } + + async home(): Promise { + // For iOS simulator/mirroring, CMD+1 opens home screen + debugPage('Navigating to home screen using CMD+1'); + await this.keyboardPress('cmd+1'); + } + + async recentApps(): Promise { + // For iOS simulator/mirroring, CMD+2 opens app switcher + debugPage('Opening app switcher using CMD+2'); + await this.keyboardPress('cmd+2'); + } + + async longPress(x: number, y: number, duration?: number): Promise { + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'click', + x: x, + y: y, + }); + } else { + const adjustedPoint = this.adjustCoordinates(x, y); + await this.executePyAutoGUIAction({ + action: 'click', + x: adjustedPoint.x, + y: adjustedPoint.y, + }); + } + + // Simulate long press by holding for duration + if (duration) { + await sleep(duration); + } + } + + async pullDown( + startPoint?: Point, + distance?: number, + duration?: number, + ): Promise { + const screenSize = await this.size(); + const start = startPoint || { + left: screenSize.width / 2, + top: screenSize.height / 4, + }; + const end = { + left: start.left, + top: start.top + (distance || screenSize.height / 3), + }; + + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'drag', + x: start.left, + y: start.top, + x2: end.left, + y2: end.top, + }); + } else { + const startAdjusted = this.adjustCoordinates(start.left, start.top); + const endAdjusted = this.adjustCoordinates(end.left, end.top); + + await this.executePyAutoGUIAction({ + action: 'drag', + x: startAdjusted.x, + y: startAdjusted.y, + x2: endAdjusted.x, + y2: endAdjusted.y, + }); + } + } + + async pullUp( + startPoint?: Point, + distance?: number, + duration?: number, + ): Promise { + const screenSize = await this.size(); + const start = startPoint || { + left: screenSize.width / 2, + top: (screenSize.height * 3) / 4, + }; + const end = { + left: start.left, + top: start.top - (distance || screenSize.height / 3), + }; + + if (this.options?.iOSMirrorConfig) { + await this.executePyAutoGUIAction({ + action: 'drag', + x: start.left, + y: start.top, + x2: end.left, + y2: end.top, + }); + } else { + const startAdjusted = this.adjustCoordinates(start.left, start.top); + const endAdjusted = this.adjustCoordinates(end.left, end.top); + + await this.executePyAutoGUIAction({ + action: 'drag', + x: startAdjusted.x, + y: startAdjusted.y, + x2: endAdjusted.x, + y2: endAdjusted.y, + }); + } + } + + async destroy(): Promise { + debugPage('destroy iOS device'); + this.destroyed = true; + } + + // Additional abstract methods from AbstractPage + async waitUntilNetworkIdle?(options?: { + idleTime?: number; + concurrency?: number; + }): Promise { + // Network idle detection is not applicable for iOS devices + await sleep(options?.idleTime || 1000); + } + + async evaluateJavaScript?(script: string): Promise { + throw new Error('evaluateJavaScript is not implemented for iOS devices'); + } +} diff --git a/packages/ios/src/utils/index.ts b/packages/ios/src/utils/index.ts new file mode 100644 index 000000000..b25b9c8d9 --- /dev/null +++ b/packages/ios/src/utils/index.ts @@ -0,0 +1,106 @@ +import { exec } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execAsync = promisify(exec); + +export interface ScreenInfo { + width: number; + height: number; + dpr: number; +} + +/** + * Get macOS screen size information + */ +export async function getScreenSize(): Promise { + try { + // Use system_profiler to get display information + const { stdout } = await execAsync( + 'system_profiler SPDisplaysDataType -json', + ); + const data = JSON.parse(stdout); + + // Find the main display + const displays = data.SPDisplaysDataType?.[0]?.spdisplays_ndrvs || []; + const mainDisplay = + displays.find( + (display: any) => + display._name?.includes('Built-in') || + display._name?.includes('Display'), + ) || displays[0]; + + if (!mainDisplay) { + throw new Error('No display found'); + } + + // Parse resolution string like "2880 x 1800" + const resolution = + mainDisplay.spdisplays_resolution || mainDisplay._spdisplays_resolution; + const match = resolution?.match(/(\d+)\s*x\s*(\d+)/); + + if (!match) { + throw new Error(`Unable to parse screen resolution: ${resolution}`); + } + + const width = Number.parseInt(match[1], 10); + const height = Number.parseInt(match[2], 10); + + // Try to get pixel ratio from system info + const pixelDensity = + mainDisplay.spdisplays_pixel_density || mainDisplay.spdisplays_density; + let dpr = 1; + + if (pixelDensity?.includes('Retina')) { + dpr = 2; // Most Retina displays have 2x pixel ratio + } + + return { + width, + height, + dpr, + }; + } catch (error) { + // Fallback: try to get screen size using screencapture + try { + console.warn('Using fallback method to get screen size'); + // This is a fallback - assuming common screen sizes + return { + width: 1920, + height: 1080, + dpr: 2, + }; + } catch (fallbackError) { + throw new Error(`Failed to get screen size: ${(error as Error).message}`); + } + } +} + +/** + * Start the PyAutoGUI server + */ +export async function startPyAutoGUIServer(port = 1412): Promise { + const { spawn } = await import('node:child_process'); + const path = await import('node:path'); + const { fileURLToPath } = await import('node:url'); + + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const serverPath = path.join(__dirname, '../../idb/auto_server.py'); + + const server = spawn('python3', [serverPath], { + stdio: 'inherit', + env: { + ...process.env, + PYTHONUNBUFFERED: '1', + }, + }); + + server.on('error', (error) => { + console.error('Failed to start PyAutoGUI server:', error); + throw error; + }); + + // Wait a bit for server to start + await new Promise((resolve) => setTimeout(resolve, 2000)); + + console.log(`PyAutoGUI server started on port ${port}`); +} diff --git a/packages/ios/tests/ios-input-test.ts b/packages/ios/tests/ios-input-test.ts new file mode 100644 index 000000000..8cfdae437 --- /dev/null +++ b/packages/ios/tests/ios-input-test.ts @@ -0,0 +1,56 @@ +import { agentFromPyAutoGUI } from '../src/agent/index'; + +async function testIOSInput() { + console.log('🧪 Testing iOS input functionality...'); + + try { + // Create iOS agent with mirror configuration + const agent = await agentFromPyAutoGUI({ + serverPort: 1412, + autoDismissKeyboard: true, + mirrorConfig: { + mirrorX: 692, + mirrorY: 161, + mirrorWidth: 344, + mirrorHeight: 764, + }, + }); + + console.log('✅ iOS agent created successfully'); + + // Test basic input functionality + console.log('🔤 Testing basic input...'); + + // Simulate clicking on a text input field (coordinates would be from AI detection) + await agent.page.tap({ left: 172, top: 300 }); // Example coordinates + console.log('📱 Tapped on input field'); + + // Test the new iOS input method + await agent.page.aiInputIOS('Hello iOS Testing!', { center: [172, 300] }); + console.log('✅ iOS input completed'); + + // Test keyboard press + await agent.page.keyboardPress('return'); + console.log('✅ Return key pressed'); + + // Test clearing input + await agent.page.clearInput({ center: [172, 300] }); + console.log('✅ Input cleared'); + + // Test regular input method + await agent.page.input('Regular input test'); + console.log('✅ Regular input completed'); + + console.log('🎉 All iOS input tests passed!'); + } catch (error) { + console.error('❌ iOS input test failed:', error); + process.exit(1); + } +} + +// Run the test +if (require.main === module) { + testIOSInput(); +} + +export { testIOSInput }; diff --git a/packages/ios/tests/unit-test/agent.test.ts b/packages/ios/tests/unit-test/agent.test.ts new file mode 100644 index 000000000..292a3d3c8 --- /dev/null +++ b/packages/ios/tests/unit-test/agent.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { iOSAgent } from '../../src/agent'; +import type { iOSDevice } from '../../src/page'; + +describe('iOS Agent', () => { + describe('constructor', () => { + it('should create an iOS agent instance', () => { + // Create a mock iOS device + const mockDevice = {} as iOSDevice; + + const agent = new iOSAgent(mockDevice); + + expect(agent).toBeDefined(); + expect(agent.page).toBe(mockDevice); + }); + }); +}); diff --git a/packages/ios/tests/unit-test/utils.test.ts b/packages/ios/tests/unit-test/utils.test.ts new file mode 100644 index 000000000..dadb95529 --- /dev/null +++ b/packages/ios/tests/unit-test/utils.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import type { ScreenInfo } from '../../src/utils'; + +describe('iOS Utils', () => { + describe('ScreenInfo interface', () => { + it('should have correct type definition', () => { + const screenInfo: ScreenInfo = { + width: 1920, + height: 1080, + dpr: 2, + }; + + expect(screenInfo.width).toBe(1920); + expect(screenInfo.height).toBe(1080); + expect(screenInfo.dpr).toBe(2); + }); + }); +}); diff --git a/packages/ios/tsconfig.json b/packages/ios/tsconfig.json new file mode 100644 index 000000000..2cf97401d --- /dev/null +++ b/packages/ios/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "allowJs": true, + "baseUrl": ".", + "declaration": true, + "emitDeclarationOnly": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "jsx": "preserve", + "lib": ["DOM", "ESNext"], + "moduleResolution": "node", + "resolveJsonModule": true, + "rootDir": "src", + "skipLibCheck": true, + "strict": true, + "module": "ES2020", + "target": "es2020", + "types": ["node"] + }, + "exclude": ["**/node_modules"], + "include": ["src"] +} diff --git a/packages/ios/vitest.config.ts b/packages/ios/vitest.config.ts new file mode 100644 index 000000000..4ac6027d5 --- /dev/null +++ b/packages/ios/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + }, +}); diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 7f8906799..2d3327f8a 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -72,7 +72,7 @@ const debug = getDebug('page-task-executor'); const defaultReplanningCycleLimit = 10; const isAndroidPage = (page: WebPage): page is AndroidDevicePage => { - return page.pageType === 'android'; + return page.pageType === 'android' || page.pageType === 'ios'; }; export class PageTaskExecutor { @@ -452,6 +452,7 @@ export class PageTaskExecutor { thought: plan.thought, locate: plan.locate, executor: async (taskParam, { element }) => { + // Clear existing content first if we have an element if (element) { await this.page.clearInput(element as unknown as ElementInfo); @@ -460,6 +461,8 @@ export class PageTaskExecutor { } } + // For iOS, the keyboard.type method will automatically use optimized input + // For other platforms, it will use the standard implementation await this.page.keyboard.type(taskParam.value, { autoDismissKeyboard: taskParam.autoDismissKeyboard, }); diff --git a/packages/web-integration/src/yaml/utils.ts b/packages/web-integration/src/yaml/utils.ts index a9d727a7e..ffadc41e7 100644 --- a/packages/web-integration/src/yaml/utils.ts +++ b/packages/web-integration/src/yaml/utils.ts @@ -42,6 +42,10 @@ export function parseYamlScript( typeof obj.android !== 'undefined' ? Object.assign({}, obj.android || {}) : undefined; + const ios = + typeof obj.ios !== 'undefined' + ? Object.assign({}, obj.ios || {}) + : undefined; const webConfig = obj.web || obj.target; // no need to handle null case, because web has required parameters url const web = typeof webConfig !== 'undefined' @@ -49,23 +53,26 @@ export function parseYamlScript( : undefined; if (!ignoreCheckingTarget) { - // make sure at least one of target/web/android is provided + // make sure at least one of target/web/android/ios is provided assert( - web || android, - `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`, + web || android || ios, + `at least one of "target", "web", "android", or "ios" properties is required in yaml script${pathTip}`, ); - // make sure only one of target/web/android is provided + // make sure only one of target/web/android/ios is provided + const configCount = [web, android, ios].filter(Boolean).length; assert( - (web && !android) || (!web && android), - `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`, + configCount === 1, + `only one of "target", "web", "android", or "ios" properties is allowed in yaml script${pathTip}`, ); // make sure the config is valid - if (web || android) { + if (web || android || ios) { assert( - typeof web === 'object' || typeof android === 'object', - `property "target/web/android" must be an object${pathTip}`, + typeof web === 'object' || + typeof android === 'object' || + typeof ios === 'object', + `property "target/web/android/ios" must be an object${pathTip}`, ); } } diff --git a/packages/web-integration/tests/unit-test/yaml/utils.test.ts b/packages/web-integration/tests/unit-test/yaml/utils.test.ts index 34cf2071f..fa203b680 100644 --- a/packages/web-integration/tests/unit-test/yaml/utils.test.ts +++ b/packages/web-integration/tests/unit-test/yaml/utils.test.ts @@ -65,6 +65,35 @@ tasks: expect(result.android?.deviceId).toBe('001234567890'); }); + test('ios configuration', () => { + const yamlContent = ` +ios: + serverPort: 1412 + serverUrl: "http://localhost:1412" + autoDismissKeyboard: true + mirrorConfig: + mirrorX: 100 + mirrorY: 200 + mirrorWidth: 400 + mirrorHeight: 800 + launch: "https://www.apple.com" + output: "./results.json" +tasks: +- sleep: 1000 +`; + + const result = parseYamlScript(yamlContent); + expect(result.ios?.serverPort).toBe(1412); + expect(result.ios?.serverUrl).toBe('http://localhost:1412'); + expect(result.ios?.autoDismissKeyboard).toBe(true); + expect(result.ios?.mirrorConfig?.mirrorX).toBe(100); + expect(result.ios?.mirrorConfig?.mirrorY).toBe(200); + expect(result.ios?.mirrorConfig?.mirrorWidth).toBe(400); + expect(result.ios?.mirrorConfig?.mirrorHeight).toBe(800); + expect(result.ios?.launch).toBe('https://www.apple.com'); + expect(result.ios?.output).toBe('./results.json'); + }); + test('illegal android deviceId', () => { const yamlContent = ` android: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82794ff9a..cc4cebb03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -462,6 +462,9 @@ importers: '@midscene/core': specifier: workspace:* version: link:../core + '@midscene/ios': + specifier: workspace:* + version: link:../ios '@midscene/shared': specifier: workspace:* version: link:../shared @@ -619,6 +622,40 @@ importers: specifier: 3.0.5 version: 3.0.5(@types/debug@4.1.12)(@types/node@22.15.3)(jsdom@26.1.0)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + packages/ios: + dependencies: + '@midscene/core': + specifier: workspace:* + version: link:../core + '@midscene/shared': + specifier: workspace:* + version: link:../shared + '@midscene/web': + specifier: workspace:* + version: link:../web-integration + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + devDependencies: + '@modern-js/module-tools': + specifier: ^2.60.3 + version: 2.60.6(debug@4.4.0)(typescript@5.8.3) + '@types/node': + specifier: ^22.10.5 + version: 22.15.3 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 + tsx: + specifier: ^4.17.0 + version: 4.19.2 + typescript: + specifier: ^5.7.2 + version: 5.8.3 + vitest: + specifier: ^2.1.8 + version: 2.1.9(@types/node@22.15.3)(jsdom@26.1.0)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + packages/mcp: dependencies: puppeteer: @@ -5812,9 +5849,6 @@ packages: '@types/node@16.9.1': resolution: {integrity: sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==} - '@types/node@18.19.118': - resolution: {integrity: sha512-hIPK0hSrrcaoAu/gJMzN3QClXE4QdCdFvaenJ0JsjIbExP1JFFVH+RHcBt25c9n8bx5dkIfqKE+uw6BmBns7ug==} - '@types/node@18.19.62': resolution: {integrity: sha512-UOGhw+yZV/icyM0qohQVh3ktpY40Sp7tdTW7HxG3pTd7AiMrlFlAJNUrGK9t5mdW0+ViQcFV74zCSIx9ZJpncA==} @@ -5929,9 +5963,23 @@ packages: peerDependencies: react: '>=18.3.1' + '@vitest/expect@2.1.9': + resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + '@vitest/expect@3.0.5': resolution: {integrity: sha512-nNIOqupgZ4v5jWuQx2DSlHLEs7Q4Oh/7AYwNyE+k0UQzG7tSmjPXShUikn1mpNGzYEN2jJbTvLejwShMitovBA==} + '@vitest/mocker@2.1.9': + resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/mocker@3.0.5': resolution: {integrity: sha512-CLPNBFBIE7x6aEGbIjaQAX03ZZlBMaWwAjBdMkIf/cAn6xzLTiM3zYqO/WAbieEjsAZir6tO71mzeHZoodThvw==} peerDependencies: @@ -5943,21 +5991,36 @@ packages: vite: optional: true + '@vitest/pretty-format@2.1.9': + resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + '@vitest/pretty-format@3.0.5': resolution: {integrity: sha512-CjUtdmpOcm4RVtB+up8r2vVDLR16Mgm/bYdkGFe3Yj/scRfCpbSi2W/BDSDcFK7ohw8UXvjMbOp9H4fByd/cOA==} '@vitest/pretty-format@3.1.1': resolution: {integrity: sha512-dg0CIzNx+hMMYfNmSqJlLSXEmnNhMswcn3sXO7Tpldr0LiGmg3eXdLLhwkv2ZqgHb/d5xg5F7ezNFRA1fA13yA==} + '@vitest/runner@2.1.9': + resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + '@vitest/runner@3.0.5': resolution: {integrity: sha512-BAiZFityFexZQi2yN4OX3OkJC6scwRo8EhRB0Z5HIGGgd2q+Nq29LgHU/+ovCtd0fOfXj5ZI6pwdlUmC5bpi8A==} + '@vitest/snapshot@2.1.9': + resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + '@vitest/snapshot@3.0.5': resolution: {integrity: sha512-GJPZYcd7v8QNUJ7vRvLDmRwl+a1fGg4T/54lZXe+UOGy47F9yUfE18hRCtXL5aHN/AONu29NGzIXSVFh9K0feA==} + '@vitest/spy@2.1.9': + resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + '@vitest/spy@3.0.5': resolution: {integrity: sha512-5fOzHj0WbUNqPK6blI/8VzZdkBlQLnT25knX0r4dbZI9qoZDf3qAdjoMmDcLG5A83W6oUUFJgUd0EYBc2P5xqg==} + '@vitest/utils@2.1.9': + resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + '@vitest/utils@3.0.5': resolution: {integrity: sha512-N9AX0NUoUtVwKwy21JtwzaqR5L5R5A99GAbrHfCCXK1lp593i/3AZAXhSP43wRQuxYsflrdzEfXZFo1reR1Nkg==} @@ -10188,6 +10251,10 @@ packages: resolution: {integrity: sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-html-parser@6.1.13: resolution: {integrity: sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg==} @@ -10587,6 +10654,9 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -12732,6 +12802,10 @@ packages: resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} engines: {node: ^18.0.0 || >=20.0.0} + tinyrainbow@1.2.0: + resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + engines: {node: '>=14.0.0'} + tinyrainbow@2.0.0: resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} engines: {node: '>=14.0.0'} @@ -13197,6 +13271,11 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite-node@2.1.9: + resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + vite-node@3.0.5: resolution: {integrity: sha512-02JEJl7SbtwSDJdYS537nU6l+ktdvcREfLksk/NDAqtdKWGqHl+joXzEubHROmS3E6pip+Xgu2tFezMu75jH7A==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -13233,6 +13312,31 @@ packages: terser: optional: true + vitest@2.1.9: + resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 2.1.9 + '@vitest/ui': 2.1.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@3.0.5: resolution: {integrity: sha512-4dof+HvqONw9bvsYxtkfUp2uHsTN9bV2CZIi1pWgoFpL1Lld8LA1ka9q/ONSsoScAKG7NVGf2stJTI7XRkXb2Q==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -20120,11 +20224,11 @@ snapshots: '@types/compression@1.8.1': dependencies: '@types/express': 4.17.21 - '@types/node': 18.19.118 + '@types/node': 22.15.3 '@types/connect@3.4.38': dependencies: - '@types/node': 18.19.62 + '@types/node': 22.15.3 '@types/conventional-commits-parser@5.0.1': dependencies: @@ -20225,7 +20329,7 @@ snapshots: '@types/http-proxy@1.17.16': dependencies: - '@types/node': 18.19.118 + '@types/node': 22.15.3 '@types/http-server@0.12.4': dependencies: @@ -20284,10 +20388,6 @@ snapshots: '@types/node@16.9.1': {} - '@types/node@18.19.118': - dependencies: - undici-types: 5.26.5 - '@types/node@18.19.62': dependencies: undici-types: 5.26.5 @@ -20405,6 +20505,13 @@ snapshots: react: 19.1.0 unhead: 2.0.13 + '@vitest/expect@2.1.9': + dependencies: + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.2.0 + tinyrainbow: 1.2.0 + '@vitest/expect@3.0.5': dependencies: '@vitest/spy': 3.0.5 @@ -20412,6 +20519,14 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 + '@vitest/mocker@2.1.9(vite@5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1))': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.17 + optionalDependencies: + vite: 5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + '@vitest/mocker@3.0.5(vite@5.4.10(@types/node@18.19.62)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1))': dependencies: '@vitest/spy': 3.0.5 @@ -20428,6 +20543,10 @@ snapshots: optionalDependencies: vite: 5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + '@vitest/pretty-format@2.1.9': + dependencies: + tinyrainbow: 1.2.0 + '@vitest/pretty-format@3.0.5': dependencies: tinyrainbow: 2.0.0 @@ -20436,21 +20555,42 @@ snapshots: dependencies: tinyrainbow: 2.0.0 + '@vitest/runner@2.1.9': + dependencies: + '@vitest/utils': 2.1.9 + pathe: 1.1.2 + '@vitest/runner@3.0.5': dependencies: '@vitest/utils': 3.0.5 pathe: 2.0.3 + '@vitest/snapshot@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + magic-string: 0.30.17 + pathe: 1.1.2 + '@vitest/snapshot@3.0.5': dependencies: '@vitest/pretty-format': 3.0.5 magic-string: 0.30.17 pathe: 2.0.3 + '@vitest/spy@2.1.9': + dependencies: + tinyspy: 3.0.2 + '@vitest/spy@3.0.5': dependencies: tinyspy: 3.0.2 + '@vitest/utils@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + loupe: 3.1.3 + tinyrainbow: 1.2.0 + '@vitest/utils@3.0.5': dependencies: '@vitest/pretty-format': 3.0.5 @@ -26065,6 +26205,12 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-html-parser@6.1.13: dependencies: css-select: 5.2.2 @@ -26546,6 +26692,8 @@ snapshots: path-type@4.0.0: {} + pathe@1.1.2: {} + pathe@2.0.3: {} pathval@2.0.0: {} @@ -29100,6 +29248,8 @@ snapshots: tinypool@1.1.1: {} + tinyrainbow@1.2.0: {} + tinyrainbow@2.0.0: {} tinyspy@3.0.2: {} @@ -29599,6 +29749,24 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.2 + vite-node@2.1.9(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1): + dependencies: + cac: 6.7.14 + debug: 4.4.0(supports-color@5.5.0) + es-module-lexer: 1.7.0 + pathe: 1.1.2 + vite: 5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite-node@3.0.5(@types/node@18.19.62)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1): dependencies: cac: 6.7.14 @@ -29661,6 +29829,42 @@ snapshots: sass-embedded: 1.86.3 terser: 5.43.1 + vitest@2.1.9(@types/node@22.15.3)(jsdom@26.1.0)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1): + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(vite@5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1)) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.2.0 + debug: 4.4.0(supports-color@5.5.0) + expect-type: 1.2.1 + magic-string: 0.30.17 + pathe: 1.1.2 + std-env: 3.9.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.0.2 + tinyrainbow: 1.2.0 + vite: 5.4.10(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + vite-node: 2.1.9(@types/node@22.15.3)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.15.3 + jsdom: 26.1.0 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vitest@3.0.5(@types/debug@4.1.12)(@types/node@18.19.62)(jsdom@26.1.0)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1): dependencies: '@vitest/expect': 3.0.5 From 777d4c2c31c6195e39dc4bf923997b5eaa7258c4 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 18:49:58 +0800 Subject: [PATCH 02/17] refactor(ios): update comments in page/index.ts --- packages/ios/src/page/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index cfcd301f4..b9e7549e0 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -568,8 +568,8 @@ export class iOSDevice implements AndroidDevicePage { get mouse(): any { return { click: async (x: number, y: number, options: { button: string }) => { - // 直接使用传入的坐标，因为这些坐标已经是iOS坐标系的 - // 在executePyAutoGUIAction中会进行iOS到macOS的坐标变换 + // Directly use the provided coordinates, as these are already in the iOS coordinate system. + // The coordinate transformation from iOS to macOS will be handled inside executePyAutoGUIAction. await this.executePyAutoGUIAction({ action: 'click', x: x, From ae52bfe296c132cc775336507aa8eadf58ee6e89 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 18:53:43 +0800 Subject: [PATCH 03/17] refactor(ios): update comments in page/index.ts --- packages/ios/src/page/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index b9e7549e0..953ddc715 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -348,8 +348,6 @@ export class iOSDevice implements AndroidDevicePage { async tap(point: Point): Promise { debugPage(`tap at (${point.left}, ${point.top})`); - // 对于iOS mirroring模式，直接传递iOS坐标，让Python服务器处理坐标变换 - // 对于非mirroring模式，使用设备像素比调整坐标 if (this.options?.iOSMirrorConfig) { await this.executePyAutoGUIAction({ action: 'click', From d80b0d628638d824a2521b3ecc1499f9df130901 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 21:15:59 +0800 Subject: [PATCH 04/17] refactor(ios): rename iOSMirrorConfig to mirrorConfig and update examples --- examples/ios-input-example.yaml | 6 ++--- examples/ios-input-test.ts | 2 +- examples/ios-input-test.yaml | 2 +- packages/cli/src/create-yaml-player.ts | 2 +- packages/ios/docs/ios-mirroring-guide.md | 11 +++----- packages/ios/examples/ios-mirroring-demo.js | 2 +- packages/ios/src/agent/index.ts | 2 +- packages/ios/src/page/index.ts | 29 ++++++++------------- 8 files changed, 23 insertions(+), 33 deletions(-) diff --git a/examples/ios-input-example.yaml b/examples/ios-input-example.yaml index fa26f528e..b8846cc0c 100644 --- a/examples/ios-input-example.yaml +++ b/examples/ios-input-example.yaml @@ -11,8 +11,8 @@ ios: # iOS device mirroring configuration for precise location targeting # These values define the position and size of the mirrored device screen mirrorConfig: - mirrorX: 692 # X position of iOS mirror on computer screen - mirrorY: 161 # Y position of iOS mirror on computer screen + mirrorX: 788 # X position of iOS mirror on computer screen + mirrorY: 133 # Y position of iOS mirror on computer screen mirrorWidth: 344 # Width of the mirrored iOS screen mirrorHeight: 764 # Height of the mirrored iOS screen (iPhone 11 Pro size) @@ -25,7 +25,7 @@ tasks: - sleep: 5000 - aiAction: "打开音乐应用" - sleep: 2000 - - aiTap: "搜索按钮" + - aiTap: "搜索图标" - sleep: 3000 - aiInput: "Coldplay" locate: "Search box" diff --git a/examples/ios-input-test.ts b/examples/ios-input-test.ts index 9c844dac9..2799540e6 100644 --- a/examples/ios-input-test.ts +++ b/examples/ios-input-test.ts @@ -23,7 +23,7 @@ async function testIOSInput() { const options: iOSDeviceOpt = { serverPort: 1412, autoDismissKeyboard: true, - iOSMirrorConfig: { + mirrorConfig: { mirrorX: 692, // X position of iOS mirror on screen mirrorY: 161, // Y position of iOS mirror on screen mirrorWidth: 344, // Width of the mirrored iOS screen diff --git a/examples/ios-input-test.yaml b/examples/ios-input-test.yaml index 2533027a9..4ef6020e5 100644 --- a/examples/ios-input-test.yaml +++ b/examples/ios-input-test.yaml @@ -5,7 +5,7 @@ ios: serverPort: 1412 autoDismissKeyboard: true - iOSMirrorConfig: + mirrorConfig: mirrorX: 692 mirrorY: 161 mirrorWidth: 344 diff --git a/packages/cli/src/create-yaml-player.ts b/packages/cli/src/create-yaml-player.ts index 916f46198..7765901d3 100644 --- a/packages/cli/src/create-yaml-player.ts +++ b/packages/cli/src/create-yaml-player.ts @@ -169,7 +169,7 @@ export async function createYamlPlayer( serverUrl: iosTarget.serverUrl, serverPort: iosTarget.serverPort, autoDismissKeyboard: iosTarget.autoDismissKeyboard, - iOSMirrorConfig: iosTarget.mirrorConfig, + mirrorConfig: iosTarget.mirrorConfig, }); if (iosTarget?.launch) { diff --git a/packages/ios/docs/ios-mirroring-guide.md b/packages/ios/docs/ios-mirroring-guide.md index 7ea4dda7c..c31ee144e 100644 --- a/packages/ios/docs/ios-mirroring-guide.md +++ b/packages/ios/docs/ios-mirroring-guide.md @@ -18,11 +18,8 @@ iOS device mirroring allows you to control iOS devices through their screen repr pip3 install flask pyautogui ``` -2. **iOS Device Mirroring Setup** (choose one): - - **QuickTime Player**: Connect iOS device → File → New Movie Recording → Select iOS device +2. **iOS Device Mirroring Setup**: - **iPhone Mirroring** (macOS Sequoia): Built-in iOS mirroring feature - - **iOS Simulator**: Xcode's iOS Simulator - - **Third-party tools**: Reflector, AirServer, etc. 3. **Screen Position**: Note the exact position and size of iOS mirror on your macOS screen @@ -35,7 +32,7 @@ import { iOSDevice, iOSAgent } from '@midscene/ios'; const device = new iOSDevice({ serverPort: 1412, - iOSMirrorConfig: { + mirrorConfig: { mirrorX: 100, // Mirror position X on macOS screen mirrorY: 50, // Mirror position Y on macOS screen mirrorWidth: 400, // Mirror width on macOS screen @@ -281,12 +278,12 @@ await device.configureIOSMirror({ // Control multiple iOS devices const device1 = new iOSDevice({ serverPort: 1412, - iOSMirrorConfig: config1 + mirrorConfig: config1 }); const device2 = new iOSDevice({ serverPort: 1413, // Different server instance - iOSMirrorConfig: config2 + mirrorConfig: config2 }); ``` diff --git a/packages/ios/examples/ios-mirroring-demo.js b/packages/ios/examples/ios-mirroring-demo.js index d3e201e0b..180c4b231 100644 --- a/packages/ios/examples/ios-mirroring-demo.js +++ b/packages/ios/examples/ios-mirroring-demo.js @@ -66,7 +66,7 @@ async function demonstrateIOSMirroring() { const device = new iOSDevice({ serverPort: 1412, - iOSMirrorConfig: mirrorConfig, + mirrorConfig: mirrorConfig, }); try { diff --git a/packages/ios/src/agent/index.ts b/packages/ios/src/agent/index.ts index e10fd8ad6..0526ad67e 100644 --- a/packages/ios/src/agent/index.ts +++ b/packages/ios/src/agent/index.ts @@ -49,7 +49,7 @@ export async function agentFromPyAutoGUI(opts?: iOSAgentOpt & iOSDeviceOpt) { serverUrl: opts?.serverUrl, serverPort, autoDismissKeyboard: opts?.autoDismissKeyboard, - iOSMirrorConfig: opts?.iOSMirrorConfig || opts?.mirrorConfig, + mirrorConfig: opts?.mirrorConfig, }); await page.connect(); diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 953ddc715..36c98259d 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -18,13 +18,6 @@ export interface iOSDeviceOpt extends AndroidDeviceInputOpt { serverPort?: number; autoDismissKeyboard?: boolean; // iOS device mirroring configuration - iOSMirrorConfig?: { - mirrorX: number; - mirrorY: number; - mirrorWidth: number; - mirrorHeight: number; - }; - // Alternative name for better API compatibility mirrorConfig?: { mirrorX: number; mirrorY: number; @@ -116,8 +109,8 @@ export class iOSDevice implements AndroidDevicePage { } // Configure iOS mirroring if provided - if (this.options?.iOSMirrorConfig) { - await this.configureIOSMirror(this.options.iOSMirrorConfig); + if (this.options?.mirrorConfig) { + await this.configureIOSMirror(this.options.mirrorConfig); } // Get screen information (will use iOS dimensions if configured) @@ -200,7 +193,7 @@ export class iOSDevice implements AndroidDevicePage { async size(): Promise { // 对于iOS镜像模式，返回iOS设备的逻辑尺寸而不是macOS屏幕尺寸 - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { // 从Python服务器获取配置信息，使用估算的iOS设备尺寸 try { const config = await this.getConfiguration(); @@ -252,7 +245,7 @@ export class iOSDevice implements AndroidDevicePage { try { // Use PyAutoGUI server's screenshot functionality for iOS mirroring - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { const result = await this.executePyAutoGUIAction({ action: 'screenshot', }); @@ -348,7 +341,7 @@ export class iOSDevice implements AndroidDevicePage { async tap(point: Point): Promise { debugPage(`tap at (${point.left}, ${point.top})`); - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'click', x: point.left, @@ -367,7 +360,7 @@ export class iOSDevice implements AndroidDevicePage { async hover(point: Point): Promise { debugPage(`hover at (${point.left}, ${point.top})`); - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'move', x: point.left, @@ -528,7 +521,7 @@ export class iOSDevice implements AndroidDevicePage { }; // Always use mouse wheel/trackpad for scrolling (better compatibility) - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { // iOS mirroring mode: use iOS coordinates directly await this.executePyAutoGUIAction(scrollAction); } else { @@ -585,7 +578,7 @@ export class iOSDevice implements AndroidDevicePage { to: { x: number; y: number }, ) => { // 对于iOS镜像模式，直接传递坐标；对于非镜像模式，使用设备像素比调整 - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'drag', x: from.x, @@ -754,7 +747,7 @@ export class iOSDevice implements AndroidDevicePage { } async longPress(x: number, y: number, duration?: number): Promise { - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'click', x: x, @@ -790,7 +783,7 @@ export class iOSDevice implements AndroidDevicePage { top: start.top + (distance || screenSize.height / 3), }; - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'drag', x: start.left, @@ -827,7 +820,7 @@ export class iOSDevice implements AndroidDevicePage { top: start.top - (distance || screenSize.height / 3), }; - if (this.options?.iOSMirrorConfig) { + if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'drag', x: start.left, From 89710677a73584bb8f78aa00d3463f3bc73ee0d5 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 21:22:38 +0800 Subject: [PATCH 05/17] refactor(ios): update comments and prompts to English --- packages/ios/examples/ios-mirroring-demo.js | 4 ++-- packages/ios/src/page/index.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/ios/examples/ios-mirroring-demo.js b/packages/ios/examples/ios-mirroring-demo.js index 180c4b231..e8092b2b3 100644 --- a/packages/ios/examples/ios-mirroring-demo.js +++ b/packages/ios/examples/ios-mirroring-demo.js @@ -137,8 +137,8 @@ async function demonstrateIOSMirroring() { // Example AI operations (commented out as they need actual iOS app content) await agent.aiTap('Settings app icon'); - await agent.ai('返回主屏幕'); - // await agent.ai("在 显示与亮度中 开启深色模式") + await agent.ai('Go back to home screen'); + // await agent.ai("Enable dark mode in Display & Brightness settings") console.log('✅ Demo completed successfully!\n'); diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 36c98259d..5dea63548 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -192,16 +192,16 @@ export class iOSDevice implements AndroidDevicePage { } async size(): Promise { - // 对于iOS镜像模式，返回iOS设备的逻辑尺寸而不是macOS屏幕尺寸 + // For iOS mirroring mode, return iOS device logical size instead of macOS screen size if (this.options?.mirrorConfig) { - // 从Python服务器获取配置信息，使用估算的iOS设备尺寸 + // Get configuration from Python server, using estimated iOS device size try { const config = await this.getConfiguration(); if (config.status === 'ok' && config.config.enabled) { return { width: config.config.estimated_ios_width, height: config.config.estimated_ios_height, - dpr: 1, // iOS坐标系不需要额外的像素比调整 + dpr: 1, // iOS coordinate system doesn't need additional pixel ratio adjustment }; } } catch (error) { @@ -209,7 +209,7 @@ export class iOSDevice implements AndroidDevicePage { } } - // 非iOS镜像模式或配置获取失败时的fallback + // Fallback for non-iOS mirroring mode or when configuration retrieval fails if (!this.screenInfo) { this.screenInfo = await getScreenSize(); } @@ -577,7 +577,7 @@ export class iOSDevice implements AndroidDevicePage { from: { x: number; y: number }, to: { x: number; y: number }, ) => { - // 对于iOS镜像模式，直接传递坐标；对于非镜像模式，使用设备像素比调整 + // For iOS mirroring mode, pass coordinates directly; for non-mirroring mode, adjust using device pixel ratio if (this.options?.mirrorConfig) { await this.executePyAutoGUIAction({ action: 'drag', From 312c0730e3312b95ee1635a4111b9ad910a0aa13 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 21:54:45 +0800 Subject: [PATCH 06/17] feat(ios): update mirror config and improve window rect script --- examples/ios-input-example.yaml | 8 ++--- packages/ios/scripts/getAppWindowRect.scpt | 35 +++++++++++++++++----- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/examples/ios-input-example.yaml b/examples/ios-input-example.yaml index b8846cc0c..f90a9cd93 100644 --- a/examples/ios-input-example.yaml +++ b/examples/ios-input-example.yaml @@ -11,10 +11,10 @@ ios: # iOS device mirroring configuration for precise location targeting # These values define the position and size of the mirrored device screen mirrorConfig: - mirrorX: 788 # X position of iOS mirror on computer screen - mirrorY: 133 # Y position of iOS mirror on computer screen - mirrorWidth: 344 # Width of the mirrored iOS screen - mirrorHeight: 764 # Height of the mirrored iOS screen (iPhone 11 Pro size) + mirrorX: 794 # X position of iOS mirror on computer screen + mirrorY: 171 # Y position of iOS mirror on computer screen + mirrorWidth: 332 # Width of the mirrored iOS screen + mirrorHeight: 720 # Height of the mirrored iOS screen (iPhone 11 Pro size) # Output file for aiQuery/aiAssert results (optional) output: "./results.json" diff --git a/packages/ios/scripts/getAppWindowRect.scpt b/packages/ios/scripts/getAppWindowRect.scpt index 735c3f8d3..fc0b35ecd 100644 --- a/packages/ios/scripts/getAppWindowRect.scpt +++ b/packages/ios/scripts/getAppWindowRect.scpt @@ -1,11 +1,30 @@ delay 4 -- you have 4 seconds to make iPhone Mirroring App foreground!! tell application "System Events" - set frontApp to name of first application process whose frontmost is true - tell application process frontApp - set win to first window - set pos to position of win - set size_ to size of win - return {frontApp, pos, size_} - end tell -end tell + set frontApp to name of first application process whose frontmost is true + tell application process frontApp + set win to first window + set pos to position of win + set size_ to size of win + + -- set margrin + set leftMargin to 6 + set rightMargin to 6 + set topMargin to 38 + set bottomMargin to 6 + + -- original + set originalX to item 1 of pos + set originalY to item 2 of pos + set originalWidth to item 1 of size_ + set originalHeight to item 2 of size_ + + -- clipped + set contentX to originalX + leftMargin + set contentY to originalY + topMargin + set contentWidth to originalWidth - leftMargin - rightMargin + set contentHeight to originalHeight - topMargin - bottomMargin + + return {frontApp, pos, size_, {contentX, contentY, contentWidth, contentHeight}} + end tell +end tell \ No newline at end of file From ee6bdd77d1376c24540d3793895c0f9926e22137 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 22:43:48 +0800 Subject: [PATCH 07/17] refactor(ios): simplify scrolling logic in auto_server.py --- packages/ios/idb/auto_server.py | 83 ++------------------------------- 1 file changed, 5 insertions(+), 78 deletions(-) diff --git a/packages/ios/idb/auto_server.py b/packages/ios/idb/auto_server.py index 4c516df65..9268e23a8 100644 --- a/packages/ios/idb/auto_server.py +++ b/packages/ios/idb/auto_server.py @@ -261,92 +261,19 @@ def handle_action(action): method = "horizontal_scroll" success = False - # Method 1: Try using hscroll if available (rarely works on macOS) if hasattr(pyautogui, 'hscroll'): - try: - scroll_amount = clicks * 3 if direction == "right" else -clicks * 3 - pyautogui.hscroll(scroll_amount, x=mac_x, y=mac_y) - print(f" ✅ Used hscroll method: {scroll_amount}") - success = True - except Exception as e: - print(f" ❌ hscroll failed: {e}") - - # Method 2: Try AppleScript system events (macOS native approach) - if not success: - try: - scroll_amount = clicks * 5 # More units for better effect - scroll_direction = "right" if direction == "right" else "left" - - # Use AppleScript to simulate horizontal scroll - applescript = f''' - tell application "System Events" - set mousePosition to {{{mac_x}, {mac_y}}} - set mouseLoc to mousePosition - repeat {clicks} times - tell application "System Events" to scroll mouseLoc horizontally by {5 if direction == "right" else -5} - delay 0.01 - end repeat - end tell - ''' - - success, stdout, stderr = execute_applescript(applescript) - if success: - print(f" ✅ Used AppleScript horizontal scroll: {scroll_direction}") - else: - print(f" ❌ AppleScript failed: {stderr}") - except Exception as e: - print(f" ❌ AppleScript method failed: {e}") - - # Method 3: Drag simulation (most reliable fallback) - if not success: - try: - print(f" 🖱️ Using drag simulation for horizontal scroll") - start_x = mac_x - start_y = mac_y - - # Calculate drag distance based on clicks (more aggressive) - drag_distance = min(clicks * 15, 300) # Cap at 300px - - if direction == "right": - end_x = start_x - drag_distance # Drag left to scroll right - else: - end_x = start_x + drag_distance # Drag right to scroll left - - # Ensure we don't drag outside reasonable bounds - screen_width = pyautogui.size().width - end_x = max(50, min(end_x, screen_width - 50)) - - # Perform smooth drag scroll - pyautogui.moveTo(start_x, start_y) - time.sleep(0.1) # Brief pause - pyautogui.mouseDown() - pyautogui.moveTo(end_x, start_y, duration=0.4) # Slower for iOS compatibility - pyautogui.mouseUp() - - print(f" ✅ Drag scroll: ({start_x}, {start_y}) -> ({end_x}, {start_y}), distance: {abs(end_x - start_x)}px") - success = True - method = "horizontal_drag_scroll" - except Exception as e: - print(f" ❌ Drag simulation failed: {e}") - - # Final fallback: shift+scroll (even though it might not work) - if not success: - print(f" ⚠️ All methods failed, trying shift+scroll fallback") - pyautogui.keyDown('shift') for i in range(clicks): - scroll_amount = 8 if direction == "left" else -8 - pyautogui.scroll(scroll_amount, x=mac_x, y=mac_y) - time.sleep(0.008) - pyautogui.keyUp('shift') - method = "horizontal_scroll_fallback" + scroll_amount = 20 if direction == "left" else -20 + pyautogui.hscroll(scroll_amount, x=mac_x, y=mac_y) + else: + raise NotImplementedError("Horizontal scrolling not supported on this platform") else: print(f"⬆️⬇️ VERTICAL SCROLL: {direction}") # Vertical scrolling (this should work fine) for i in range(clicks): - scroll_amount = 8 if direction == "up" else -8 + scroll_amount = 20 if direction == "up" else -20 pyautogui.scroll(scroll_amount, x=mac_x, y=mac_y) - time.sleep(0.008) # Fast succession method = "vertical_scroll" print(f"✅ Scroll completed: {direction} ({clicks} iterations)") From 26aefaa4934f65d2914c3775866c79ac3a058398 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 4 Aug 2025 22:57:03 +0800 Subject: [PATCH 08/17] feat(ios): activate mirroring app on connect and update example --- examples/ios-input-example.yaml | 2 +- packages/ios/src/page/index.ts | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/examples/ios-input-example.yaml b/examples/ios-input-example.yaml index f90a9cd93..393e8da89 100644 --- a/examples/ios-input-example.yaml +++ b/examples/ios-input-example.yaml @@ -28,7 +28,7 @@ tasks: - aiTap: "搜索图标" - sleep: 3000 - aiInput: "Coldplay" - locate: "Search box" + locate: "Search input field" - sleep: 2000 - aiKeyboardPress: "Enter" - sleep: 3000 diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 5dea63548..8cb8e310b 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -102,6 +102,27 @@ export class iOSDevice implements AndroidDevicePage { } const healthData = await response.json(); debugPage(`Python server is running: ${JSON.stringify(healthData)}`); + + // Make iPhone mirroring app foreground + try { + // Use fixed mirroring app name for iOS device screen mirroring + const mirroringAppName = 'iPhone Mirroring'; + + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + // Activate the mirroring application using AppleScript + await execAsync( + `osascript -e 'tell application "${mirroringAppName}" to activate'`, + ); + debugPage(`Activated iOS mirroring app: ${mirroringAppName}`); + } catch (mirrorError: any) { + debugPage( + `Warning: Failed to bring iOS mirroring app to foreground: ${mirrorError.message}`, + ); + // Continue execution even if this fails - it's not critical + } } catch (error: any) { throw new Error( `Failed to connect to Python server at ${this.serverUrl}: ${error.message}`, From c525143fe308fcb19f92c331da30a7662bb0d130 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Tue, 5 Aug 2025 07:16:33 +0800 Subject: [PATCH 09/17] refactor(ios): reorganize iOS-related files and remove obsolete examples --- examples/README-iOS.md | 112 ------- examples/ios-input-test.ts | 82 ----- examples/ios-input-test.yaml | 46 --- examples/ios-yaml-example.yaml | 70 ---- packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md | 131 -------- packages/ios/docs/ios-mirroring-guide.md | 312 ------------------ .../ios/examples}/ios-input-example.yaml | 4 +- packages/ios/getAppWindowRect.scpt | Bin 2224 -> 0 bytes 8 files changed, 2 insertions(+), 755 deletions(-) delete mode 100644 examples/README-iOS.md delete mode 100644 examples/ios-input-test.ts delete mode 100644 examples/ios-input-test.yaml delete mode 100644 examples/ios-yaml-example.yaml delete mode 100644 packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md delete mode 100644 packages/ios/docs/ios-mirroring-guide.md rename {examples => packages/ios/examples}/ios-input-example.yaml (93%) delete mode 100644 packages/ios/getAppWindowRect.scpt diff --git a/examples/README-iOS.md b/examples/README-iOS.md deleted file mode 100644 index 94c754a00..000000000 --- a/examples/README-iOS.md +++ /dev/null @@ -1,112 +0,0 @@ -# iOS YAML Automation Examples - -This directory contains examples of using Midscene.js with iOS devices through YAML configuration files. - -## Prerequisites - -1. **PyAutoGUI Server**: You need to have a PyAutoGUI server running on your system to communicate with the iOS device. -2. **iOS Device Mirroring**: Your iOS device should be mirrored to your computer screen (using tools like QuickTime Player, AirServer, or similar). -3. **Midscene CLI**: Install the Midscene CLI tool: `npm install -g @midscene/cli` - -## Configuration - -### Basic iOS Configuration - -```yaml -ios: - # Server configuration (required for iOS automation) - serverPort: 1412 - serverUrl: "http://localhost:1412" - - # Mirror configuration (required for precise targeting) - mirrorConfig: - mirrorX: 100 # X position of the mirrored iOS screen - mirrorY: 200 # Y position of the mirrored iOS screen - mirrorWidth: 414 # Width of the mirrored screen - mirrorHeight: 896 # Height of the mirrored screen -``` - -### Optional Configuration - -```yaml -ios: - # Auto dismiss keyboard after input (optional) - autoDismissKeyboard: true - - # Launch URL or app when starting (optional) - launch: "https://example.com" - - # Output file for results (optional) - output: "./results.json" -``` - -## Examples - -### 1. Simple iOS Test (`ios-yaml-example.yaml`) - -A basic example showing iOS automation with Safari browser interaction. - -### 2. Comprehensive Example (`ios-comprehensive-example.yaml`) - -A more complex example demonstrating: -- Safari navigation -- Search functionality -- Data extraction -- Settings app interaction -- Home screen operations - -### 3. Configuration File (`ios-config.yaml`) - -Shows how to use a configuration file to set global iOS settings for multiple test scripts. - -## Running the Examples - -### Single Script - -```bash -# Run a single iOS automation script -midscene ./ios-yaml-example.yaml -``` - -### Multiple Scripts with Configuration - -```bash -# Run multiple scripts using a configuration file -midscene --config ./ios-config.yaml -``` - -### Command Line Options - -You can override iOS settings from the command line: - -```bash -# Override mirror settings -midscene --ios.mirrorX 150 --ios.mirrorY 250 ./ios-yaml-example.yaml - -# Override server port -midscene --ios.serverPort 1413 ./ios-yaml-example.yaml -``` - -## Mirror Configuration Setup - -1. **Connect your iOS device** to your computer -2. **Enable mirroring** (e.g., using QuickTime Player's "New Movie Recording" and select your iOS device) -3. **Measure the mirror position and size** on your computer screen -4. **Update the mirrorConfig** values in your YAML file: - - `mirrorX` and `mirrorY`: Top-left corner coordinates of the mirrored screen - - `mirrorWidth` and `mirrorHeight`: Dimensions of the mirrored screen - -## Tips - -- Make sure the PyAutoGUI server is running before executing the scripts -- Adjust the `sleep` durations based on your device's performance -- Test the mirror configuration with simple actions first -- Use descriptive prompts in `aiAction` for better AI understanding -- The `aiAssert` statements help verify that actions completed successfully - -## Troubleshooting - -- **Connection issues**: Verify the PyAutoGUI server is running on the specified port -- **Targeting issues**: Double-check your mirror configuration coordinates -- **Performance issues**: Increase sleep durations between actions -- **Recognition issues**: Use more descriptive text in your AI prompts diff --git a/examples/ios-input-test.ts b/examples/ios-input-test.ts deleted file mode 100644 index 2799540e6..000000000 --- a/examples/ios-input-test.ts +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env tsx -/** - * iOS Input Test - Demonstrates the improved iOS input functionality - * - * This test shows how the iOS input system now automatically handles: - * - Element focusing by tapping - * - Content clearing with cmd+a and delete - * - Optimized typing with proper intervals for iOS keyboards - * - Automatic keyboard dismissal - * - * The beauty is that it all happens transparently - no special iOS methods needed! - */ - -import { agentFromPyAutoGUI } from '../packages/ios/src/agent'; -import type { iOSDeviceOpt } from '../packages/ios/src/page'; - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -async function testIOSInput() { - console.log('🚀 Starting iOS Input Test...'); - - // Configure for iOS device mirroring - adjust these coordinates for your setup - const options: iOSDeviceOpt = { - serverPort: 1412, - autoDismissKeyboard: true, - mirrorConfig: { - mirrorX: 692, // X position of iOS mirror on screen - mirrorY: 161, // Y position of iOS mirror on screen - mirrorWidth: 344, // Width of the mirrored iOS screen - mirrorHeight: 764, // Height of the mirrored iOS screen - }, - }; - - try { - // Create agent - this will automatically start the PyAutoGUI server if needed - const agent = await agentFromPyAutoGUI(options); - console.log('✅ iOS Agent created successfully'); - - // Test 1: Simple text input - console.log('\n📝 Test 1: Simple text input using aiInput'); - await agent.aiInput('Hello iOS!', 'search box or text field'); - await sleep(2000); - - // Test 2: Email input with special characters - console.log('\n📧 Test 2: Email input with special characters'); - await agent.aiInput('test@example.com', 'email input field'); - await sleep(2000); - - // Test 3: Multi-word text with spaces - console.log('\n📄 Test 3: Multi-word text input'); - await agent.aiInput( - 'This is a longer text message with spaces', - 'text area or message field', - ); - await sleep(2000); - - // Test 4: Numbers and symbols - console.log('\n🔢 Test 4: Numbers and symbols'); - await agent.aiInput('Password123!@#', 'password field'); - await sleep(2000); - - // Test 5: Clear and replace existing text - console.log('\n🔄 Test 5: Clear and replace existing text'); - await agent.aiInput('', 'input field'); // Clear the field - await sleep(1000); - await agent.aiInput('New replacement text', 'same input field'); - await sleep(2000); - - console.log('\n✅ All iOS input tests completed successfully!'); - console.log('🎉 The iOS input system is working properly with:'); - console.log(' - Automatic element focusing'); - console.log(' - Smart content clearing'); - console.log(' - Optimized typing intervals'); - console.log(' - Automatic keyboard dismissal'); - } catch (error) { - console.error('❌ iOS Input Test failed:', error); - process.exit(1); - } -} - -// Run the test -testIOSInput().catch(console.error); diff --git a/examples/ios-input-test.yaml b/examples/ios-input-test.yaml deleted file mode 100644 index 4ef6020e5..000000000 --- a/examples/ios-input-test.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# iOS Input Test - YAML Version -# This demonstrates how the improved iOS input system works seamlessly -# through the standard YAML automation interface - -ios: - serverPort: 1412 - autoDismissKeyboard: true - mirrorConfig: - mirrorX: 692 - mirrorY: 161 - mirrorWidth: 344 - mirrorHeight: 764 - -tasks: - - name: test basic text input - flow: - - aiInput: "Hello from iOS!" - locate: "search box or text field" - - sleep: 2000 - - - name: test email input - flow: - - aiInput: "user@example.com" - locate: "email input field" - - sleep: 2000 - - - name: test password with symbols - flow: - - aiInput: "SecurePass123!@#" - locate: "password field" - - sleep: 2000 - - - name: test clear and replace - flow: - - aiInput: "" # Clear the field - locate: "text input" - - sleep: 1000 - - aiInput: "Replaced text content" - locate: "same text input" - - sleep: 2000 - - - name: test long text - flow: - - aiInput: "This is a much longer text message that tests the iOS keyboard input with multiple words and proper spacing between characters" - locate: "text area or message field" - - sleep: 3000 diff --git a/examples/ios-yaml-example.yaml b/examples/ios-yaml-example.yaml deleted file mode 100644 index b20af69af..000000000 --- a/examples/ios-yaml-example.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# iOS automation with YAML script example -# This example shows how to automate iOS devices using PyAutoGUI server - -ios: - # PyAutoGUI server configuration - serverUrl: "http://localhost:1412" - - # Auto dismiss keyboard after input (optional) - autoDismissKeyboard: true - - # iOS device mirroring configuration for precise location targeting - # These values define the position and size of the mirrored device screen - mirrorConfig: - mirrorX: 692 # X position of iOS mirror on computer screen - mirrorY: 161 # Y position of iOS mirror on computer screen - mirrorWidth: 344 # Width of the mirrored iOS screen - mirrorHeight: 764 # Height of the mirrored iOS screen (iPhone 11 Pro size) - - # Output file for aiQuery/aiAssert results (optional) - output: "./results.json" - -tasks: - - name: Open Safari and search - flow: - - aiAction: "Open Safari browser" - - sleep: 2000 - - aiAction: "Navigate to google.com" - - sleep: 3000 - - aiAction: "Type 'iOS automation' in the search box" - - aiAction: "Tap the search button" - - sleep: 5000 - - aiAssert: "Search results are displayed" - - - name: Extract search results - flow: - - aiQuery: > - {title: string, description: string}[], - return the titles and descriptions of the first 3 search results - name: search_results - - - name: Navigate to settings - flow: - - aiAction: "Open Settings app" - - sleep: 2000 - - aiAction: "Scroll down to find General settings" - - aiAction: "Tap on General" - - sleep: 1000 - - aiAssert: "General settings page is displayed" - - - name: Test iOS input functionality - flow: - - aiAction: "Open Notes app" - - sleep: 2000 - - aiAction: "Tap on new note button or create new note" - - sleep: 1000 - - aiInput: "This is a test note created with iOS automation. The input should work properly with character-by-character typing and keyboard dismissal." - - sleep: 1000 - - aiAction: "Save the note" - - aiAssert: "Note is saved successfully" - - - name: Test iOS search input - flow: - - aiAction: "Go to iOS home screen" - - sleep: 1000 - - aiAction: "Swipe down to open search" - - sleep: 1000 - - aiInput: "Settings" - - sleep: 2000 - - aiAction: "Tap on Settings in search results" - - aiAssert: "Settings app is opened" diff --git a/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md b/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md deleted file mode 100644 index ce2b94d61..000000000 --- a/packages/ios/docs/iOS_INPUT_IMPROVEMENTS.md +++ /dev/null @@ -1,131 +0,0 @@ -# iOS Input Implementation Improvements - -## 问题描述 -iOS的aiInput之前存在问题，因为Android可以调用adb直接输入，但iOS只能通过模拟键盘输入来实现。这导致了输入不稳定、字符丢失等问题。 - -## 解决方案 - -### 1. 优化的输入流程 -新的iOS输入实现包含以下步骤： - -1. **聚焦输入框**: 先点击输入框来获得焦点 -2. **清空现有内容**: 使用Cmd+A选择全部，然后删除 -3. **字符间隔输入**: 使用适当的间隔时间逐字符输入 -4. **自动关闭键盘**: 输入完成后自动关闭键盘 - -### 2. Python服务器改进 -- 添加了`ios_input`动作类型，专门处理iOS输入 -- 支持`interval`参数，控制字符间输入间隔 -- 为iOS设置默认的最小间隔时间(20ms)以确保字符正确输入 - -### 3. TypeScript实现改进 -- 添加了`aiInputIOS`方法，提供专门的iOS输入处理 -- 更新了`clearInput`方法，先点击聚焦再清空 -- 添加了`dismissKeyboard`方法，自动关闭iOS键盘 -- 在任务执行器中添加了iOS特定的输入逻辑 - -## 新增功能 - -### PyAutoGUI服务器新动作 -```python -{ - "action": "ios_input", - "x": 100, # 输入框的x坐标（可选） - "y": 200, # 输入框的y坐标（可选） - "text": "Hello", # 要输入的文本 - "interval": 0.05, # 字符间间隔（秒） - "clear_first": true # 是否先清空现有内容 -} -``` - -### iOS设备新方法 -```typescript -// 专门的iOS输入方法 -await iosDevice.aiInputIOS(text, element, options); - -// 改进的键盘关闭方法 -await iosDevice.dismissKeyboard(); - -// 改进的清空输入方法 -await iosDevice.clearInput(element); -``` - -## 使用示例 - -### YAML配置示例 -```yaml -ios: - serverUrl: "http://localhost:1412" - autoDismissKeyboard: true - mirrorConfig: - mirrorX: 692 - mirrorY: 161 - mirrorWidth: 344 - mirrorHeight: 764 - -tasks: - - name: Test iOS input - flow: - - aiAction: "Open Notes app" - - aiInput: "This text will be input properly on iOS" - - aiAssert: "Text is entered correctly" -``` - -### 编程接口示例 -```typescript -const agent = await agentFromPyAutoGUI({ - serverPort: 1412, - autoDismissKeyboard: true, - mirrorConfig: { - mirrorX: 692, - mirrorY: 161, - mirrorWidth: 344, - mirrorHeight: 764, - }, -}); - -// 使用aiInput，现在对iOS优化 -await agent.aiInput('Hello iOS!', 'in the text input field'); -``` - -## 技术改进细节 - -### 1. 字符间隔控制 -- 默认间隔: 20ms (Android的adb输入不需要间隔) -- 可配置间隔: 通过`interval`参数自定义 -- 自动调整: 为iOS设备自动设置合适的默认值 - -### 2. 聚焦处理 -- 自动点击: 在输入前先点击输入框获得焦点 -- 等待时间: 点击后等待300ms确保焦点获得 -- 坐标转换: 自动处理iOS到macOS的坐标转换 - -### 3. 键盘管理 -- 自动关闭: 输入完成后自动关闭iOS键盘 -- 多种方法: 尝试Return键或点击键盘外区域 -- 可配置: 通过`autoDismissKeyboard`选项控制 - -### 4. 错误处理 -- 优雅降级: 如果特殊方法失败，回退到基本方法 -- 日志记录: 详细的调试日志帮助排查问题 -- 类型安全: 通过TypeScript类型检查确保正确使用 - -## 测试 -运行测试脚本验证功能： -```bash -cd packages/ios -npm run test:input -``` - -## 兼容性 -- ✅ iOS模拟器 -- ✅ iOS真机（通过屏幕镜像） -- ✅ 向后兼容现有API -- ✅ 支持中文输入 -- ✅ 支持特殊字符 - -## 性能优化 -- 减少不必要的延迟 -- 优化字符输入速度 -- 智能键盘关闭策略 -- 高效的坐标转换 diff --git a/packages/ios/docs/ios-mirroring-guide.md b/packages/ios/docs/ios-mirroring-guide.md deleted file mode 100644 index c31ee144e..000000000 --- a/packages/ios/docs/ios-mirroring-guide.md +++ /dev/null @@ -1,312 +0,0 @@ -# iOS Device Mirroring with Coordinate Mapping - -This document explains how to use Midscene.js with iOS device mirroring through macOS, including automatic coordinate transformation for accurate touch events. - -## Overview - -iOS device mirroring allows you to control iOS devices through their screen representation on macOS. This is useful for: - -- **App Testing**: Automated testing of iOS apps without physical interaction -- **Screen Recording**: Capture iOS interactions for documentation -- **Remote Control**: Control iOS devices through macOS automation -- **CI/CD Integration**: Automated iOS testing in continuous integration - -## Prerequisites - -1. **Python Dependencies**: - ```bash - pip3 install flask pyautogui - ``` - -2. **iOS Device Mirroring Setup**: - - **iPhone Mirroring** (macOS Sequoia): Built-in iOS mirroring feature - -3. **Screen Position**: Note the exact position and size of iOS mirror on your macOS screen - -## Configuration - -### 1. Basic Setup - -```typescript -import { iOSDevice, iOSAgent } from '@midscene/ios'; - -const device = new iOSDevice({ - serverPort: 1412, - mirrorConfig: { - mirrorX: 100, // Mirror position X on macOS screen - mirrorY: 50, // Mirror position Y on macOS screen - mirrorWidth: 400, // Mirror width on macOS screen - mirrorHeight: 800 // Mirror height on macOS screen - } -}); - -await device.connect(); -const agent = new iOSAgent(device); -``` - -### 2. Finding Mirror Coordinates - -**Method 1: Manual Measurement** -1. Position iOS mirror window on your screen -2. Use macOS's built-in screenshot tool to measure: - - Press `Cmd + Shift + 4` - - Drag from top-left to bottom-right of iOS mirror - - Note the coordinates and dimensions - -**Method 2: Using Digital Color Meter** -1. Open Digital Color Meter (Applications → Utilities) -2. Move cursor to top-left corner of iOS mirror → note coordinates -3. Move cursor to bottom-right corner → calculate width/height - -**Method 3: Programmatic Detection** (Advanced) -```python -# Use this Python script to help find iOS mirror region -import pyautogui -import cv2 -import numpy as np - -def find_ios_mirror(): - # Take screenshot - screenshot = pyautogui.screenshot() - - # Convert to OpenCV format - img = cv2.cvtColor(np.array(screenshot), cv2.COLOR_RGB2BGR) - - # Look for iOS-specific UI patterns (status bar, home indicator, etc.) - # This is a simplified example - real implementation would be more complex - - # Return detected region - return {"x": 100, "y": 50, "width": 400, "height": 800} -``` - -## Coordinate Transformation - -The system automatically transforms iOS coordinates to macOS screen coordinates: - -### Transformation Formula -``` -macOS_x = mirror_x + (iOS_x × scale_x) -macOS_y = mirror_y + (iOS_y × scale_y) - -where: -scale_x = mirror_width / ios_width -scale_y = mirror_height / ios_height -``` - -### Example -``` -iOS Device: 393×852 (iPhone 15 Pro) -Mirror Region: (100, 50) with size 400×800 - -iOS coordinate (100, 200) transforms to: -macOS_x = 100 + (100 × 400/393) = 100 + 101.8 = ~202 -macOS_y = 50 + (200 × 800/852) = 50 + 187.8 = ~238 -``` - -## Usage Examples - -### Basic Touch Operations - -```typescript -// Tap at iOS coordinates - automatically transformed -await device.tap({ left: 100, top: 200 }); - -// Drag gesture -await device.drag( - { left: 100, top: 300 }, // Start point - { left: 300, top: 300 } // End point -); - -// Scroll -await device.scroll({ - direction: 'down', - startPoint: { left: 200, top: 400 }, - distance: 200 -}); -``` - -### AI-Powered Automation - -```typescript -// AI can understand iOS interface elements -await agent.aiTap('Settings app icon'); -await agent.aiInput('Wi-Fi', 'Search bar'); -await agent.aiScroll({ direction: 'down', scrollType: 'once' }); - -// Extract data from iOS interface -const appList = await agent.aiQuery('string[], visible app names on home screen'); - -// Verify iOS interface state -await agent.aiAssert('Control Center is open'); -``` - -### Screenshots - -```typescript -// Takes screenshot of iOS mirror region only -const screenshot = await device.screenshotBase64(); - -// Screenshot is automatically cropped to iOS mirror area -// Perfect for AI analysis of iOS interface -``` - -## Common Device Information (For Reference) - -Note: These logical resolutions are automatically detected by the system. You only need to configure the mirror position and size on your macOS screen. - -### iPhone Models -```typescript -// iPhone 15 Pro / 14 Pro: 393 x 852 logical pixels -// iPhone 15 Plus / 14 Plus: 428 x 926 logical pixels -// iPhone SE (3rd generation): 375 x 667 logical pixels -// iPhone 15 / 14: 393 x 852 logical pixels -``` - -### iPad Models -```typescript -// iPad Pro 12.9" (6th generation): 1024 x 1366 logical pixels -// iPad Pro 11" (4th generation): 834 x 1194 logical pixels -// iPad Air (5th generation): 820 x 1180 logical pixels -``` - -## Best Practices - -### 1. Calibration Testing -```typescript -// Always test coordinate accuracy first -const testPoints = [ - { left: 50, top: 50 }, // Top-left corner - { left: 196, top: 426 }, // Center (for iPhone 15 Pro) - { left: 343, top: 802 } // Bottom-right corner -]; - -for (const point of testPoints) { - await device.tap(point); - await new Promise(resolve => setTimeout(resolve, 1000)); -} -``` - -### 2. Handle Different Mirror Sizes -```typescript -// Support multiple mirror configurations -const configs = { - small: { mirrorWidth: 300, mirrorHeight: 600 }, - medium: { mirrorWidth: 400, mirrorHeight: 800 }, - large: { mirrorWidth: 500, mirrorHeight: 1000 } -}; - -// Choose based on screen size or user preference -const config = configs.medium; -``` - -### 3. Error Handling -```typescript -try { - await device.connect(); -} catch (error) { - if (error.message.includes('Python server')) { - console.error('Start Python server: python3 auto_server.py 1412'); - } else if (error.message.includes('configuration')) { - console.error('Check mirror coordinates and iOS device size'); - } - throw error; -} -``` - -### 4. Performance Optimization -```typescript -// Batch operations for better performance -const actions = [ - { action: 'click', x: 100, y: 200 }, - { action: 'sleep', seconds: 0.5 }, - { action: 'click', x: 200, y: 300 } -]; - -// All actions use coordinate transformation -await device.executeBatchActions(actions); -``` - -## Troubleshooting - -### Common Issues - -**1. Coordinate Misalignment** -- **Problem**: Taps don't hit intended targets -- **Solution**: Re-measure mirror position and size, ensure iOS device orientation is correct - -**2. Python Server Connection Failed** -- **Problem**: `Failed to connect to Python server` -- **Solution**: Start server with `python3 auto_server.py 1412`, check firewall settings - -**3. Screenshots Show Wrong Region** -- **Problem**: Screenshots include macOS desktop instead of iOS mirror -- **Solution**: Verify mirror coordinates, ensure iOS window is not minimized - -**4. Scale Factor Issues** -- **Problem**: Coordinates are consistently off by same ratio -- **Solution**: Double-check iOS device logical resolution vs mirror size - -### Debug Tools - -```typescript -// Check current configuration -const config = await device.getConfiguration(); -console.log('Current mapping:', config); - -// Test coordinate transformation -const testCoord = { left: 100, top: 200 }; -console.log('iOS:', testCoord); -// Tap will show transformed macOS coordinates in logs -await device.tap(testCoord); -``` - -## Advanced Features - -### Dynamic Reconfiguration -```typescript -// Change mirror configuration at runtime -await device.configureIOSMirror({ - mirrorX: 200, // New position - mirrorY: 100, - mirrorWidth: 450, // New size for different device - mirrorHeight: 950 -}); -``` - -### Multiple Device Support -```typescript -// Control multiple iOS devices -const device1 = new iOSDevice({ - serverPort: 1412, - mirrorConfig: config1 -}); - -const device2 = new iOSDevice({ - serverPort: 1413, // Different server instance - mirrorConfig: config2 -}); -``` - -### Integration with Test Frameworks -```typescript -// Jest/Vitest example -describe('iOS App Tests', () => { - let device, agent; - - beforeAll(async () => { - device = new iOSDevice({ /* config */ }); - await device.connect(); - agent = new iOSAgent(device); - }); - - test('should login successfully', async () => { - await agent.aiTap('Login button'); - await agent.aiInput('user@example.com', 'Email field'); - await agent.aiInput('password123', 'Password field'); - await agent.aiTap('Sign in button'); - await agent.aiAssert('Dashboard is visible'); - }); -}); -``` - -This coordinate mapping system makes iOS device automation through macOS screen mirroring seamless and accurate, enabling powerful AI-driven testing and automation workflows. diff --git a/examples/ios-input-example.yaml b/packages/ios/examples/ios-input-example.yaml similarity index 93% rename from examples/ios-input-example.yaml rename to packages/ios/examples/ios-input-example.yaml index 393e8da89..044436dad 100644 --- a/examples/ios-input-example.yaml +++ b/packages/ios/examples/ios-input-example.yaml @@ -28,11 +28,11 @@ tasks: - aiTap: "搜索图标" - sleep: 3000 - aiInput: "Coldplay" - locate: "Search input field" + locate: "底部Search input field" - sleep: 2000 - aiKeyboardPress: "Enter" - sleep: 3000 - aiWaitFor: "Search results are displayed" - - aiAction: "随机播放一首歌曲" + - aiAction: "播放第一首歌曲" - sleep: 3000 - aiAction: "返回Home" \ No newline at end of file diff --git a/packages/ios/getAppWindowRect.scpt b/packages/ios/getAppWindowRect.scpt deleted file mode 100644 index 88b8718218cc0243408c17b12a3c60219ce249d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2224 zcmb_d*;5op9R6k&5oc%?5RZ7^5WKPiyQ_F1s3@dNN(-wfUlOZQ<1)wwc4up6cT@P} zbDr{#ZFg^3DEW}#9MA|sT6joGNe9K~Pc3raa6f5MWF7Jm!d zg3-Vq@J8Hhab!f~Fm%mk4&)Uq7#VF9hJ1*~2U?epKsAl+M3*z_7o8!uq5;HCWR zK;RU7C*ux#PI~W3rVLr?Us0?p)3p^z+`qykE=?G&SicYA^@JPq6D?;}9H- z!bg$+A-E{PW2>DtU@JvQ76WWMD6s7y!#Ts@oHX`QL(8?s9;A`y=d-3cc93J;$nkS9 z7>;@z)AC|yu?bBkDj7{Rlp-)=IzN#2@=o3w@?Nj~orbrdM6uS$8^zeenU&azsJxa} z3Q?GSRMaE2s8fV+Q!I3$-b6hW=b}Lo%si)ASd4j?uL$M_L-#+y0xVPnGaJflIkHF* zj3oOtsFjzNyhMeS%eonbMKIgBxfa9u1B)#JB`G(jXgJ(7E=XQP}S(u~ir94izdTv#(! z$?sTc5h|?Icj>Ic%5roouu>jJ^w8}$E9_nQH0o}EfZO5=i9@z3=%CI=N@bJ?*v-98y?Yww*vy?%Hy9y4%E*+dH=0!JGtU z1?fZHH`;L<>pHouXy4e)UE07rw<}&Dm#j-7Ux;lFw( ziM8L$m(nSZ(IOG=C_C=hoj#Cv^P?TpCrrqu1r;Z|EN43V$-n{m)g!w)=nXkI%YMMK z<%S#@CGAv`V_h<`W>?L{0om8VF7%M&TFZXp6Mo~<{l@<#8FOL8`P^uFeXZe@J|k~5 z+mVE$wWO?wTqA#MHTWZ=c5%*CORn94X=T_`6v4hTMVW-JAPLusbQ!eV*uISh<>nOHs=%(m`oJXgelV8yz zXXT8XmQ!+4PRMaN7Lf}))s)ZtyjpT@;^14fz$-vruVET7*OIfkPfE7s%nX;nlGB`{ ziP4r*QzT)@Nv%o(v*pCpCAH+Z9wx!qa%}p&tdmRFcH`S>7=5?uu|76793OAF{q>!e z{a<;TlD!xDVq-_*^;jIcHyrQ1)OYkjOYfDwqmNp8ul61J<#?tP-M2652B|!sqUv|o HuKoBIYt#&a From 694f85acdf923adb8e747de43c6d546d19e5e6f0 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Tue, 5 Aug 2025 11:28:26 +0800 Subject: [PATCH 10/17] refactor(ios): remove unused ios-input-test.ts The test file 'packages/ios/tests/ios-input-test.ts' is no longer needed and has been removed. --- packages/ios/tests/ios-input-test.ts | 56 ---------------------------- 1 file changed, 56 deletions(-) delete mode 100644 packages/ios/tests/ios-input-test.ts diff --git a/packages/ios/tests/ios-input-test.ts b/packages/ios/tests/ios-input-test.ts deleted file mode 100644 index 8cfdae437..000000000 --- a/packages/ios/tests/ios-input-test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { agentFromPyAutoGUI } from '../src/agent/index'; - -async function testIOSInput() { - console.log('🧪 Testing iOS input functionality...'); - - try { - // Create iOS agent with mirror configuration - const agent = await agentFromPyAutoGUI({ - serverPort: 1412, - autoDismissKeyboard: true, - mirrorConfig: { - mirrorX: 692, - mirrorY: 161, - mirrorWidth: 344, - mirrorHeight: 764, - }, - }); - - console.log('✅ iOS agent created successfully'); - - // Test basic input functionality - console.log('🔤 Testing basic input...'); - - // Simulate clicking on a text input field (coordinates would be from AI detection) - await agent.page.tap({ left: 172, top: 300 }); // Example coordinates - console.log('📱 Tapped on input field'); - - // Test the new iOS input method - await agent.page.aiInputIOS('Hello iOS Testing!', { center: [172, 300] }); - console.log('✅ iOS input completed'); - - // Test keyboard press - await agent.page.keyboardPress('return'); - console.log('✅ Return key pressed'); - - // Test clearing input - await agent.page.clearInput({ center: [172, 300] }); - console.log('✅ Input cleared'); - - // Test regular input method - await agent.page.input('Regular input test'); - console.log('✅ Regular input completed'); - - console.log('🎉 All iOS input tests passed!'); - } catch (error) { - console.error('❌ iOS input test failed:', error); - process.exit(1); - } -} - -// Run the test -if (require.main === module) { - testIOSInput(); -} - -export { testIOSInput }; From 07eb8f49649ba4fc56a7a32ff6781dd1a5706c8b Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Fri, 8 Aug 2025 23:47:04 +0800 Subject: [PATCH 11/17] feat(ios): clear input before typing and remove unused scripts --- packages/ios/idb/auto_server.py | 2 ++ packages/ios/package.json | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ios/idb/auto_server.py b/packages/ios/idb/auto_server.py index 9268e23a8..8b99a3b7d 100644 --- a/packages/ios/idb/auto_server.py +++ b/packages/ios/idb/auto_server.py @@ -145,6 +145,8 @@ def handle_action(action): return {"status": "ok", "action": "drag", "ios_from": [x, y], "ios_to": [x2, y2], "mac_from": [mac_x, mac_y], "mac_to": [mac_x2, mac_y2]} elif act == "type": + # select all + pyautogui.hotkey('command', 'a') text = action["text"] interval = float(action.get("interval", 0.0)) # For iOS, we need slower typing to ensure proper character registration diff --git a/packages/ios/package.json b/packages/ios/package.json index a56a44fa7..6fdcc0474 100644 --- a/packages/ios/package.json +++ b/packages/ios/package.json @@ -25,8 +25,6 @@ "dev": "modern dev", "test": "vitest", "server": "node bin/server.js", - "server:test": "python3 idb/test.py", - "example": "tsx tests/example.ts", "setup": "./setup.sh", "prepack": "npm run build" }, From ecc739ddfa6d6b47b03f2861cc0f7e43c5a22539 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Mon, 11 Aug 2025 22:30:59 +0800 Subject: [PATCH 12/17] feat(ios): introduce interactive playground with auto-detection --- apps/ios-playground/README.md | 66 +++++ apps/ios-playground/package.json | 35 +++ apps/ios-playground/rsbuild.config.ts | 88 ++++++ apps/ios-playground/src/App.less | 183 ++++++++++++ apps/ios-playground/src/App.tsx | 277 ++++++++++++++++++ apps/ios-playground/src/env.d.ts | 10 + apps/ios-playground/src/favicon.ico | Bin 0 -> 61841 bytes apps/ios-playground/src/icons/linked.svg | 3 + apps/ios-playground/src/icons/screenshot.svg | 11 + apps/ios-playground/src/icons/unlink.svg | 3 + apps/ios-playground/src/index.tsx | 8 + apps/ios-playground/src/ios-device/index.less | 44 +++ apps/ios-playground/src/ios-device/index.tsx | 129 ++++++++ apps/ios-playground/src/ios-player/index.less | 60 ++++ apps/ios-playground/src/ios-player/index.tsx | 171 +++++++++++ .../src/scripts/blank_polyfill.ts | 1 + apps/ios-playground/tsconfig.json | 23 ++ packages/cli/src/index.ts | 14 +- packages/ios-playground/README.md | 147 ++++++++++ packages/ios-playground/bin/ios-playground | 34 +++ packages/ios-playground/bin/server.js | 162 ++++++++++ packages/ios-playground/modern.config.ts | 13 + packages/ios-playground/package.json | 34 +++ packages/ios-playground/test-health.js | 28 ++ packages/ios-playground/tsconfig.json | 11 + packages/ios/bin/server.js | 12 +- packages/ios/examples/ios-input-example.yaml | 7 +- packages/ios/examples/ios-mirroring-demo.js | 51 +--- packages/ios/idb/auto_server.py | 223 +++++++++++++- packages/ios/setup.sh | 3 +- packages/ios/src/agent/index.ts | 21 +- packages/ios/src/page/index.ts | 275 +++++++++++++++-- packages/ios/src/utils/index.ts | 15 +- pnpm-lock.yaml | 214 ++++++++++---- 34 files changed, 2222 insertions(+), 154 deletions(-) create mode 100644 apps/ios-playground/README.md create mode 100644 apps/ios-playground/package.json create mode 100644 apps/ios-playground/rsbuild.config.ts create mode 100644 apps/ios-playground/src/App.less create mode 100644 apps/ios-playground/src/App.tsx create mode 100644 apps/ios-playground/src/env.d.ts create mode 100644 apps/ios-playground/src/favicon.ico create mode 100644 apps/ios-playground/src/icons/linked.svg create mode 100644 apps/ios-playground/src/icons/screenshot.svg create mode 100644 apps/ios-playground/src/icons/unlink.svg create mode 100644 apps/ios-playground/src/index.tsx create mode 100644 apps/ios-playground/src/ios-device/index.less create mode 100644 apps/ios-playground/src/ios-device/index.tsx create mode 100644 apps/ios-playground/src/ios-player/index.less create mode 100644 apps/ios-playground/src/ios-player/index.tsx create mode 100644 apps/ios-playground/src/scripts/blank_polyfill.ts create mode 100644 apps/ios-playground/tsconfig.json create mode 100644 packages/ios-playground/README.md create mode 100755 packages/ios-playground/bin/ios-playground create mode 100644 packages/ios-playground/bin/server.js create mode 100644 packages/ios-playground/modern.config.ts create mode 100644 packages/ios-playground/package.json create mode 100644 packages/ios-playground/test-health.js create mode 100644 packages/ios-playground/tsconfig.json diff --git a/apps/ios-playground/README.md b/apps/ios-playground/README.md new file mode 100644 index 000000000..da0aed396 --- /dev/null +++ b/apps/ios-playground/README.md @@ -0,0 +1,66 @@ +# Midscene iOS Playground + +A playground for testing Midscene iOS automation features with automatic device mirroring setup. + +See https://midscenejs.com/ for details. + +## Features + +### ✨ Auto-Detection of iPhone Mirroring + +The playground can automatically detect and configure the iPhone Mirroring app window: + +1. **Automatic Setup**: When you connect, the playground automatically tries to detect your iPhone Mirroring window +2. **Smart Configuration**: It calculates the optimal screen mapping based on window size and device type +3. **Manual Override**: If auto-detection doesn't work, you can manually configure the mirror settings + +### 🎯 How Auto-Detection Works + +1. **Window Detection**: Uses AppleScript to find the iPhone Mirroring app window +2. **Content Area Calculation**: Automatically calculates the device screen area within the window (excluding title bars and padding) +3. **Device Matching**: Matches the aspect ratio to common iOS devices for optimal coordinate mapping +4. **Instant Configuration**: Sets up the coordinate transformation automatically + +## Usage + +### Prerequisites + +1. **macOS with iPhone Mirroring**: Ensure iPhone Mirroring is available and working +2. **iOS Device**: Connected and mirroring to your Mac +3. **Python Server**: The PyAutoGUI server running on port 1412 + +### Quick Start + +1. **Start the server**: + ```bash + cd packages/ios/idb + python auto_server.py + ``` + +2. **Launch the playground**: + ```bash + npm run dev + ``` + +3. **Open iPhone Mirroring app** on your Mac + +4. **Auto-configure**: Click "Auto Detect" to automatically set up the mirroring coordinates + +### UI Controls + +- **📷 Screenshot**: Take a screenshot of the configured iOS device area +- **🔍 Auto Detect**: Automatically detect and configure iPhone Mirroring window +- **⚙️ Manual Config**: Manually set mirror window coordinates + +## Troubleshooting + +### Auto-Detection Issues + +1. **"iPhone Mirroring app not found"**: Make sure iPhone Mirroring app is open and visible +2. **"Window seems too small"**: Try resizing the iPhone Mirroring window to be larger +3. **Coordinates seem wrong**: Use manual configuration to fine-tune the coordinates + +### Server Connection Issues + +1. **Server not responding**: Check if server is running on port 1412 +2. **Permission issues**: Ensure macOS accessibility permissions are granted to Terminal/Python diff --git a/apps/ios-playground/package.json b/apps/ios-playground/package.json new file mode 100644 index 000000000..50c8ab069 --- /dev/null +++ b/apps/ios-playground/package.json @@ -0,0 +1,35 @@ +{ + "name": "ios-playground", + "private": true, + "version": "0.12.4", + "type": "module", + "scripts": { + "build": "rsbuild build", + "dev": "rsbuild dev --open", + "preview": "rsbuild preview" + }, + "dependencies": { + "@ant-design/icons": "^5.3.1", + "@midscene/ios": "workspace:*", + "@midscene/core": "workspace:*", + "@midscene/shared": "workspace:*", + "@midscene/visualizer": "workspace:*", + "@midscene/web": "workspace:*", + "antd": "^5.21.6", + "dayjs": "^1.11.11", + "react": "18.3.1", + "react-dom": "18.3.1" + }, + "devDependencies": { + "@rsbuild/core": "^1.3.22", + "@rsbuild/plugin-less": "^1.2.4", + "@rsbuild/plugin-node-polyfill": "1.3.0", + "@rsbuild/plugin-react": "^1.3.1", + "@rsbuild/plugin-svgr": "^1.1.1", + "@types/react": "^18.3.1", + "@types/react-dom": "^18.3.1", + "archiver": "^6.0.0", + "less": "^4.2.0", + "typescript": "^5.8.3" + } +} diff --git a/apps/ios-playground/rsbuild.config.ts b/apps/ios-playground/rsbuild.config.ts new file mode 100644 index 000000000..1449447f1 --- /dev/null +++ b/apps/ios-playground/rsbuild.config.ts @@ -0,0 +1,88 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { defineConfig } from '@rsbuild/core'; +import { pluginLess } from '@rsbuild/plugin-less'; +import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill'; +import { pluginReact } from '@rsbuild/plugin-react'; +import { pluginSvgr } from '@rsbuild/plugin-svgr'; +import { pluginTypeCheck } from '@rsbuild/plugin-type-check'; + +const copyAndroidPlaygroundStatic = () => ({ + name: 'copy-android-playground-static', + setup(api) { + api.onAfterBuild(async () => { + const srcDir = path.join(__dirname, 'dist'); + const destDir = path.join( + __dirname, + '..', + '..', + 'packages', + 'android-playground', + 'static', + ); + const faviconSrc = path.join(__dirname, 'src', 'favicon.ico'); + const faviconDest = path.join(destDir, 'favicon.ico'); + + await fs.promises.mkdir(destDir, { recursive: true }); + // Copy directory contents recursively + await fs.promises.cp(srcDir, destDir, { recursive: true }); + // Copy favicon + await fs.promises.copyFile(faviconSrc, faviconDest); + + console.log(`Copied build artifacts to ${destDir}`); + console.log(`Copied favicon to ${faviconDest}`); + }); + }, +}); + +export default defineConfig({ + environments: { + web: { + source: { + entry: { + index: './src/index.tsx', + }, + }, + output: { + target: 'web', + sourceMap: true, + }, + html: { + title: 'Midscene iOS Playground', + }, + }, + }, + dev: { + writeToDisk: true, + }, + server: { + proxy: { + '/api/pyautogui': { + target: 'http://localhost:1412', + changeOrigin: true, + pathRewrite: { + '^/api/pyautogui': '' + } + } + } + }, + resolve: { + alias: { + async_hooks: path.join(__dirname, './src/scripts/blank_polyfill.ts'), + 'node:async_hooks': path.join( + __dirname, + './src/scripts/blank_polyfill.ts', + ), + react: path.resolve(__dirname, 'node_modules/react'), + 'react-dom': path.resolve(__dirname, 'node_modules/react-dom'), + }, + }, + plugins: [ + pluginReact(), + pluginNodePolyfill(), + pluginLess(), + pluginSvgr(), + copyAndroidPlaygroundStatic(), + pluginTypeCheck(), + ], +}); diff --git a/apps/ios-playground/src/App.less b/apps/ios-playground/src/App.less new file mode 100644 index 000000000..f01312f66 --- /dev/null +++ b/apps/ios-playground/src/App.less @@ -0,0 +1,183 @@ +body { + margin: 0; + padding: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', + Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji'; + font-size: 14px; +} + +.app-container { + width: 100%; + height: 100vh; + display: flex; + flex-direction: column; + background-color: #f5f5f5; +} + +.app-content { + height: 100vh; + overflow: hidden; +} + +.app-grid-layout { + height: 100%; + display: flex; + + .ant-row { + flex: 1; + width: 100%; + height: 100%; + display: flex; + flex-wrap: nowrap; + width: 100%; + } +} + +.app-panel { + height: 100%; + background-color: #fff; + border-radius: 0; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); + transition: box-shadow 0.3s; + overflow: hidden; + + &:hover { + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.09); + } + + &.left-panel { + width: 480px; + flex: none; + overflow: hidden; + height: 100%; + display: flex; + flex-direction: column; + } + + &.right-panel { + border-radius: 0; + flex: 1; + overflow: hidden; + box-shadow: -4px 0px 20px 0px #0000000A; + } +} + +.panel-content { + padding: 12px 24px 24px 24px; + height: 100%; + overflow: auto; + display: flex; + flex-direction: column; + border-left: 1px solid rgba(0, 0, 0, 0.08); + + &.left-panel-content { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; + } + + &.right-panel-content { + border-radius: 0; + } + + h2 { + color: #000; + font-size: 18px; + margin-top: 16px; + margin-bottom: 12px; + } + + canvas { + max-width: 100%; + margin-top: 16px; + border: 1px solid #f0f0f0; + border-radius: 4px; + } +} + +.command-form { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; + + .form-content { + display: flex; + flex-direction: column; + height: 100%; + gap: 24px; + } + + .command-input-wrapper { + margin-top: 8px; + } +} + +.result-container { + flex: 1; + overflow: hidden; + display: flex; + flex-direction: column; + min-height: 0; + position: relative; + height: 100%; +} + +@media (max-width: 768px) { + .app-container { + height: auto; + min-height: 100vh; + } + + .app-grid-layout .ant-row { + flex-wrap: wrap !important; + } + + .app-panel { + margin-bottom: 16px; + height: auto; + min-height: 200px; + width: 100% !important; + flex: 0 0 100% !important; + + &:first-child { + border-radius: 20px; + + .panel-content { + border-radius: 20px; + } + } + } + + .panel-content { + padding: 12px; + + h2 { + font-size: 16px; + margin-bottom: 12px; + padding-bottom: 6px; + } + + textarea { + min-height: 100px; + } + } +} + +@media (min-width: 769px) and (max-width: 992px) { + .app-panel { + margin-bottom: 16px; + min-height: 300px; + } +} + +.resize-handle { + width: 2px; + background-color: #f0f0f0; + transition: background-color 0.2s; + + &:hover { + background-color: #1677ff; + } +} \ No newline at end of file diff --git a/apps/ios-playground/src/App.tsx b/apps/ios-playground/src/App.tsx new file mode 100644 index 000000000..5a3f31f01 --- /dev/null +++ b/apps/ios-playground/src/App.tsx @@ -0,0 +1,277 @@ +import './App.less'; +import { overrideAIConfig } from '@midscene/shared/env'; +import { + EnvConfig, + Logo, + type PlaygroundResult, + PlaygroundResultView, + PromptInput, + type ReplayScriptsInfo, + allScriptsFromDump, + cancelTask, + getTaskProgress, + globalThemeConfig, + overrideServerConfig, + requestPlaygroundServer, + useEnvConfig, + useServerValid, +} from '@midscene/visualizer'; +import { Col, ConfigProvider, Form, Layout, Row, message } from 'antd'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import IOSPlayer, { type IOSPlayerRefMethods } from './ios-player'; + +import '@midscene/visualizer/index.css'; +import './ios-device/index.less'; + +const { Content } = Layout; + +export default function App() { + const [form] = Form.useForm(); + const selectedType = Form.useWatch('type', form); + const [loading, setLoading] = useState(false); + const [messageApi, contextHolder] = message.useMessage(); + const [result, setResult] = useState({ + result: undefined, + dump: null, + reportHTML: null, + error: null, + }); + const [replayCounter, setReplayCounter] = useState(0); + const [replayScriptsInfo, setReplayScriptsInfo] = + useState(null); + const { config, deepThink } = useEnvConfig(); + const [loadingProgressText, setLoadingProgressText] = useState(''); + const currentRequestIdRef = useRef(null); + const pollIntervalRef = useRef | null>(null); + const configAlreadySet = Object.keys(config || {}).length >= 1; + const serverValid = useServerValid(true); + + // iOS Player ref + const iosPlayerRef = useRef(null); + + // clear the polling interval + const clearPollingInterval = useCallback(() => { + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + pollIntervalRef.current = null; + } + }, []); + + // start polling task progress + const startPollingProgress = useCallback( + (requestId: string) => { + clearPollingInterval(); + + // set polling interval to 500ms + pollIntervalRef.current = setInterval(async () => { + try { + const data = await getTaskProgress(requestId); + + if (data.tip) { + setLoadingProgressText(data.tip); + } + } catch (error) { + console.error('Failed to poll task progress:', error); + } + }, 500); + }, + [clearPollingInterval], + ); + + // clean up the polling when the component unmounts + useEffect(() => { + return () => { + clearPollingInterval(); + }; + }, [clearPollingInterval]); + + // Override AI configuration + useEffect(() => { + overrideAIConfig(config); + overrideServerConfig(config); + }, [config]); + + // handle run button click + const handleRun = useCallback(async () => { + if (!serverValid) { + messageApi.warning( + 'Playground server is not ready, please try again later', + ); + return; + } + + setLoading(true); + setResult(null); + setReplayScriptsInfo(null); + setLoadingProgressText(''); + + const { type, prompt } = form.getFieldsValue(); + + const thisRunningId = Date.now().toString(); + + currentRequestIdRef.current = thisRunningId; + + // start polling progress immediately + startPollingProgress(thisRunningId); + + try { + // Use a fixed context string for iOS since we don't have device selection + const res = await requestPlaygroundServer( + 'ios-device', + type, + prompt, + { + requestId: thisRunningId, + deepThink, + }, + ); + + // stop polling + clearPollingInterval(); + + setResult(res); + setLoading(false); + + if (!res) { + throw new Error('server returned empty response'); + } + + // handle the special case of aiAction type, extract script information + if (res?.dump && !['aiQuery', 'aiAssert'].includes(type)) { + const info = allScriptsFromDump(res.dump); + setReplayScriptsInfo(info); + setReplayCounter((c) => c + 1); + } else { + setReplayScriptsInfo(null); + } + messageApi.success('Command executed'); + + } catch (error) { + clearPollingInterval(); + setLoading(false); + console.error('execute command error:', error); + messageApi.error( + `Command execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + }, [ + messageApi, + serverValid, + form, + startPollingProgress, + clearPollingInterval, + deepThink, + ]); + + const resetResult = () => { + setResult(null); + setReplayScriptsInfo(null); + setLoading(false); + }; + + // handle stop button click + const handleStop = useCallback(async () => { + clearPollingInterval(); + setLoading(false); + resetResult(); + if (currentRequestIdRef.current) { + await cancelTask(currentRequestIdRef.current); + } + messageApi.info('Operation stopped'); + }, [messageApi, clearPollingInterval]); + + return ( + + {contextHolder} + + +
+ + {/* left panel: PromptInput */} + +
+
+ + +
+

Command input

+
+
+
+ +
+
+ + Don't worry, just one more step to launch the + playground server. +
+ + npx --yes @midscene/ios-playground + +
+ And make sure PyAutoGUI server is running on port 1412 + + } + /> +
+
+
+
+ + + {/* right panel: IOSPlayer */} + +
+ +
+ + +
+ + + + ); +} diff --git a/apps/ios-playground/src/env.d.ts b/apps/ios-playground/src/env.d.ts new file mode 100644 index 000000000..6fb53468e --- /dev/null +++ b/apps/ios-playground/src/env.d.ts @@ -0,0 +1,10 @@ +/// + +declare module '*.svg' { + const content: string; + export default content; +} +declare module '*.svg?react' { + const ReactComponent: React.FunctionComponent>; + export default ReactComponent; +} diff --git a/apps/ios-playground/src/favicon.ico b/apps/ios-playground/src/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..3780090a91ef5494e78a0f675a9f674be3e8fcbf GIT binary patch literal 61841 zcmbTdWn7z2&@POZ;8GyCTe08;iff@*OL0nYcPQ>|MT>jT;%>#grMOEe?(TkW{?GHi zzw`Ng2$I~{nVp@TxpwB-a1|w4Of(WSI5;@W_i|Dn;ouNNffp4j3h<64E2=y23uG^+ zD0p5gn`Y0;_S204i3;cs*Cax$B2Ui_~{$PL%2Y1TwUP@fe4gTN} z{U_P93nN;k9vg_-AeIyF23`;k`WLGjBZ*vrjOtw!JUWVXpttWop|15gZp7R6*dgc& z`bK2x(7LP9ijc!rq%h@;Pr}=3eA7mFGSXpMXjVTJ`4CFkD=G@v9fy{ya=q#4 zw%LWsA0ogX-ZPZ)MXrZtpJ&%stRAly;4n}EyPy#8|KqP80dOdRt9=_SE(%cojHRT* zgalI4W~3Ckr>wHG{2&k%5)zt;u#x!MkWq|>K7WZ!dK^U)728g;!-o+N%98Q`0TX`6 zYPIG1wmz93n?PlN2adv(l!%~Ek{j<~8^2o!5t^|BWQ-#xP@=#PO;*EQ`!oV$HEN_o zIZadKv#v<-Xq=5Cf(re*Qr4koJt^mMW$*CA7549XyKC9sv%{kpWsK8`lL8;ZUOuiJ zlNnQ9F!gZ899g94>kitM+whlICcGDwKRpu8~h-sS29EB z1#7%~z^1kY%FfX|?GM)?kSxlyU(znwpfU_U&;L0!x zb4pO}gBLhzk817ePua6U6e6&U{RmbgE&t>R-*SV$8`1c{yoDiPS}X?9sA@l)PHbZo z7>XQd*6ZJrOF}0!3IcpVhGG}(@ZiW0Vi_1~{b&%yr+G3Awx}9R?E+(IJ`96Rsy0J8 zC(Iic6JBK9b@N-g-Z9QKb*r8Z?E+KK2-IQ${)1%Y z1E!7SHPg?LIGTMr(Gze$`gxuBd1z&`%JA0lt@Au}CsYlS`ICa#_3=f8UX{b7>D=3T zOG5L$^CWnrPoQ&UIwHkZsJ3BqLYuZ3g zrWgMs`|CT?lCYn$=`vqH#C>Xjft|fVKao^pfFLCN}-xllBlW5-WT0RQ8vB`ZAjf4_l&0)x-tt~XEfQOY!- zS?8l!iIey-uMvh?qR8e8L}{*J#FuD2h42PiNSNu>(XPy4hR?g7-6~-=_+6^4W3j>R z6-r>7KO0~;$umfH09v($B{u~8*>BI?Ixbz$!AEMu&`;Y#aAyX$+QzQy|$1RR9{ zD2ovg{Yl5KtC*xg`n}U!-aA(p5gI4g)ej#jJ&xGxg|B)ORCu>OKfr>zz;yp(jX++?;_84luvgUjArYsS9nA#bDDfWNV7*bN z=B~3;y*4!A$QZuGFtFhz(19Oa8_oL+*rI1U=?O)T138Ap$$h`V zH&4Dbc3=X7bTK#=-nZnL?;vwYlQ`TvyRL*TjF?JJ?_a>NCrYB>P`~*Y@I7o-f7&j zZyk^3k^!jDRk4S+2&O=)Nksm~SQu*#7CZP2WH`MmbgG zizb0Mb9o7^E#?@MaFcsZla#)=M{UpIV7x-4Fr7T6JCXz{8T_t|(DAo)`DQfK!u;3? z7ducOMZ7kzlDOl4MZ=F3VC>|&c{7pxOxEt|dddA04f=+7F8bPQ``8xP?BYPuGsj?< zE!V+2j;Ktgs`L->2W0bAfl@KK@T(9C+@XzEJ2#^EYfc*6VKGhxWHA(@sdM5>e_;zQ+YQ{ z2qaI_{2j=?o{K9zsH?`(0Sw>X5%P)NPqzAZ6%zxAYAS+Z-{Ti_vlrKX@^ZGzumO(q zT~&CRq-}*)NV1K*P_T!euQ0U;Ek$C&rC8KLR}!U+ji%>Fu*Sgs+U3x|d$7aLpXUT_ zJ8{`n$YHC&ktaL1w<{IszBBkUn1P|9q&J!|bNWxc)I zsbo~XRF>><5jN>W3R6Yu=lR+VZJAsMwpexOyHLF;J*5&6oFnKj#su_K3vqC82(|~( zN&!0rr3YDu$E@ekL~U<_Ugty2ljVh;9`%Qz;>#h*rD`8U2$;YR*x&LwNPkZMUWqb{ z9y=Kf=O$sREbe_$hYu8At-0G_=O8^DFRI)39q64V=*FYo{CM>*Lk2PFv_k3Nr|H`u;U@d4saRyE(#Z1>Djip;A%h@>_%A{8v)6^7*gR zDE}9~fU0nMZiw7^gAv7@L;*+48C=%sriU;wy;JGi@+T@{;zoB(*%TMAraA zP}=!%;_+M~GKgW2^AzVQ1x<`Xv-^S<5D}52S-p--*g26zTFh%~e;KT*_ z$b>)_|D1@E)>Xx3YU$*;Id&;**=p4zd}5~GMhs?tdHf}++#%?AKDb?kGp53Eer(eH z60qkjIL0d)apSc+`a}js0iH`C0G^Yl%^v#hrHe_6LJ)WRkDtSTcUMn^HwBLKwRKB& z#l8Yd5qbte`#>)v60q&>056m=h&ubgF73gt(U}m^IIKborCYr>^$kS9`BQt7qZE;Pd=?-y}l0t*Zgd;9= z-Tuy>te%f34t>MiTlHCv=I@%ZSqoswc1&R zbW(r#6k=Lgb^O570ZKNb&uP7k>GX52*I-rF&TQfe`4L!cXDQ{ZA*M}$2@Ha@qtu4$ zre9yh;qtPS%HvMJ0)ctInJaAC-p#rZxQDQ=!Fsk(R6=80dtMW#E&N0-be-nz*D}n- zlT;>i8IJ!mD&h^C9FTx*1Kw$B-c!}Vf&*a9`>@eI`<)5N9Q~<7gx0-Z%5=8PPJW#2 zIg!b4#&SZ@fXo;=T4Ghn@EfiEW^8(`D}Kh+=NVW}(v=oo^6N0aWhe$BfYqJFmvo~; zYqE6DQu(iz|Jz#6a5g1>!MA19_Tm7PGZwR%f3If`UroTJG3n>j+^7k$X@H| zG(-Xw;pA<_zfjW8>utoAUaxIcGiJlXumfH1wldz@xqj|yZ8Q7Bn83+x?*goM`1AT7 z1uR2!92kdOFRMqdtp2D~N-rbj=Un}^iA?Ezmd11Da02srY6SyJg-Pl0H3-3Lht2VU zm44OSZT6IhVb%P*tWDWc`Zppdm@*anxBs-v2(!9)QKuCBrdT1QpGj&7$sWDKDE6@l0uRcLmM@ol zTtfDtP>8q}`$UR7U+fa`w)O{JBrWLn=>Ykwjs_1GOL)sBiXGh<_&rz_M!~}AQhmbA{Wbgsi6kn>FLRn&eFac#GIWsqm^Kn z(+~)6R{kQRNTm{BPQYmHRI6u@5RPol)X>6j6?96LUhHGf)y0wH9ZacY!+Wv`H=F39g z1on=ypAXDBkoq+PhelIiyWOVJZ)}^6T>s7DYiJ~t3G)?K|LC;E7Cyr-CW(M!&a1zh z`uxF91~uYlBP~1Qrg#kGA|`F~)TrT=mwx1cpO6NwlO>Gwpc(GZRHsWRwCz}DhypK*V?3G-OXXow^zAhVDWLEsHlnAV(TLp+k)RM4ZoJZD$3 zsS&p3NVRXgfeI;&CeaEt>&dx7r4}8D_14ylCCBYq#)6g@MfvNz7SZ8hIyjFbe#y zZbVkNe*4o)J0J9RZ%^SgP14h9P$u(yRdbnF7s#epP8(0ekXCs8{8QyHW+9>cXVDyr z+xM5b^zvf3IV>vX>1tU2Q-3E8*jIE>vEkTHk@tg>kNY;u z2fvW8mhe%cr|bQ)S~@*W*NahUUA1mi*_BuM<+0O6962M>@>fbattemKS6r8DHFdp) zzq{xiA5gOK*(!>GGui+wg_Ikn{L^adekd@PlafO$UYhi*XzF&KzJlr|*eMh+)B ze4%Vfch+m`6D?({Su7NRr@c7)=3yJ_6jV8b?EkSL{E#n&y_^F67b0!p^RB|I(A^uX zl1rq)y98LAErdag_GPG`LR`vAmV<~GUnKU9i?!;kKhqq03}PK_tHFtIoIiRXVWYW& zYwO|N*0*Ik_7|fg^Iyoh624C}4sKt1?9%wZHVd|`l3KES6Y{10jf=m{N;M)0JTNv; z`k)N}#wMsc>EEvizNY7J3n2xf4167p_~&(^(YDyD+O&=Yc@?44=BJhmk^L?kgWL)> z;z8`z?(S-zn!%BYol1QAO$_ zXv{p%cFk$>GObDLX3MOZT4748fnYkDCtNx1>o0_Tk4~_HTx;SsrT5hM$zIHVNg4zIo{XPN{q;9CZdPRGNxvddFFP2N2J7b$_inX-arWM1H&b7x~vZJDR>K z`2CJA*$feL96`xWXH-phg|cJ8;k~rwqpBcPJ!}7n#*dDU04Tzg8=xZuI8PK)^7`Lw zoO=3rJF-FU1|X=h9-PD9nHB>>XhbaMBLs7fszsl=FH8Cc+*Z+-M{s4TnK!lFXS?&6 zHnWxV&B?mkN43A`hxTwnsO^Od5Ivsn)qDQ-K3)!Uk9LoBAr+4UQ&2IEOAn)Id+j}gn0Jot8cbL3*zH~?PKeQW)OA^s{y}N# z?h{X5t!E?CM|#wf$QZI{xf_7pJav?pr0yP#$2F zNx~9uc&Hqm*sVpT@O7hADfWndILdOuTQUH6K>Y(5Zie1xl){fptQm!|S%mpGNg_)n zc{z8spK{phod;-4#xYB#SBZG(sPbYR$vJ|@-vQ$dDf zDozFJkHof3yD>RfOlvS65f=P4NrU>Qx6t1`lYj5S25C!l-fNbByh^%b6xp6 z0%$~Lm)$$ao*yOxB;mk0H*pm%vP+=g<34&YhM=B-;1mf@n-{^h?P8XY-l*lr| zfNx-I^cHwDlr_K@*&Rt>?u+fkD<;QM{Ufs z3z$*4{zId&r%$yrnXFP?uYk4ie^~8Z?gL{1d-v7LVPWOzA-WznyENhcU^wVzZU)gs zOB5jihNG9O9_yT46`p)fxGb^byo*L1mF

7X(Nxg#r;LrO+mY2&;5i&;NT9w7G9FpTQ)!;d|wo;grsfaINT0SpRysm}J>1Mtgv zXQE)Yti2gk+N+{z(Z^RU%H{1@wjr&93?zgor|=LP z3F>_X9J-3UIgI zWTkxrVxg%j8tPh_Y%&PPzZdwJqv;5N0@im^M#EK3muEC4WAcEbwUX`y3cPpscZ6+% z$L^y)7;thD82|qgjjB+}0|xwV4T0yPsJ!h02#n{rW|VIF8_cfe=I_cFln7Gc1*4p* zY71KKR=2bT?N(G@sjj`Ogj#=K~K0=+==SwBxb7)0{6v+)qhf*EW=4cJQs?T(4|yt+~fFHyT~a zw#%0hcEt~2T7zN>GeN9!8qVn0B%efm1a@a`FLd?lTt&O^Dln|NFy7$gbwfb0gN2<= zIK3oY8$~kz^{5ikZ#%kU0T5nubK*z<3(SC9KFvH9CvUp;e>M5^_op!^3O`NEOLwqu zM32ju8gEK3zts-y5}ptpb%+>`%}Cl0$W6Mo9LWl0 zIw+_hyd%d7kA|-}#5K{F2Q!4?XwAQFp^1}W%Cqz{0?Ywu?iXC&+6xB5lq<_RH|x4F z?fuzSnNKeLhmnYlb`FqMYWVF?CD;^r~R!*&vM_=0^_NlV^r+{%V9{?{;(se@bsY@r?a-D zAKL{#CzRp|R3_p=Ko+a307H`Ynu;graC<^7MzrMoM`N`nfILdbwP91_rdl-J|Ndis zhw_HHW8Gmu)oYwAk=M7U$Qr|=xno}XciTVT(P$!+Y18K(ir(rSlYAx1ae9HKjgJpz zqWAQWH}DDBXyd8#E|?e~--lq*l$$8(*YMgu%{kv)3@YbX7VCFLm`CHm{|X=NFY)-I z`$n)OBK`|X;z3iH$jBWIjo2Mtj`Wsec+Qb*whaU8`U&ojqhc~cTSi=bx6Lb>d}TM0 z{oVm>_1~fj#{h@YrSiK!yDc9A`Cd}ah)8Anz~0w*^~R*=B`CTayd^d3VJ=?;?rU#dI1!E1 z9Xvvv2iQZU{l@&Sy@7u_bvjQ|L2$w+|0H{~(OLD7T+VBipMCu~?IE`%aJy-GAAg92 zNa8BIAem6?EEfhyKk02={s%vgrmJ&)zO0!)om%1S9D1(skS&xHxXY71j%R^)7bh^4SEiC)&U37y>YO|`lQRun zl>f#WUI*0?)X~va>DUCBz@m-sc}Q`qd!^*JX@lx*uVMK?a`7<;n1k=XqGb%;o^SV5 z{@m9)0n&I^h2AvBTkEd9a9B*{!o#pKC47G;5T-jdJ+Fd&M>bGDTD=g3?8jUOJJ?rRtb5aPLz#P{#t1 z6SCnA|7a2ErY>_()`YsW>h9y#@=5NfU2&JEZ|5>DpFi2@U15 zhmE~9d4uHfKG~<}es;XfhA@f!16Fg7Ge1%J3d=tRy9V;D>M)Ab#4YeS^<3fN+`=?^{o7oowzx#Fc?G;114_eVjCGKi6s80U~iCCwi1G!NP(SZx8h?a@AFOI zjtPvyr(_c{I{h}&d#qPh)~zAA>R^(kHC$OPh?UGmq6(Lch`2jgkdKWTgC!F6BKH;F z6sBy$*}R#*Tq3a)ZUCbkk)A?crrsXD; zH&IiIyiz+6GmWCyqZ31WgEiK0C?tQo_+K?<#uJ4?b^yiHcy>ho|WvgV$Iv)G0FQbYfhIh_q=I`}3*i?weUHTKz`A@_S1K;0Cymg7ekO7BC+Z0*`PTy3bv$2hEgi z+CqFw*VjW2m_qxHZdi;ch=1uqwjRlSIQOU)zOAW%1!AfY1HPZa= z-V%*w3L|S`gb9)0xv;|GAA7Nh`s-B9>W8qz53D*ZuQyAlXZ5pvXsJJ>69vdD5au7i^%}>~2s45pQlKg5jfEi?id_u|*|4Lyrq6irA<&g; z6@M$|Sl0WFkAM6Ym6;1mRh|n)TrPpmOx87BH4jM%LF%`SQdtH+1CpE;t5v_6mFm6* zf0_R1U3NUQQG-(Ds6zACr3&|yq?V!urLjG4|5E4saQ82}reFRFB=Sbcp!8JT>`0hd z9*S=YKJQe}+)aES;^1 z5$dB`vq7clI40hc0^cAtGHCRIy!8=buTvf_{734pv|U7c@V;>MBSt_fy`L=^#3(SF zuJWx?(f6mW{hsBJkRg7$Y;+|HI+#Y*h>3DmX;P_Ad zMy0axw$gPwiDBX^w@EGB-b=1*wSV#str5jJE#b<0rING=&jgQT(; z-w~3Skyj<8{QRQIzERWg+EQ|YE2t*+9O}G8?Tav7uT$WthqJu$XiZ&EQG;fQGM50l zoo|PIaS%DVxSjw+AoMu8GW(xbDg#^&qXsB~ao+pWn&M7@=S6x+cY!KRW^!#E6XgEzhkom?3uWqH(=*`AKW1&L?AoMpk zt8LfQZ76fyy%jEwhL5)-5HtXwfiQZO`>eilqC^8=^Bi^`A_)5d(){V>@B?P`@h9uD zl}YWB+mNJv=#wFldPNTY>ryjjQsN)e0szxysw&BDY<%{?1&~*ffFSq^^`w zAgXvJ_Q@*t*a=E-`u^;dNUZP=3s3^rybDkDR}Vcp?3<12^N`)>voPZN^$B6>@Ml7p z9LsVa2g3GeM$u&kYU@W;gq_oZm$Dil;6wYt8nwObZde`Vt@|WFvq>S{0=2)fIr2;AaUf zKM}VX;`$0?KI^E7G&d2<*sv-rrqCM44XK>LNHV{cAde%4y{GO!s2uJ)1&4WtqxK%$ zf9j0BCLL<#ow;-zm^cqsTn@+D%hR!#g?J|MnL9Oo(@W# zvPc-heUP7D{ja*qo5k15Une{8l9>%}L4TgoM14gvS}X~c9s?<-%AXCcHQ+w@ynC75 zX_7UIC^A8&k=ck9T(Ofx zHlG4a;_ykwZ6aElJ<35s7I+Mtk#GTor1GXgutc0K0SGIUstyLZm)!Qx(12;Il4^uV zhkLrC-cfEorTzOCY74nN{TKHA_`&CF8&Zrbzun`HLEO`w3%XU8FgGczbP}I>le=cL zY`=UQwIR{TGr>LmJVzrQ2xmm^$o3nAE9bh{u#v7FkL8UwS0J;$vySiY@V-2LK55st zhCBvINTB92+2^a~FO_BlB)MJRuhuK@pDruld=GGCDSfY#%(BuDLf{!hvC0JY16UGD zZzchz$UDj}C7GPoOw@HWVgRcvtzuZ@hdLE+`hL92cF(cy`Fz}?cwC!TedyYGtA_4IZc*bL6Q>EBr2+aLT>>jNs1u>$`4^G^CAkeq9aa5wf0)}5{944 zb)ISSGj!c2PW970_DYsJ#h_W zM@6I+tzS2MKMa1eDB5?#gGS$#6)}g~@4FsUT2R8*G<%d`qUx%CR7iw`O8n4FpMG8_ zQ}OU-4yygr{WvF}8$(AxCxZ6oX|(r@7TP0R&kp&CWV+G)KYdw%DyM0G6DRxp{>%)| z54&*1*L_^a4S~nqr-E=^mzJm9 z(hI4h?q)KeaKX$njEQ#vkMhdcUJG)2x9-@gSJrm`upT1RX)cyvd!2 zyqID9NQ`6>n8@5Es30os7e(N5MBv@bWOsY^)+C>`&E9>Z-3mNFHyULWr7Fzjle^gv zUb$`6@rw%>iV68Pcxze=g1xqm?y@(~Iiu1*lEcjfXkKr;b6C)K!v`qSm4>oQXC^WZ zn!;iWWVTl-xKWiqVdGG#&6pG5>_X1yZ3`BYdJngd z#=DrqS1#-|U!OAFc@@eogrv@@pYezzQ}2Xd;x&g?NTky=FF&xLULg`9t-z4GfK-t z@NcffE&pYD)o zf#gD8JL|JcXCJ2{@b*UmegyQJ5e4L9sj`r*K8RPmxmE20Kv%60({h;VFUArC#Utx! z-ENoBXljM*n1>7t7gB}9V?fdp|1JQ1KQCvEB0{pbQBvv)a7h@;#iZL+*JC`G>c<2uLIi44%8%Lk?1KB zR|p(nyGe|nMJE;{1`us?yx^IR#zRMhUn#w2 zCB3q91As>yz-Aqzj4!3X&~O#y9hBmHKR)=L5UL73l!wul%pQCv!a9t(N{#=9V_m1X zQu*xCe7&POph7{dIAQj@Tr?53;X)l@X8G8)A~o!OmatScE2#)|B?{~+Mr)a7TybVm z(PnrKtwNPY&^27aIUQRT^B^}+BVr_{_8R>w=MBjH;{geJiG6(pz$&;lB43&!#lHbQK$HaI+OdmmNJoF z%Py<#+Xw)$RV{k>AI;ds+0y2B-=idC2|`vRrhAswpiZ;H9AoMDQ9+d~q%jMz7SVa; z;5H7U;khLryems}@b)-j@}{gZhy0R0W%CV9F?z`*BQNSO+pxe{g3l1I zjH78BH{edBVH>Cq8XQKRC7)R2laiK4`BVaEEefy3sZ_eT5X8X<)jy@6+A&^7T%<@% z1ADbJsk??3vgU#K6H2H~!U>Mvnf6{b)xJqgfpEi|z}TO*lgrdI|95N}B7(r>wgd%7 zXFTe#&qyo=NF!5=2#tK9SBz{fEi*X&1Q3b+w#ASyp%0(kyNJ>THY96xd#i+W>oh!C z>$Kt>uhS#>rY#;-QT&@tA5%YV6xo)Jy39Ti0@o$_t8#QcU!;gOQwb&sW2)dt@v({l5BWfqE+2280*2roE4O7*6F|2+6nA(<`s ze6*b7tTsgwa6(C+y}CPHHwtANGYGgRG^VUcdc1A$9iOVLDuco7K#B?7&61IDM}`PY z!Q5o0j4M;@y^U{XRUEs)ydz7J?d{&oZq*F7_sNnV;BKbllLF!|0~VJYChKd%RH6Yw zHNqH?Xn~dQu6;AqnP&3wAC#vBxnDcEj!jPQv2va{)rG~u5PEKn?{(tp#pv=jpTn1`TYXOwSeCec=pX-z)t@WKqH7tWP8Bh2aF)TMUX3+qQNd(9l3Y1N%5Kmp`Cr+R{7R?&;ng zuOIYJMc^%cxQ7;OT?u{0aV&jJ`Ltuk{4o`syQ4qrp$fPy0fl%}HMi}&Iaz)jySH*$ zp5qnk7AHT`ov-ymy-&0Y2D7R&te)17p9M@!M|`ZIx98RtFsQ~!g%wl?8xUv5ju1)x1i4^s3aWc>MOy{DXK#IZv45>;PC0yoim-CsD2O0fgGzy?iH;}#?b9+c??eL+&`Q%nH|P40BR64Vu9O?t=Z#TNcy zXpql~m%l4?VNUm9KWCcHb0y>B*K*yT`b}%LZ!^(87>k>~UfX@jtL@wG-FFQrUyfQ4 zLXuM^Vrc}h;#C7|`jZ2W?&K#u$g4(89dW5v<@Swx=sv|;H#B&(p{HBCvQA1@^3^6b zsG=w3#nUjrp#+HT z1ZwiMK_-;9W%y{TZ|3HyTWHq4C<Tsr=Y= zAeYp5cSI#!%%UbqvZ)N+9~FZ%Y%n$tR)mst{?>fa8b$Az<(1;~8$XL(ILKfW=3D?B z94JEa;FYz6)jJ`1SC?j_1-?>+EEx~#lQ{eYwav6F+ja<>Qp4Aar^^mF+X4N|nd~d) zR>EcLWw62-xBJm;f{T{78!@2FO^4D z({pq=xQ)=!O3P*@n^W)1@L*7xv3Qkk%8y%lz2p#xKVGo|chxp%VFzyq;}$)}1L2mD z3#Z<(neS^VyAzzbdG5kb~|aC#PHY#{3*Swia2`Z%btMN8hr$ z34Yxus>lg>*8@byC|cugzq;=Pg7-Z8;nXL)hbX8zYL2KmBsqUOmXm^%qFX$Bg5c{* z3g}~Z_nR`5YA${rMRA8BGmlOxN0(>*#j-Gu(tY)h<`Wua`S=~TZTEIHjp$HgZ9X7? zZ1!cduvq(JFIxqeRNANaNsTsM;V;cjwlwKlg+Vn3=v!;6jYD(7u(&~5jzLCacT*mj zABc@#g}MBiXef>G@|qa0kjPxfc1XVxQQs&M?#RjsvBczPxC5NRL2_#iujF!IX~lDV z5-iq`?7S-CkB8|W4o^*(D6d+ILr-ksNs=l?2=?a}AH=Z0DjI6a$R8TYd)SW5`s5t7 z&>k~$A?dYL*cL-@TVmxQRJqy*Ty7(XxIrBt(H^`*yz|s^lMC2BtZY=Y+~@Sa5DN;b z)GRFgZR^@lz$3<@3?i+8Y)UE54KN1`KsjE&!|x{h^rWHrYo_>7YXZ0{Mjr+04|$m4 zZOhcK1!Da=MlPG6{IY&PlUKfaR!bqyd!|lbsqx3|%tVXRHHCeR76QQTBhaX?BRqCm5QMf6G^(}p^L_cxnunL_?3)3wq9-N# z*<|WpA9^*6|7_ZfqLFW=gVSqEF-MJ-Dr)bYo+Q)OM3ZS9duDF<)~0haLL!NMo_XQ6 zRHrpaeP*%{!*y|gMIx!i+v?CfRYtTZ`wG4oF)o8RGHldH+-KqN3h>_lt}g-J%Py&p zCAnhe{XW5a;&bQ683jM4Ra_`LBfLN`=OPD#Wc*I$@l7Z9Un-<5!oqhF(5&R7GS3jGC%vAzOcIpyFq2Uf%;|<=Xha;Ao_xrU*pajTjsQ0*iHlL>#PzReGV!vHNB6~!FAvaWG?41_vz>9mx=uwfQ)NZQ?THc#zk=3k&wDAQ8C+2b`jY4muJP*S1klkZ( zTcfDfxfh3tR&lYXgGRQ5Cw z=^q?@s+>E_tZQj3)E_;{J>=*JOPiOO_-u6g6xp(vXDsxuvv2pYpaE!%t#;B!LJbWC zXjs};-Y~f1j&N0wTL{13b_N~H1dM@^-Ya_M{!V`RsdA{<6gNa%%H_$fUX8A>@ zH|EosO-^k&uju-D+>ph_=N0U36a0>m8=NqFx!209`q2PCg^JUesr&ivg0#xweCsGC z=wblqE$_CSdq}uVZ9DWugwsMzBtq2bq;`7OV-$~4H@gID#C&4%P!s%|S$Ugn`U^od z#|hf^DL>~1!?L)_6&H4b=3QsIk*@lT4j;wky+p1(`NcEk`tR)P%>V9^&Qz{|FbCvf zDe3KP2M&FMNld@)Hf?cduWho*iwRd912@`$J32)Zq;?fJnLfF4VOZ5D}DT~*u18RKT=je9KTsm>TPWSuu5zUD8ZvLxGuW&ZeDNNJneaH6sTznRT za*&>Db23@(W5tz6)5D5Siw{eSpUzS9`LPHHQ+9nw`HlVG>8IHiq!GdPyCUCie@x-n z@Mw*=4B3Rk_Ipad=x~-uPH@==A7mDUI>Xyl`!!D4o|;CojE6TOWK@=yXwtZ0GlN4{ z{b-K*Itp}^C9_xgltr!`aj~g~Rl7tTekC>>bQOikTZx;sX!)Gh--NLKwptQoxE)lN zED}#0;($9&EsJkw~duQCojSbV>WPtXDN{vK9;k|_zEzSVT!Q0)X$Y-FVlvwd&+%FnQ2x<1O+Z8Z8WAwH8q z{Ou9!$1&&Q6Y)jc#b4J-`ErR;W$(u6k3V#*4vb*g9~^@Uez#;bvm)=++e}ju)Ay7Q zA((tOIsNf`q65LG+phPmA41rXNyr9*pPP$eJQWDoGsvmL7 z@Ty$<*VzA1v~Z&EWOX<`DjrCKt%x!kw{v7q)1L}Yb$Y((2(5{enP)Kd?Z|LgNOOi& zRMVtU36B04BTh&WoOc$zar(NXBw-@HWBpoKC;PcA{p0=v4rE$;>TbGl&zD5OM^G?Z z5rR}T*8Iw3^VY+V@3Zf33E_pHsDzF~_SICY9PIlHy3g-p9ogb&q<&**rL~MX@{_>c zCPvERW<1z^7vz(&Hr{>}AY;AN-A3DbErV`|d+FlKbXF-BtNl$*kuT?t+_OXYY~6{T z+A5ZwOy$mB(B`Ohp3q?M-up?|(!ha6BQoGK9qUtTVGd)lE-29xKbJ%!s@ z3Vnlbkb0;LYQ3%ou~u04jOmzY{|`;)7#@fFwC&i9ZQE{avq@uH8?-@#jcuo~)wnS> zww+(Y#@5DuxBusOzwMXZV=+7T%stn2j^#cJW;Nc_A)f%E)qG9q*nQgjIWfrK1=#R> zAq%5;J$5u`S?Gy=kMKd;k?Su3jBY-gJH^T46?^UY%5a*db&QWFuZi?4`4ak%$?a46W-=G+V`q4&G%S3}8 zg=O!A+hWB{61wYFGA79PYV&RPyid9x{iFiJEugXMsT$sM&_78PkE(B=rl0(*YV7QO zpE5*EA-@ZB83U^PAP$ekEZv%x7%UEvBfn~(-~~7c;^ZDs(MUz{F;Yp%*1p8T8N)%0 z|I3Ece5Wj@S_a5bRb78{8cLaET3d_weuMR`C0BzM!QN8=&#xNPM-00?*ZCjEYXTC| zvy5P#YWs5sos6svzKjAJ0PrsTBXbi?Y)>aI-RVZ!KXkScQznrp{pC z@&PYJJ6}blVXQ&0Q#!uz84n0*okKf~qA+XdVki%L%w$At8l4NLUukktMHcVLn-GrC zEuh9O`lX6Xec)f^=qTOk>6L*Uxat1pvC*78OeSmVvZUzqt&c;G72iXjbKOlt#YNoR zB)T{<&$>cjN-*D$cSXWhFAjA~)2cP%_3rrHzp7*8==0?KAGFzDh4Gba2VTYlVn?*2 zPUVRCbQNeeJRp!OML*l7&hh025X0|yA^^R)Sn{H7Smb&+57<}Mv0?0>)JZM&ohrJ~ z;>q-B)wHCs&95T)b`L*tojK{{Bzv!j33iUp?BV^yIZ=q>K14z-_G+?GMAT<1#q0SI z2`eXOQsLackgF6tQw$^epLA+WzcmR$oDoS$qy`rDEk(eM7Nd?vQYH8=qgI)k_*PCc zaKhck;ce{N3u0V1eOvtqU7bt(V;a|H_~lh_{8TIrRe8B z_KqdCwbTK#<#n7)&9`<_F!QDzqE%8>W3ao8Y=DsS=VBS2H`eb;W%_%Ihggm4M|)+5 zwN81SRp>Hg!K+G9vu2lZp1)90QYi5lj(#h%pF}@+OvM3s#p2YlkQl3u3uo=(XV=d6 zQ6@~`n_t7`fy=RSlkddOxqQ`T;ik_693??p!$jlFrPZR@B3PM-iax(HOjE_Ic0Q|l zj`ym??j$u3az&f3v*vqHW?L2Jsh$U66&BGRSrV9{WdLQS-aYD2D88AGNe~w>)Z>}h?@AZv-O|J7Y$h&bhDh}(U&%c52oJv zIW#}x@ny@;^||Pyyl@zxxde=S`!8Vo{rFIJtmWe!bpbE6Q=zkf9fOzpmS_o{S!39W z!z=%Gp6kqKH_mE4Jo0fvaB0b8V%Jw=U|y-q4$df(^#>lMy0z9e%gk+R6csN(3-cL5gYDH5Y5`A2yknWf{K5;l5gH;# z*zwD+MLp5h<_fH>4M2Ha6#cmMVlh)7WPYkN+!L90^sFp!Hhl%_b~U!i91^HI;$uMc zMxJFdW(}l7eVGc1(`y)}Nw1MA*>jxlfU`HPi3J(Z*|jztp>;9)4kB>Y^_XXRq!>gLc?;p11=@tzAF z=TNaq-oqc<yk4Ssd+b|;*I@}~^d~3& z_w7`xWI+C8TNKJc-?1dWtNA>8WDk;wdlMo4(YzFajz4p%w6+b%J$fvykA9_eM4Pp5 zga$)UJ4kV}x4ithyj0@)`f+GU`iwtGxAmEioj5>#5wVPn*ta_|9DVV*!sfnG;htS! zW+(6WjQXcAKpJN8t4Mu(1?g?`GB?3j^;i)*Qtl7csHFoaLE1m)`2h+UXB9x?JE~A? z@?Rvaz8ojFB+E$3N;Zjn5y@mEL@+g?AV5~aTDs0ef{s}KUW7rK%%ZZm_zmih?l)4A zmKO$OBA*UiqA#>|c|OBa7R!2AqKR3b?jgzW7jcaQs30tLy`-~G&CkxnE5bx)$H(D* zrPZb9|4)w1t6#-%!91}F)671N(K{k)j27paaWO17*3&Het6xG7LsQhlw6+8?6UhL0 z*S8n_%Ngx*{yrY1Njige%v6$w*vfPFp|}8lb780VeaNHpAjuGe=-)uDpBH`CVSwrxjLo}^QdLD8x1H<8Xu+pAuZhyF(2 zyzmU0XxSH?m1b(mMz^S`65^VtX+JyyHT@vSb<1Z?N4<3o-Wi=CzMI(o**7FWKJS_) zR)K&I{TLeYK=i8!E?CQp_R!6!gWBpmvnGZ`ZNu;e6@-MjtuKR|@SZuA5c7KbyczTC zLv!8dve^Mkp6|m>B`ushvu3_Fy;8tj@Kbg+#xS%jMc_XlE7YGvsBB{B+Ty_1#HX0; zwxNl?S8upu-l(JvS5r}r4w7;T7Ag|%zX$dZ(I68qXX~eEC{fNg={E5{(Pk~Q3b_lJ zd%idAj}7>ve3d!xdly_aaLv;5z;MEeY=ge+i(Y;|Q z|Eov^!v;0~#RDqZ+^rW-iRw*p3WdsQ1-*hDhHfLrnZtOKiVtxFVZok!(_KqrcS{eL zSmg~Y`@`bqW^C)6v-!MKiB*Q`?|;IeONLhRr=7b;v(EdDpx02&hol}4DXiRG488Qk zoDl5fMp`P72wE8i8YUw1Sp}Kcv3P3C1~qDhZZ+rmCQ1LMCfuL(C3_Jk&I+b~=O3ZG zw^1$E_sUVA;+W%*R6^%dRb;R#mB%zf$7Y>>z>=@salYPEZ3P`Zypc-%WJ5-2?*}*c zPaAcX?U8(Sui;FseN>2NQ{Os_CP}@gYh1|0GH~5l^FWlqb)69Pdf#9ulv{jns#rTI zh9Syx?|0)w*SKKAOI~j#Uva7yl?|JWzODXzowbnhHIQJ?HfS zS{u8|TDpDZ$Cm6*i?!Xa)IwlOrG;j(Drc`5QJ$X9UP5R#Gznh*5fubQpg!PYcL~k z$AnO}G;;r0U4Ax3S-ZlwSB7yb0D_}{&yXahJu!Kx(gAC=F4^XkHM0n;=!&RxnK~vs z(`|&aB1B;(EVKXYwOf76l;1tYX;>oTk0?Oqcbf=c{ z4aU@Sb@JWZyed)Eqq2;#1Z^$P$6)K8*5R3sKNm~;;ByRTLY#xD{&uGNe^^2GFB4b8 zkhW|0Uf0a$vYWQ1Z09lzPwxV>||I#wrPXV=L^Rw>9LHS=$ zdG)uJZ5j#w9oWB=B2raac@x$ivURoZ`i0I9x{$HXDx}MRy!W3Qr8eJ$suyI#)vw8R9U!-67 zcV2&tVVJ>4jhAD55&abUS|I|uY6D?BgpFdDz)%+LMMPJvJqix0aDdnv zFaf^%cY_X17%koJ2xqGx4jQ>S+aQXi!%4Au}`C9VoJUm=I)7u zy2#0qk|`l|u!4qus7+|)A2W#AFlkxKHrAkn5xJk^BwkAyX@D3mbnr3q)5(OU8nAZh z?Q8K}IX_sde94y7#w(1P?@(DQRZ`3YNdI@oXXa%(uyOi>uT>%mvGO13J?^3XI%Y+{gwZ`OqRv(}^Fq*4{8w+fSS;OB9_cW@WqyN1F zn56GC2*#lm=UE(Ilmna&Dj|>0aag6rH$s$!bqqSi_Hk%T^Ck12TRauC>cr@V_ZmY} zyGZK`O3NHeJGG>E0!)<57Hz|6vwX^{<12zu=seF`D7#=xFSKSaSst|AC8BN?Grz z>7`R&DU~%s6Hcw4s70<$FtDljOuKmAs4_eH$EU2cM_Sb3Gu^wC((VpVW{lut*u*IT zwXAElf#jMerBL}1A`$D5T0(jvsPt2Fz-bHF70JhYYXN%`e)Fc0{Cd6tM4kYNSR|Xr zf0q9D`G$FC`qec?@hk**EQj9wk<2-g-#iCn&)IaG2u?K8xcOC*Gfp zo%eTLgc{uTPXomLHrri~e)~`xC@1Pd6i2Oa%yOs>y7){g5?}HDx>S;Q?va!qGPHS(uwoReR?lB=Yph-e5%#( z`_fqQ@*7jP^iGY$doVtjjYNyXj%-?Bgu2;s&L`;Nk`}-kT+$eF+Dv90vXaM({lIYR z7xj!i>>a7;O?B6O!wzejY zJa07Co*ovIRvN}zwHJ!xWF3wdv#4+V)^&W047MuOZc=`*7`$uqUPJE+g@={^@CE^chi7SF&3@t^E*4OU~4!&)dE z55IL9%|mOo?;oCAMV|uzgeOBI3mF7yGaA=}*3MAd2Sar{n5% zf4W5|z8c{E;5S0P3L@hb|980`N89o>Ptms}j>{5UB?~J!jx#FE!;LFKk8BfP<{N#^ z+c{m{gR98_BNohT5TZ+C9C+T?#QkFlIUft(<-IQ`vqEbyrvv6 zYuxh^{6tB>M_E!FaL9A;Xm#Dba(z=xspS`4B`~%8K25I`!h`Ay_l7DKrGlaN5s*V- zq-cNPyTS{-GZ*J{NDZ5O7AB?^6reND$3Cc*BjCwB=&Pu3(MVG1R?F76A2X7U=8}%k zBApZL2UE+Qj{94@xRO5frP=??2EN-HJ>c3@DA^7F0v0*S9hfTX7S(!M;;WD1Zy#S@l4xHv7B zVk)8-T@YGLQDJtsPW-4DrgWM9SWD%Ej}mC3JdqXoa(^Bo-2DPt3~RF3iFEe9tvhO_ zIgf}!XV~HxwP$+X<#icYN=Gj>?KpMradwc815^cwjY(3;_IN~LYv5vU7E*=j6vJZFOK+hc%`cOiOSi02f> z-ds6EluZV($G{_HU0dIb7{EF8O+RQS9*;5DlM1FJao7Z>|8*0ak>I~^yoqfx7hgjm zOm3%BS73ovL^7eM>DD2&HhYDalksx?9<~T;!@J)nW~tJ>!z)PA5_|g2t&P^~H?*H& zf>s<#Jy>4R&49-z|MIy;lv!=2{j@ePhW$fT@NchxS2G8fJFjp7KxUx$)IlQI?>VJU z*l0f84Kr~46DVIISGOy6PVk1mGdne}OXaQTH01a01sOptjpEi9lGY=U319X2CIHna zOu~IjD1Uqt=x1A5pq>A%yMIx~vENy4M|d_W=~hB1$7p{1D%a6*7lp3jEvb_Ep5u7j zbS!gC^@T&fX$k-LN?ntaeM=+pi@_7c)+P=RyW4k-W=ShV@X7vJdhb}$mO8jNI|G4EO_ zdqghb%&BPP*{?(18ulRjD1_5?PVUV;gXIx|BKcA@!y5Ba`yE8qegr3YH&P8AA&CI{gtu>?_!etSuWIH0ZKACSWc%t_S{8(5h4Z?+&S94 z+yqZ*X=y++(}+9x_=4x#;2i?LdBtTd9VT?&TK~93UMSMdd$e+9U#mk6YXw=t8+l9- z{LLNd5xy+YU9m$`Dy%LYS<68C%I^H9K}X(2D~F=x^(UO%rGkP|`>XvrKLqUx#GssSgWX9P;iW z9ly|la1PcCO_UdqW}B?`j#-|%K18hHh3YKfefn4BE)ZCRPWc&>QlXd#{u34N5}=cC z`TRrC6(JpieF)EkDj%A`8Z+eB+5*uw*f1cGrZ;F#d+M7!+Dys=3tkFg~jOZN4so zSYsuFld%FSymiAF`_J#e^72cr!qMq+|5H@g_tFpif9P&aau>D8nU9xI@xW%6rvg6l zxC_>Mo)zZry|RIeU|A&?E|BBuuerH}%@q2yBkX*va4U`5q+)co@3@q$kj1M2=1_OD-Ywe^CB zI4xTwvZ$O};BORA$Df810L~2VxWMEMHbqR{bj+^X!#K(}b|~7PWdR!-fgVErM}{y< z7*Eg1v=l?a+(w#urzM?Z;AFi3xU6@iho!gqj#|cV>kU(%(lg+-2^2a=ZkV>nLau?^ z-%%S>^chHbWQvfsC|Mlh+LGn;@)O7t0bs*nb1_ZZl(WuByaPn_p;?m}&by$Cn;<*|~jYzd`w9-}kpm zf2{bIw1RkgU`l8{;G;-lG#{|S{umAZZ! zJG}`XcKI_Nrx}{jP2+2Y6-2Jj;!m@h&T!&Z9q0Uooh?YN}8;59`f=7R(@-@ zCEY7WemgX}(Btd{uR*h(XEk8n0)gLHGxsMQRGBhj8r}O3VlC8XuD5 zwW|*jfU;$^_5`k@54w39&25uRnLqTS9$^2jTY@FOHQ{upspdTIf)djDX$|^zgsPAv z^U2-H8F*7}>9;r&-ukl-%S^x-=-3 z63DeTTC~lfcMGsWjpIw#gN5VDTy?%><9$Hih|G%|6i80D7V`0cUV&ftT#lEgfopP+ z%##FlPcl9C|4<9OGZP+!2&pAO^qTx+0wOWgFjycw5eDA{moQi(_qc3cbNmH0E) zvY_87w21pSlJ^4}Vu{S;MES4N#YN~2WWh@0KA9h6;Ez4Bp1gt@E4L zA00d0);UT@)_bPtfl@|4us{2k2nD81p0V1l>RD>MW!WT75TgJHKT-T&zQ)KkMQN~f zuLLp1GnfYwGCtYQxYR*}%G(hq*F4ks{JHAvuIPR8e*PYJnM~JnVN?G?vx2~J6HN7Z zv?lI2HGwqSWkzz0`y&SGxh0^D2TxLfY#c``;>vrrwyw{?pIYz}bGQM9^*(ri)}7HS zL-lY$N!Rw_)URvs5M%Jz-2(%L_2L{u?q#Q;+iEpmxSqzdz1yhY zFUFc_`?^}9otpqmN@N+AS-iE%M1tkZr)?{GoA19u=(meqzGD?Uh3)*qrJaNR;V!&8 zaK9{QYIT94{$~I}2<2pBgJoLA^XZs^A@Hw%(xJ%ctV%;@N9pWLZC|{v7rr;X48vd; zEI$_>x8NbJc*`6AfjBb^zWoe--{6GbuZ-K@ zmt<;opI4z^dwW^M9ny8d@H*Jx1|v~4^{HW58?u_(hbhA`^+&T9z9UL)M>uICasSE> zIf@@M$qU>417h}LZiZlw3JU%fuR+L=g(mlH$^9P{SmQHfi2y=qUpojvkdZ*!D8CH# zpWGzDW22V(f*r5<&jF1IINjRy&)1c{{x|*I0f^L4bqEpt9W$r-mS)&r)^Al)4@z_f zoTqUb06d;B4JzARVOWt`Qs}~BsFGbUhax0=s-&qe%-UaaXRN`HWof7huNH%Fp3?t7 zM?V@}Q#c2Qpqm`rTD~-K_4%oLQ+gj1Tpu{D{FK8eI~WfPBT(rr% zOgMhzDm5aho>|G|MuB*&{L=?<9pe)lC96FFL}i&cTmXutGjZuE12%+;j(9;(WHfs0H?Qu;mP1NnJheZe(u|L?^uqdcnCSgM%#n~8_m zK)Iw^$Y8<;TaW!iFS$Ezp#RfBC2G0}GKrTcI>+T&c$PL#ueK9K{=0m84>cCa1R?Ov z#kt0jYp(lx^P&1zNC^|i0)crtVdQ=}UNb|X1#j*n3`({8EY*5K#!rn{9f zgY9=TePP*8ab+9N8GpPe{Z5d!_TeC+05V?DF*`(!#sHVysodT!=Kg01)2mDe<%DWh zqNcpi57waj$`-NA;aSuHW!TC_-8WOXvZs5#W7ut*m@6w?)htJ5`&hZ)QeMG=L>FDq zVuWBiHFKDqKPblvMg@rA`L%NP9n?q;ea|$Q98W7(^Y@e+Mug1x&H1FXbUOjhro&5v z7NeYw$W}LC@myJNJjL%5msx9?zft>iSw_4o@D$fg-E@A#vy6!v7}Uv80GxA_?%42Z z?^dW%ncr)RYuT`_{1{mrP6P+p;@)?o>~vtiBT61s?k-^>-+Y$b)t(&~CU?wHpFi>R z-jX~V;HeWwSMPeaoYfqBD;=D1z>f82Ut09KtF;FgZF;nIQ-3aY7bh{p|Jl4>4l3}; z&|P@Tc{2dc|1rk*T7l40LtwXZj{R_zX*eL7cBlpU<0 zx2&j1?d!Mp_gKsNmAv}FFtkK5{xu5QBDH6?r^Mt%4Ahp213U%b#~dYB3S9(|n!NYK zVgq^}_5aqAXZ6{OGnOF@dIif{9%ftDvsh&~5*Z?ZBnNP~DZyqhLLd6V`RO$qLKKzH z7K^(dB3o(UO!qqOD|;K;edqm({cZkwq;F+grvjTgxp9ZQ@%QUu;e+=jp#g9e`LHP} zXDi^|+RRS>$mYiT1zaf8O{YF`_FRM^0rjhs>t+KO_P3^PF4&nIcPsFS&b6EJ$>y)5 zLAPB`gGZ06^MRn~6E_Me!H)!3*0(4AI`)y#;t?V|mx*&VV2WMT^H{qz z<7gFlK3zm0u-dbQm>PcT<|+a9RbHZ>&WQ7`JZnYFS+sQI>nG(4-HNH=3qKU!Y4&`v zFpti`yM}mU(1ss(Vnf6oBJ&%gP}y8W*L?RkDa8Lf!rM|8de^HUY@C0ajD8(ect_Dj z;u`|~D{W;ej?ek$GUEzQIj80_mkwp{Kl4CdM^74yBKyv^OktgZ)7a8s=%H+9aMzL3 z=*(VSD*vZ(PJZ;tjl=Jy!?}@s?)yrIVwTAjz#GFZhoHfx`G4wQ8^lT zikA30d_t88l#VtF?objeDcBzssJE2>A2#P%nI;4~Vu4Smai0_z&Z}^YR_>FIbr) zm|jZN`{QKgok2TFFg;}~|85rXx#d+Q&-a~HJCXx8Sa~L5m+-LHwr)0S6fdH!<%8>w?Xt4XAm)m@Xse4bv2AYq~+|z?A$J z+_Y)%7rllZBs*#d;;jWG7g%lFhd=Lkr(gZg*A?3y`aofooWK6O8=|V?b_qKb_w?Oj zX_Ms72RFf+qrX-2H889CpGy!cu@(3gu$Q{(6lXY3Lq<+G|2S+9n;5x3V@cmcR2EOPWs^3`v0-MaK$ zfWO+~HNL4zbMQ49rIsE#ifA&hE~w-RP!rnzQ9*p}>C63e!p|mH=IFdhVUi0 zW3-x`1M2jgvMwJg^!NvvIbd*=gt{M708UR4$K@e7mg8%;u)*{u)wacYWJHkM8+aC zTJ7R@T@!4}hiY{3A{H6^ubPE*zc05XNuuD1J@LB*lR4ZgtauRDYJ|kcMwztr7-wwm zUXVNmS{QM|9+rQeSh3Q&dQ*3S<&8-1B9$tu2$l}*TG>WIVKCM9_bhp&l5ar1q*um` zgy;vY^VHW0jt%ty=ejVSv{PW#3HZ0V3kD`d%&hmZp>F?tYYk+9AtdBfNjpc~fnlLD z{2T*DogZ`y$mby*j_r1#_02A@O{2c4ni+7JEEuaVvvuOsVNr>ylTu@vhWU;H5fWS%z2> zg%?7e!%WsH)U|ewQLpw(>)C4F!mIPA=oRLKuRHaNO_ZxxxTg}+hqLn2|FqUlP`L$^6aup3SIcV4B4cI$f*;3WIp@6%%{p#q zJ(8Z*eHWXRB`l5~Z~c-x4&f5-niWhXbw%7NqjsLc<>s)l3$hs^%YUVqK*eZ1tfX!~ zJJ6lC2WYPunI_PSPiEWk>V4LHsy^4g?b^P-D|HtK>iehuN5m}i{Z6cGshlEXWlc8^ z&c)}38}b974X+Q6#i$G1H^WXC%a~0+LtI-n2L3u~+AfoJmn?OX7cs92iD?}mIi*Y3 z_kC6;_BzWgAi}?TEyV4t|AF`v7-nSJ5*RUlzBJg}t^HrtuMo2P#(v-b^E`g7ebCpr zjyfSoFt`;f><#D7$oT!bm;T<(UkR`tM;Jdu&ol9H1t=GDZO3&JPb35Fy=`2-uDRV6 zJ=*}=sVQI|b&$(}LrvX-n#q>jI#An}Bnr3YGj1h-B*vci?{ecO+4W~UtMfBdQy8QK z090U4zcByHXB;3vLV#(~E!kIQYrIMI;m$}fNL?*Ej}VrO{eT+(`3@&4c3goT_Q$ z;+4hG84IqjkKCa~-3seSGScLTd+VV43ua^_Ct4J>BzVP!`BW!P?dJ)ic3~g!djY5u z=Er2S9{rQBvH;}O4A1zj_P^RiFPi1%Cg3d-S4nTORoaWi;i>Fx6+Kwhf#}6-t?gJ| zFFzW$-BWjqGr zC9BcmUCBQ#)J?DO&WWpGKdw45^vKW>}D(L}gc8KPOighH6ZNC3>`j#yGIX2`& ze$*OQt62OAOk-gyQ2uD?l~h-E*80iBK$>vN1%t~?3whvN!RTw?TJqq2xTi*|PVc$3 zoX~R0k9F>PkzhD1M&)(sy_}swnY?Z|Cdc@N3<|>yGa3-}Z>xD*6+@2vC&I#WjyW1$m4q`J=*eaKf6!0cHu>oTBu)G_LYtbdRs=cO>QYW&z6de|qu`~K+h+1(x?3amw!UW-gUf|rD) z0n!M~x2WKdG+EC)DAyIE1`f%5a3{2=j>hLrKSyhHQ(E?oY%rxB;=Ex*)1=g*mKhH? zfEo$&dEUA-MUx7jqM~dV?~N8^F4O$wgh-2Q6&HLw{M4+n?ab|nYcgeb9u;!p?Xyn4 zbwS3-zo4ldfzZdw44mD3lYu!n5W@wKL@q!vsy>@hGcMsQeeNb`(W`u_NIPR!9Tg*r zhde<>UFJtxA7U6)@?!0EWDWs1Fa}MLR1_UlKmXQ6;4kv_5I92N3e>?OTxVe8!{(T`=D53vZ<&u26~tD4UxUN&zhMg@ZUlB~hLmu@OT!fZ zrj@rTA2bUyx+i0yhejs2baWn7wH%+YV6ISKi6N|&%21k^RtKlC>U>&u zCxy>3qlfIRXHy*0Q_{VDd!NO8h6J#%C8#6BmAE|*io=vD@=6XQ`K<&^L)%^s7| zqmG#K2Ml)Mn6b>g!RdN4)Os~fXC&K`uGWu9f1P<`?i|)-0;hs2tUljEqqy$ZVKL_o!chl;55iPD=7ME+)BfiI~Dz! zID_L}EmcKnt_|f*-jYJPu$YlA?_t#D$S|SJ4YspOE1dO_WQG zbIr2LmhhQoA|{}ZBzl^=lP)_z9Pz~Z!6hK&Z;`-HkM|AM*aZ~_U3y`cMtZ(@KyeM- z&aa{y35&%?8d8Un@5nA+aF51kydrUq#>?NNyv-J#hv4v<5H8;$k=|6YE?vL%|2{A1 zL63__?(Dj4t`QqtFjj%-*nTE^I(ZvptOp z+mPrCebg3urk}q3jVOS&5qMDi@pnK6>fa!cke!rkYcXtGqr`Rs!wv7O-o9d}P1B;o zmkF$O7hLvl4X$>&-W&V4TGbt`xW9`lU0Ao~MmFG~J{@aFB0_8fXujLdz>j{2(u`=I3UWMaiZrEtJUg{D_~dM@hxz zVVRT8yR{pQq9(91G19+?M6@5%4yxExAO^@EB;&Xo0lol9g_M#7_*acF9VTx(`+9=z^4BvoE%FQ2fq#sB8tRJAt{;&k}&;C z&q0(lKLJBjen>|VdUY^Cj)FZ6+x*Zu(?-S1onx~Xb@X5>}Pt>pwwxcho9`KBYT_eIpe{p$%hr385m z%+F#9{krnwKDh(gs*F~#d)n>B`uECD2*p)%51jLl_vyu39>s1R9PwXl{uKf|4U^ov zEFwe0wBl>O+&S)|VDT?8(uxL8F2XiU&Rc#G7J%k@cVuG<>_=52grs;tPS?P6)tsMf zj+#*Z4AFFTp-cR1Tj+bz*A0&%4)$Vqy+0;bD#1gp*GF;f#@v--8QS=RUJ1L;8W8Ek z+b`;05Z4t#qHH=X)r zxq*Q+0Rym!t_7YvjtKGz`6?-OKkcVvdY^#E1O?LrnXXzWu0{u<{iFs!C_<;Rd9#5v2aoe|)xBgUEr+x$b*K}mOc$?1hR(+ZC^Xc#TL_Ys^ z55C`P-*O8U+_J9X;|ujS5m*yv=@&2_>T#IDbFUpy7qQD}8b&N_%w5O>-vU^xouI_E zIv)~Za5x%NSqlD(!DYDAR!ApGBJ5MU7};ZQy6@AmH32_p@s^(LaMaD&a8&KUgw zl$88mu%zrJ>msd?J-8-zS)J#t{n?*;ss2SI$_}M@_BCGpJ0^@(ZuJ6`wrGQWSwUf0 z+nRDT-PaCKR$DbIl1U`n{oG z8_KUhS#LF8I5^Pc#ur*gM~yPjYrR?urRCnr4pSd3db+RsPbE}RfCta1i1Qa6qj;rc z1aMkiS7+yb@QQKG5u`0X5)zaDjwkde)LfyiO_(6bqS#8t*|_(i06Fv5DHFrg`g-;7 zD<}iEr$(%S0L3+luuid0@K_6oUZFR4I}v(p;^tr;6i2_(g-aIqu_#a9B87HrNp$Od zmJWeW*5&PM?9=csYf z+E|8QOJpL{#SgYe^Rm%UG!R-vx2$?ncW-NVN^S4tT5=Epu@J`mhVD@eEiuI!>Zc3> z`p@6eTaO$neyj*Oi}1SooAP4x)^T8Of;Ecj;$dQ&;NefxL(oc;TjiTLhQ^{x*^<#W z*V4h8_jit6D!WoZP7+U@5<;@78!qScUwqXhvurl zCys@iQ&(#~CmHd7g`TWifu7A-<6GOZ`9g7bw@_^&(Am&kqDalCK7l}FQT73iwC z!fW`vpZaXl*FSo#XUm*%ex)I|7xfy89VHifD4#S};vgJq(zsD>wJH!{(umEGHih*7 zgepleCM<3VrstzrHp~AglF01#KI0WZ7ikSfJu4(~7&TqH&FTp$G#5V4N65MJ+dGk$ zTtoMsx*WR!wYMInV%>AqND{(^ah3Q)87cC4BzIm3i7A>q3I3WXb3vw~jfu3Th)b_W zPxfniEX9Y)*?CCEKSooyhTZ&Q+r;dg+zmu?pPjQ&Flc*jGCM9KPiI{ph-L~#vt8N_ zdk@nGA8cJ$K|Ii84~d;FJYiVM$|3PBk_Or}+mzAZH)xI=K4qQ%|a2@Zwg?(XhxLBHJ3JI41v87DbspDk<6Ij_0b#rcpKM>aX# z+Z(!0tJ#|MMKLL~540j@notFA8tGbUUN*4^y1OMbDyiEbv3J^{r*1*QhxIwl0#YhC zAH!Ot#LlxK*At(E;X?@BmW&W?)=>T1c|&HauMj`0-WgFc7lH^0HVZ!9fPza+dE9S> zIW%)qaToQt`8E?i9gd<05D(_M_gB`@PTVHXUP&g+i|P}Q9HL9vgM*%aKyp(}W=EA? ziWkxB__#XiTQD?2E<;TYc_*f*-21oPqLkfY$3E^jP-(5qob22HTisWU6Z|_ZbZ(z} z0M+k4*I=NaiMR;7RBiR-6z{z}s@=-bS9aM3h0oC>uLZ!vCZL?*248e%!Pqst|3L}& z`AFpoSYpOrRN_|}hon!L{&HS;5|x*qgow?jq4|Km{Z{=3%~K$OH$1NC5H_e{s`2w= zItg43P^_gJVfi`%&Ub^m6&V;=g}HB?PWySeUzB_ZUl*!@2!*2$V9BK`m>8 zO{AJSkv&prxW5X^?;8KkYU`_y1aS-dRqZeqwK6#>T9Bd~x=4Q&ro@b`jrt+~ zRb=;BeCCoa-Y&c^kR7`$1lmJzv!O`ih<;RV_CD2fmGCRQ3b)70bad01_m=IUH)6c) z(A!)R8_oyY1B5k{Y-)rbnlVwxHwKBDI9>SmJE|Cj5~swqm04M6qNy5ruIYyGcxbq} zCJiw!p2KN_(Ue@O@f38Id(Q8fc~`8b#76#CDU7OfPXP97BRaQnokuvo7yha3ci0?<|sQa%+c0nbA$%Is%=YW0ODd|Jq`x%UUnjdIDLgJUQ| z89FJjG7&hXnw2sz1s_E+BOlq?ZI-tuPof|S;OG8!A!{fgJj4T`+JS2dB;o~QYjhD+*&ywJzd90vZ7&}odP#@O#kBWa*sL7}%J zF(Cr!WE2x~@Hy1hJ0rgP5O>0v>B=GmIzlb{IUVdkf3KJp^IOd?7nZg_8Cf43XklD} z6lP^!@R1PvII`mIL)&w_vmRHDWF+Y}D&z{fQOakld7-|Hg|1xGaK)&A3xcaS?)Nol zFSDR6DB3FmuRsyuDQ`kxV>kZt3o=pvf%Ge5go^OTD|8c)9)p`xrLOhKqw^2rlIh`w z2Da6*T+C`~ubg(lh|d%-(1`{*aRj@7syBzvKc(@lbyx)s2u@LAK$u#8_4D~h^bRYm zk8;Kszx;O;&&OUg_ktS2ARkjg{L2`=HJT$8K}ubv9zL;KwYtFAXvIIb;W|2bQ2(`4Me5=mHws{1Z(&Tmn9DEmE%Ea2y_>qB zTlS1n;?EpsjWyLZNdAqmTgTi^3pn*elr{6QL z0r4+R3A?L2qE9^uJ9)NOSV1^hd=_0c63f^_qe4^X=MwXlh|83YG^hguz(5s>ou;dVE)nbr)}Vp!O8u8-;*U= zgr$$Dmd~(`a6fAI;dqk9aXVsUJu+fPw=CoR=y4Jf2#9>4xiy-QyC*2<^Iq|d7g(PO zPwF`Mlb7Ry1!wx>)BF!~@ScEeWY5}ZM3z&Yxfp1t4!4qpNya|rW5f`@G39l_ad?CG zinhL|k8C2&RGzib7N8)Gpnzk-!9RL->e>@1d6np+ zzCHz(X4bclC^4sRI{~7mD)OY}fxna55pXrw#eW5b(ueF!p`Hx1gR$dp+2FC6UfSy! za}SMkV9?2vo9y>k+R(XhnbnkU)GGIiEm>M0nkXY6n@jJaS;zlH*hRh7h^Cx){pr)4 zeIXE|Lzqwq6zZQ?1$%S-HzM>Cp^FqDujwMLvzNmEPIWw^E&o;vZ(Wkz)fh|$mZ{F|4ManFxYg;G&@v?%<=1b$>dzPC<74E{{`{AQe zeD#U{U*}w@_0khQg_uC7z&7UhCo+bqEgX#x(>cBBfZZv4EvS!ET1idF88dbu7iORzy|4MqcV2}?T$t~J_;y}_ zJ6I1xA9p{1W}^2_xq6L0cipSy<^l{vj-A&$GAn@$+Auc{Sk+ z=<+~~#DA_xqKq=$0frBr6B_c z%Xrk@YCFLsihtYZb%86>1^VH8>^&mq7KqoYq6?)k7ee1JsEo%xzi0(H^W`E(GgyKP z?=T68GOCpDV1yYmaXQV*3yZEOW9|RKaEmDP!hS%{N;q+VMrBiIwB>o>_vmW)2P3ne ztTPn8T;(E%R#ss^lOL>6wVG=RZLSP;p7XHbZ+%#o?rm_A`7HaJq_R`8T1{sx0iD;= z@$(7O4@Ms$P_ZN5NE*nPhXoUa?mnH1t|Ue@i{@Av2GjR|Pe8SWgOhfkm0`e5V(Qa0 zd2O~7%iI?|Y5I%MR9epxj&B7S)3Wr`O__lFVBmMnikx`kZn1yBL8W)?|LWRYV1qGe ze)WG(>>J|zi+Y%Ax2agvqM%r0ZPm%&OyamA%P5-d?_x%%yd+i$iI`1IXXRVx@Pscs8jqVIc(kvayw#1hpwYY_8>r-I@^wTeoPl} zL=&)LRqC0LkcgKl2G>3X?Mbu%`@jJI-u1bh=iqM=)&IuhcZ99fK6y}t-_ZZR1Fl|k z{&o8*M{MMW<-77+6N)mfLE1^(xv^knK+lTNWFO(GSlw_&v8pYHc9X8h^ew(huK1P7U!UXs->?H9MsVK zk6;=-3a)pxHn?`c}?ww`Uejp{iM?y4H_|UargDE{BiWx(mXw&9CHI3*U zhcrF%UE0?}6zLhV!+0U0lu%okMm?L{3!c6?)mEIpHG(ZZVS3q9l;)0FiTR^gI0=HkJS zxXCX@^r@Zcl*yo?yT_U*OH=;zEJy|C;_FtMBz)v`#>mlHq+U^rZDIE z#!Ly$>&AniSkb<#U>XaL;R1Shf<}O;0GqrR zg1x9734EcU_+8CW_K%CO8oM@E z_3Da$>aiGG!a5t742c69d*;$UG;^+%7g$%Tjpf%fZG(F@DE;35f~rJr^uG#4zIkm3 zeWs?A--_u|GhsFDH?;_$AOHNCSi_JNvXmDd8OjR_-LB9&x1lPLXnkV)2O7fj2$@v( zyk^@bs(`>QliIRKHVn#bc&+*qDa+rn&?{#u-Rw{hEuzpr(!BoJW9{S=qidIGqLf!& z`@Wp?{EVN~0K2y(Uc%p#G~64$b`eHt{oNzEBIdF|J(5u$1Onuio=770*t4xjip=OXG1Pr2 zcG>bKFBi<$*f_iUMj}_(eA1ega!zp~P%41#|4w6}egQ0IU)rGrukN}+IYXSqf35$U zG-pxx;jjGQk-m6&tm3$Op)R6+F!EfmNVyZQ#aB&+zdMqxQIz6cW$m|-oq?$$iX9*3 z=|Gc7Fh!u(B8yP}v~S0G$L4vJa<71Yz9U;);knH7Rb+DwH^KjDP5(9{evhMFq=Eji zSGnJChN+T9Fq+Dy?bEm_`~GT1L3oj<`M@`(^4bF&^mxvEmgCA_hG_*B0HN4s}wR<=Hvg0#ZoQjF;_v#>R+c9Je)LrmKb!w{|Be_ z_v@v8aA9~NGHuPoRMO9z9WU~+W1HE~g<^A#-Fq5HEdlp(&B{msJmG`!C+_M&kXQL@ z0LgzdZs($P=Go8R`aBM0UUT1Lb{UuwvV9D`;LGB^WZL+B75Q&e91`)2#Ml8jm_{}D z+d58BT08E+2WK|nJnF&!tBDyH4~vb(NcnFluIdll*jgEuVEy2G9{!Y$kBM4FmcR}5 zYYfX>9^2Sh>qEYZBwhc$SLIEv)kXTIxMe9&Gy9#~{=omo2|=i?ZI%OB4YaSxbPJ2u{B&}RX)}mm)isyK^gGAiv3;zmT>E6J*&|;0v5zSvt zTzV07?V5TaWb=87G+j=erVGC%12a+E>dC$H4`^P)ct6_0~r; zTnSftgynS-HrSSgO_BpyE(q=Kxm9tiU3#HQA@ibq@U&B_O_ARG|J7os+SwIO!WcQN zlk&pTvKNXs{0!+1@0V)~U&o=DwlGeZrV7D%`D0UTl5K!BBm*t&Zs)1yJC>p(aZf#! z{SF#<5ULxc=JLF2e+U3kXTWG19B)FduR*}o(5GIF7sf<@%q=DjFd<2=KjRj0H7}a% zCxw6QmzFt|&Rza=+@FUn(!eSf)kW=j)=O_9HF97CZ&mm(@ES*7qtOF;sE`}xE4OBkHf{s3mE_L{!W zBKxRNb=}GT`Y?M2GcQ5&cX@U>(4OUoGW{j72;A_S`bz(qH4XU`8!aToWx zZ5y;CgQ0JyvmQBZCtHB;^h=R@;y7QqWu6zy`VV?b2VVWYy5lc~99nLbTbs!1)#SZ{ z#>$YDrLZ&tNP*5XWDT1!938_-G^sqUeslvTW5nj@58-Rm?`;l)PcGk zeRLtuaPy7P7r!{N_>wwlK)~oRmRRn#Dhr`sM8y0$VR*-Rq)($%R4@-omdxbL45{#eyxYDzZP||M=U_ zWOSRS_8DjN=zRD(KdB|92`~|{+ZKjdUxi2+=}5fZDb*}?^hnNln*Jw|0aRZiKBDEf z06M4!Fz0>o6XEA(qGr46m|6a-$J4j7bLrT_?SRjq=wCR8j4W9q9XzxBo@`E)$APTa`x z>EKzd^oO4yOspoVZE_Ds-b>%lHA>qC%uZoVGyL7N1_Qn29f zOy`va=T|h(s$i2GXZhu6fZufWOs=^`zlr0mgP?E5b1W8&c-g|}nfst=!M6mgjqEmJEjGBe3C)Y?! z3D59Nze0L$IG?jt3*!V@oDEq|y+)Q724R;e;(xT7=lfGX4S*-HtCF4nkw%uHw9;bF zN?H>N;h8TVtF}8ItJbxx`wZ*=3>1@cx!uWRoGsFyVOq2hv5!j?T*fAX%9P2hT7?6# z9GO@D!_vk(ckKi@H#6# z0=qoB8R}c(vVJ?%*oqDG#awTA)P$0LCNtCy+ zx`S>Kh%-D_!K|81y8L=@2p*dy??3W579ZcDI8iCxV)}rnyUSuO!O!}mPvT6M?nrFs zg5J8qP04pp1$(8)vrZ~rCfVHRA;Dn@v zx#!RG`-q~5L}p2!ME8>8yAJUAA}kBpcOL>p!7aYUR@f6C@jUasX_vj%(X1oq(#TgV z-6SHel@Sru4KG*`5opd}!%K*>_c$|Ifqy)lkU}8`knSAYhqSlgY!P4hN99`Z6RoM` z2Gsm(SVgzbAh74Gnur~LQM{{7+*4nr>B)2V$Q$%xqMEv%6-Mrc4hQIBP^kX$y7#ob zzhZFU6!=ky=X9wJo(Bk_x_k*(*=(lsB_bRZDd1I{@JF3!HCze~yNMz~ZLy8}&4AN| zf(VhdV|-ZQWs?nfc-wlJ+w7oQ6i6ap0_G$RtqhP?*E0eo|%+0NJBtK6koj|ZkO zn8yPF2N&|K9I^-GL-t*$sGg5Q8zBeaDr6+%)ytPGhA2?(*N;co3VV$fd5c=PR@VuN zlgxfSy9^%mLKq+NN^5<7Y0HP!T6ml+YC<8FJ{TDN%?1ZI8~y;n>(H8myKI4x0uu(B zuQz*?nm8AVO&(Y4L8qq*_ZM>D)bS0_Q76QqIAb-lNP-_(%7@b9xJm7EuR#>N>*)$Y z^;n8$wzA2Aiq^b%E>yxLl_$VL5Pg#`A6K2Iwb#O$Mtbri73&Y7jAguIug{+3v8{6M zs7iY_(T%%Eq3R~ay8eLmeD_Z0fCbA5$DdpF4N2}M)g5iXH<#j85>K=J8Sh8+5TntJ<%$lt{b^Z?TjJ&Ni5ohZ*$HH zL8(k>27Xtcr_*|S2K6zopqQ=ZHfkV*g}5s~Xqy<;)s28Hjt<;XhEpS;xIl%f;}L^3 zfRbV;@8V`1Q}G(xIl1V4uAT;NKrE@Oc;NsvX@On5j%Pi+loqmdC~ ziX65O+&a~fiK4MFXD_Cdc`97Z;~w2WsUsiPj^PWwsaOyH#)J&cS^lIfid;4h8~jb)~Lljmj*jXKtB+Nt<$S3w(qUZz@=+Np*e_O zHUr1-4bw|k5o{_AwIVT_syTXpMsV2SYLK0nTf=L(PW)$?|9szv1C?V_&gx`833C$X zMB@}@pj5o z`k-P_w%BrPc^>|?swS?86Jpt@O_n2Ga&)dkcJlODAz4IL+t%gP=% zJ0p?t$`QBr+6MO{N5E{I$rQW`>ZI@|(Wt3^PHk-U?Iw(!N+Z-F3Lk~hekwU}kna4* zt=UI*Wk||KRA%B)0P+yx1K|?gM`vFjcV11hG@XW@tB82J{q00lI6`P4EN+O}9U2B3 zjv3Fg*Vq1&SF?bq;yAdZSY_X^-aQ2hEL)xF(sMlDBGAzrBt;&`Ta=LS0=Dyr{9P{h zBg&b7nwZR1ZklAEU-P!pUd2;Lny!?WD9s2%^4+iJgiI)PAus0lS$bU6*bMsf-qNH7nD5@ZIY~lvHZs-mgDTlT{>ymC zM_(%xT$KJjN~>zTV@h36y~-(54P7vm{rqesI>mjs5YPE6E}TAFv2|)E<=V`e6a~$ zo7=|)ftcJMevatGTffBRnWF3X@YrZ}si_ODt!YX&i1FPtU_;$>^_{Hq*`>&x{%fOP zGI^yy5h1b^jtLlg;!bb7t3=5`ESdNQrUE6{_#|*c=g{AxKhdVmRi$Lii$veXW3iCw z6Q}n!!DCK`Lhs6WgdLI@nrdL7rwvxAPzi76(Z~la09LiU2qH(jJ451J2p=!SZQtpV z4KE8b9o!KfndZ}dQEsmT-+jyR_%Y2#v;v6@rS6OA$uA!!@eN;NwEz#47Bo5jeu^L_41N4)vy=2sCk@rW^~M6&g7srdQRbX;5KYV15wuagMG%YZWt zOUhIK72+24dc)7<86g@+^7SXkt^wyhBIvfn@+7?DlVWC1$A8!K>g!3w82XEi&P^GK zD0}3a1%YVzbrT~BK7@Sh6LOAO^6}<%iD(n2*s|GYW3|m;s)E4YF;}~`dQrmXSXz5C zm4v01D|zs)N+q=_6Xjgc{)AmMs-;?fhzxp>&$y1WBN+PotV9DA1@YaJfT5uv~$f z{Mcl`Bg`A+5i1cGO=z~Ghopj>{2W_{@)4c!nxN6OI?gLgh9vUFp?Du0sSpB{2uT5a zRphFAdFKue8YRud$KeyD?((**fWUhZMHY0;VmcU|bR);w<`+T80O5K}1NUufK5&U! zYLu5w?;Vf5$QbxXa*|fYWgp4k z_#dUUmXlyc;t7`PQ^5+{!U61d{=@gnKF5DBFPDYM7dg>wRo3@h_D!IUycl7q-x>Am zjP<{#yI{nLLRJx|;X(C3mRL?{4eC%&E7ceNVn?qt?{f!h50&Xtj`sCO920+U-<4pm zq>n$CXiA(!(-G{CL&I#plN!hE-AXYkJ&kDpUU4Hg-Ktzs3ZyvofiStB1^aT{Pb7N< zx_n4gwyj9*jfal8L**^{`%fa|+i<^fO6moXf*pNH(y(zC#qH-_cJNW>^%avXSqvz=I9$7R}9yS-ct}~Lv20V=?MEg0S9nWyJ z|9MYtt;))OX7{9wdwn8bNqq#%9KQI3h4XBU-TTcNIw&KOvmf6{o>eh*826>Gxi^w= zl6VN*bRaaUk-zJxgL9I`bKLhD8Ps681E(Yfn@$|u=Zx-95m6goshcm`>vM-nRuDdUx!a9bn%$qP1p(iYSp2NWT@V*0e{c*1jceoyyTa$YX zYMjUCOCFhvygRv1)V?D_GlTxcxGe^cuYZhA{5^iX1I=B%@Gr`<+Rn#mG-=#rfGD0T zcBT#wDpjG_<(>Z!8+(FE)jrYi#JSze_3s`@c-+8m_%*ffJos6Cr-jw0PK9wPCady1 z8cj~}h7V{cBx3kEwX6q|#3w3x?RDhM6sEsS70J-=?*;9;Z+Au;;_Io;6R^jK{r=}R z&N?#{Y!<0xoaMe9FRi2y}RrsnR4o{1TPf{MX% z+tEJ|q-9`p?|_PqZAg{<+TLHbOj`xYK%%M06a&V;)5mHGK#R`K3Fnvvx7IMWJ_x@ zay>*?vitMvp`*J)kDVB;=YJMRyPbNM$KTWE7|3UefgaQ>+fFlg3wL*s>RNs_ z2+}Pn5~OV&Nmpg17ZY~R4A9ri@1Te(lmW1AaEkppXpfqiE;u5WdmS!MdnNrx((+m@ z^bi6C1_pfESx7*yq}r-K>rreJnGnH(qS&vNC%WLj*roi+j(wai6l*??fZ}8cs!?Kr znR(EJmI=Eyx{qSe&p9kO`C6T_7~R!sX>7Pb!(E1yJ+sRK$v!Q@d{jw$DZrU)?K}=c z_(sN%wPO3PBb9~5_JN0Fr?cc)`pEG%x;1q2nn1=!0HW-kf0uxCSGo2n{g=j3 z;_Z%flje##2zb*v%4%DwJ}|R3D3g1@4HP-I4KWq&(P6RV+}J%QQoEK=={^2JH!Af$ z%b}v9ep}akikE)Jz-;ikWZwAJUh|s$SuYG+*sib=1TH0$w}S3vm+9H6xkWP-j{Mbi zsX>wX>)%iMWjq>6ua`rHBjl}T))MFY>I?VoqVEgsdS2e5EgzoDzL_63M_=c)^~FIz z1wCMIxs}8kO~Ny=9XMaOILYTGFxTvQnDNL#j^<0271 z+c(SxkM)hNjTh^bg6Lbys>eaKwb1xy>MBt*=QpXx=Ndhh>id?cUh`LDr?V&ei+D_L)2E?l=NZLJy zTq^HAM&ABBZnroK+$j8h;+i~NVkp;@FTG-swmWt4i_P$A@ZJ|D!iRw%DqybocTV3> zixXBp$r|)y_@aSO#E1pL2mz3%$mH?NS zmod{ZRl@mcyeUIBH#GLGy&iiFLRriWV{N|8&7;&g#k5<*gLD#ZAw111TKqcx$+%)k zrjb{BDN(N1T_TNE=BuRoK;X^F^7osH_R7s9Fl>hcQnQ*TlLcw+dGh*_>RNIUgWB_Q z+u5&d+7ZpgDyKU+=f17blv+(Qe~)nggS?xGuN5Lk$3MhQlx!N`Y&^E+SrsD{(DOQa z5k9QNJuAJ^Uf6=Y^`2?#RE))Qn-JGOvU;aivN@ljz?4l&KsH+%5$BPIuwcbWsn{^% zf*Ur`r$CBysQQkx5XI{;8DaADs=J6;KwjVtF!tD6(k`9BzBL~{g4HvpgA;SZbKjFw z3HbP7AFWlFF|#vbcvzda{l8#m%d+-L?3-FBX~DBt-(6=$>*Kd;KBOz z&1g0-nexJp1bwAd@26smi_&L{JJZ*t(xeXe$`-jJ&VKZVj!VpcYlWO5>-ZG{5`_J~ zh!M+UdW^qHx}N{I$e-keMp!(|+%jFD4b$6sZC=!m`sJn&>6jJJpI2wU>`)c?l0-3c zy}n=@Jjj{7M2puevN}|1==dHwFr7@yQ#>GW3)dB} zBgdW$CA+T1zBOQVoH-+3X^{af&-3w&6JQYgD;qK2ypLvB5c^ScYY})?c^s{!HaiFd zHbfkg&7Kp-2YA}b!`kEw%_JLz(cGYnLUl} zs6kUHBCY*DnDuZEwMm>H5w4`@j4LHY#hItGymb9IYqt_f&XyPzBJuuX)Y{Egs@R@l zl=~#G!(iUo?LxamK>KW=Nb^g4WDZsdau?7e)zzO^2dtH3b?nu88tghgDHqkriP?eX z5ts0!jxKQ%OQ;b0ESJ(iwg&I_Re)>Rf|V+jf9YmymJk;+bMmWjfgLP;682lE(P0cN zxiO0L29iIuNiZJ4t9PU{>2k(k{T8@@q$#w)*vglFIWMUb)xrg=u9VpDP-$+P65KZO z5*JsQu;5)DBTm6y%@@HVnw>%cIif3Y)`|)^k?S^%&t}?vt1Ex?eG1f3Wq}qfEEX$- zrA9C3O<9xJgJ-ODbq38|EgxD=`_6>~Hd`v&PZ5nT9}luOFsjeF!j{hV>12X!xTijUeg03E=t+)-7BY*uL*(mx|OECUd4mb$nh`o3#60`ai zTJt;ygwEV()k0o6q$@uLY?5l5ukoou`3(WO&F{oIU5jN&9R|@9#`el^2cb>NMU749 zEp7K&lH4woU?dKHn7ykL*5g8JYr5 z&UD`3h_{!`C$gLlH*F`KyH%pnZyHTK=$CE#-PPE6wk+mZAG7bJGK@CU7cxDfxF2HT zE=sMN)9e=-fTUjL;P)*sXUW4v#YrgS6^6fX-{ax~XD3md67W5DN~x>%`z&qOg@DV? z&0|WzC%}DA=i!ml${UlD6AGsb_0zr+osKPngoOEetg?KzQR{51^^hlg{D~?Dls*F3 z$ORucn4DJxK!X{w95S9fYBJ%KI?eGvo@f_$<$I^LPE#(uJzVY(jUtvuB79L#acJkL zf{c|Ls4It#lz-?3a-xrppCKxbBtEq9ynKm5o1JxI_EB5|sKnX7e3=IzGQqaqd4}P_ z9@|j2*XkVxHj`gU#Qh=yHF zv7kc4>%PO{xBvr};*l4#|AAq!QOph(wh->c;{sL=su+C(7f$gADH~_dZ^+FK+syNc z_VM2SB;9D_z7ZnRoF)T69FEW7b_fR54`!;{Mygz_KPA@Y5!#LURANH^ZraN%*Pb4t zp!e<7RM=2%@5DWzknKcDA@`*loMTgcmJ*P??u5=s{7S3WzunT!X3TY}#h{@FK{W>8 z@;Z&wo%W56vM=67b|E6=15OveB5thR7SFf;63dJkS^LUNYaYxGN7lny)O?Wg6{4cH zjvp_G_#ywu8g%q{89MG@fwzxn?#nb9^Pp{)c#yRUqvZWC^4H)NK7G)GXG9a@*+1iL zvU!&0PRPczQOj3<(FeQzg&A4cByr$3Z_%cXk{-5~nfzJGlh_K{;Ss^I0idez_)#mc z7ksvILMs9Y)H0giWeCi;6BSo3YETJ-B>dZZQymp9c!!M%zl9X@{7rBJkt}<;nT~uw zk>RnoEuBj_Unut1W_@Xe%}}hLaQ?WwZ0L-|C&Vl<13x}iv~$(k_*b_NqhUfANdT$@mk?9rr}vjj@6wq z^e0&_MV21eNsTZNtECSK1$JUk+s59aU)%J^lga5=Shg0(@dvlp_ZEPTuhSAPj^ojYV+^ulW^dnh3_nG8^u%O!Apsebq^rq;fjIhgTVhb(&pJw`Y z_~1R}{X1}p+FPHHTFqNAuX~u%VqIA~-29T>5cW~ojyWdYMx{*lO2AS}I{}io&#^tI zbk5MO4dh%@PzET}&@%Ha#NLN{qQa#X#a{P>Z+MVLHbp5>F#bU;!q@*sW9Z6>q8MJ6 zD5xSdWz%o8!NvGUb_?pL7Jk+E?s0dGJ$6~A1xhmssWw`Q%&ZpCdMTR5ZSKO*98CDF zoI#ZjMNvDlPPd`vv*mMK1Qv@f*FzBYz;&j}JjYMdx5!5EK2`%r&3C0ij}6ZZ!t9w! z{B0-kQx0xS-ql#pQl0*Pym~Do>fK`Yg#|{`><fye5e(bXK~UxfijKeh7OMSZ1ZCzJzeoKW zm4RZc8PEPMC`E2ao z{0_0}5wYv+OVdbvWNbFUN!Dw%HE|aHc3x)rHDtc$7ofaZ4I}DBa;jY?2g&=@?z=UY z_?^KU6nC!ale!Ogc&V{TCu)WG_}qDw4Xu>FrtZ_l6H+LHNw{SHu3C7o%1|R+`BAL5 zf)l|I+I?kr7jBaT$uB1CEMm*>I>N4zx&N@Bl%vpz4YMhsQgNA}wi10>!9t8W2!Qb%-0B=}2>J8(YiQM_$j%HC#3-HLNu{JV)M()s9k3 z|2<8fBf}+&-Tp-Pv~%)K-Yhgn=iU758+*AWNagxvr?<+#+dc|ath`8jZ*PkGD&NSS zhFfqKo6q(f``~@`@%02Q2_eJlDPoWqWxc)!8jmPnX*3iQ+f2{;aOS$htR@ z4(U2)f5l_oi=Or3&K$-fq{}dS%bauc9(7@6Wf`I=t9!`|r+F$qu*=ay&d#Gg#uv_X zq+$UM{Z$FDu5OLr?q7c9RVJ2itj$%#{|61#>8O>h08wXPx2D=3JvYJg9(#{AeTSX> zh#32Gzqj!u#`jF!Yq ztzbG^2tF<@Z^_Rxc?aNUBo8u*t!`f5E1MbO80B2}yC|S~a4CEcIoAc`F(s*g%zE4* zToxhd!WY!J5d+C&@FUAm^7zu9^SkDgig$V#Oy>*{}+ zGh3~Q^!KOOaZ==mNH|8O{9rS4wqdR50L^YCpfK28P%av?;+9n`=IJWfHZSD&qSh$Z z;~R4eUUKh!ikWPSc}Fni7eE&>ATQ=y^B1@em3h3pNY4~U&v!ov5g-4(^O$rp(bv@! z&BF}ddH0(08hXD`4HHaW!eF{`I)dA6<(nY*x^xq50{|L?cx=yB)qXX6VHg)Hq zK}lG41;)n89z7l+9Jd$B&nZ}u*;WjUjatOdGh1|lw^P_Y3O?t0wst8m-$@(Byq?3V zSyym7DTbL)haQukubrc6?T~TTjhQ%2B-Eb-Xe7sIC=FAMw{)6JYy^M#whun!T=)O# ze@`{7*yd44>hg`Rf%A+#s?N;a&Ip0jUbK99)2)n(N_PkOw72uwWeh!>UCpm~HD{lD zK|&kMO;`|BP7*&*?ge2V1hVTm!e?H_VXLefESUF=HK`JtRE*~NX9}d5RdEZPO1bvi zrm_7*ZK`0;+M=-aBQ}{>IDl!TZ301H8b<%KOgpjt8rgG+GEy;YD=6%7jk1b!lT<4 z`f?0osZtK~+zZ7r9U$j4cAw@0?BLkP(bd{d`hNiYygt4T2B_X0aAD;E{68sUDVj5G ze6w^OcyApy*ai`0QN?-~#%DBV(TRu=?iSK2(w=wrZByF8Zt&4mn7O{F76R2&P~@Xc zn}61Z7huzIJ%+EgB|oI4Rgf7y`=a-AKzC5m$v^JWrWnPUKBX*WyzHIRH+xjU0e2Sk zoaIi+37^Wu!PbkyXjudbcaQuH$o)65b0>)G%#d&QN7DAVl09)1j?%Ik*2^5Vqgt^T zyTXJW-}Cscw&_r-_NIt(QscKyzJ-H9vY6!d{Dx*>%2fc1`*P%QGtXY%22cU6!Z-7vQFrbcq#!+5oz*ry-J$l6Qjei+tHOdX)xQLAFaNqpOkQ> z=m}%Es`HBoP_+evU-mgHPd7z7la;P=dO!*l^nK_XtS(nbcnG}$g!iMymzmwrlow82{#7brTKRdmm%Q{>} zzqC1&H;g=bA(8kd?g5HCZw_OAy}E_T?z7E+bvAUko{5goqaolRE~;eCC-zcG!Ol0q ze#d;bm})zvVhMb1S)KdJ+>QGU(Y6gVc0;$O@TVT#GJ)2)Kbl+kWhWBjvR1HcxFR4> z2i^7V|Euh)-=h4UK$nt~T^gj3mK2bNrMp3pQa~h>?rupzNr9zPK)RNc?iT4-x?!m$ z7P$NQ-sk=S_x*L^nRm{dIdkUBIrA@3_zUqshqPDXWYiiwu7*99#CXw zcKcz~e#*dO0AM)5^h*j+wCRUarAgS z{$3AflYi0H**v_t`1-9 z4tLTH6B_C%+-)d%8yN%(e?P<*gJD-Q()TeFAh>rSXbGnm)Xh={H zhlG}tQoKTf!N63(vZ+~|;jGXpK714o|4;`@D-Y7hW3ZjIqW)J|Og{M<-eY}QC+&G2 zeSWh4*V0JP|HIDc>k}3KGvc+;Frk7IAR$TGPGur zePKmL*^BVfLtJHEO|kNJ>4d)BT2jUG5c+35u2FY{^S?>pog`FFlU6i4z8l-K}FqoT@drO?L1N{E}dGt*`ag z@#lA9JNp*l)Ox{PFzr&L>*)sNOU+~yf2g`P=GaRh>v&-FrshR;Nmkr%)p>LJ%0oES zM3!=b8(DO!IL}>h3^r})uv`DMc;25;n;$v2!XtK!>*HGg>O`ECEA`!t_Md_q%g~4k zjvS;&Ulw&5x8d(k-MI_f#~r)H_P%;fGh8tU7f#V$}hX8gYa=6Zb_Ve-{eq^h9oaPP@g=#2;P8{}Ee;>w(l( zYZ#m%1DR}U^oPkf987Cnb3D+EH*>-hOn0vb((v`4ci!;3`eS0n(mO$|xrF;$=3!2p zR`2wWZa1%)IPi)r*K*`0o+Re4@`l?TT;HY<{}c1ieSCZ$7?}tN>X2Mnx$M<5@$a+| z7Sh|64MkATh>i^>eNwU!Zr?SdH|%V5cv@(eN8U85$t`>1F$Z)&%t@85oo6i z_)FFB$mqnHC+OIO%ugq|n;bRD2{RRB@|McG$L(}N*a+HdUK}p&rs0uIj?04~ZI}2e z!?`QJ@E{-SZ< zZ9iFGLMcyROqEr5SGY9udPIBw+V{!{>Tggf89nK4c3_Gfz$538wobW^R- z#cCzSg7>1~T-J_BSassGGxE)exk?qCe_Q^{jBQjbyxY_>eUpQJM&V)e%q zER_j<3{_n53s)}ara8H+Auf@APd^PSkWH<5-_A|I-|wxKl2mWn5Gy6JNbUOGiIAGW z1NxSnH^5cdD*8pv1eNWi2ahQ|r>NLpq&t?)x`_yuaY;4wrpiYl2mCnYdVoworf`Ls zB6s2T2Aet}azV10?K=IE&J@>keBomMchef7diV}^xEEMRLzWgbl0V{G4t^_OZnO69 z9nWPKhgBitSyz3+S_*M2l}bU^!)sq!DNQXy&t2PG?8Xz=$Izx%!{quur_Y$!; z2M?qXqK`9nH)C<}EK`tD_MLGfXe%B{vf{Rb{QGUUt9EaU-r5mP-mKsKvEy1kNIp+? z3uwM&)gt;KSp4Q#;n%&J_m}TdcOh$N$R6iUp#dWxd zJl13|C{QltUK<67zs^x3efw{KROGF6#q=Di=LTl{snCV+NPZ&U&qw3bMjgK;=qYsU z*Xzp~i7@Ld9pBrr$o++YrWv>Ulft|yL-yTz&C6!%qk-h-ZZ4J=h{|@wvF)FvvKp?e z+jpj@e!h`*&>M?x_n@2g{_%;SoTwm6svg7VDa3hHji%HtAwPZPzrU=?@3&?@S|3-k z{W!;JK%2UsQ5onf3cd?TsPb*Mg81eXo5*dMbP@F-nU*r0)WrGswm1!Dae55$bs4uY zIm#6Wa|qgmUu^y_U*@1%E@y#C#u`@$SGJn9mRxvzfSU(3c?fk1q3lWx`c8 z1%A$v3GeOu1pCZ3!M(Dpe(K5u4?91h`3GEkHl5gexrj4EB1iw5uFV~_5Mzn)`K$1D z2q#n9K{L{vr~YY#cdozxs|l0msEZ2wFb&0A(`Zf2m?5{rTxtRhZJ{AhN}d`!P*uEW zNkHQ5!nVs(>aSS3>@&}4v0}@^@3BQQJraj6E7nIcB_!OgV?-Fps;22n9Gm)dU8^H4JUJX>I^;4sBsd>Rr`A-^N<*oLk zTb+Ju)HKih^8M}&V(e8MV_*hvVEoqCz3nid_S0n6C9*@@_6ZgUeKI7MIPlpF}XFAZ5=qAJ@Oj=SU z($g{b2-fbnI?OMS{IW2zPt`5(uK=Z00CbajY0*l>4(Yo-pjx6MV$nk(5kl!X0`_YO^1tb$hKS36{mU%Hl1ju1|bHx;f^ zbtE;O++)9Zaa=Mrbg;5#mZlU#mob`FA?&%(lRq#X*?fdY!dSg@5q8*7f~TL@zqiM} z;=EaV)@@dJcjw=35D=hh9xJ<5s5Kg(n_Wn*r_9Wc-0rHe=A|fw;OqY!{2?wc%pOWP zf)hQv@vH`vkt}O_lN=U%D4OH z`CmgGBL8q=_j1OD^=d+EGEcQf_iHKuE``6H#H2s55bOd(r>{hp-=j7IzoxH;oume@ zU}ed?LHw&`HlHJ;dlIrlGU>c;bPB%7bAJPlG)t58T;`?Q{IcQmd$%ttoxP6G=|UYH zc!XXejJ1Uqjr1ssZ}^>3`Fi6X=lyx4d>Xx5LoKz?ycmf1q5$$>NP@1-b5L>-lmXcv7eC^|!I@#pa9~ z7~g!}uG3o!-Ynzdnn70g7qPzS!}Sz0C&0*#=A!`PxlvT`z{Wm0%Ly-dcpq5t+XNW_|nIT{Vp@35a;(`^Cy= zo}BF^_mjGcr=LV~I_zL6NX}w<*l|3hN{P{Jx&o>tLUo5YKM}+WX;NG*9y($UcpDR5 zxH0}ApriiMZ||zz&x9wGQtHgeM_74V^)HD0Vvv@qI6BWk@Nu!g{qkxo+0O>}!#%vW zeuFs!B;B>a*7^i=|2Wa}ae7aXe0h@3E`_7v&-i|^w8X-ndWv}C9xfd1lL;3a8Kkx~ zyr+9d>r_Hkd=<&&K`n8~*Y5GrMnhEM!xM?fJaXK4t$sE(KPz050>) zu|>Dzr@_IAfCVe+6T2bI;%yu=5uz>}?T;J4_$0GeboLnbC9p<%E;I() zr(`%?wh0yhB-ejI26~*U4L8rG`*8nx0>&aiF|ak#w8y8m!u@m+=h#~GYsQGVCeMQV zYzUR4z}N9u#LrV*dtfB;3@TQG2Yx}&OqYaBHKlcLwmbCjHx_YPl^DBb)Qpj@0aCUu zlXq??mwxlFrIx;aO641#ZHMGl2q;FfdJa4P!?-+`;qWGWEFiFvR8p#y(*sH9;eYn| z%jw(puvPZz(PM)X8rpn<{S_^BXm{XuTl9m+2j|${lQws3Lo3K-Zh#EInThj8+$4tBIAtm0xtfG z`~y=|B|K^9|DH3Rdlq^He==027uW5&hcGU6nw5Es)o5Z-sQk1#4tkPIi-X77k82gj z)%DP1%xgxnAtpnnO7_osbKU(Ef0>)wq;LB+R@(WO!AkM#)D60Noa?${J6-6^&cEUR zy?y!I`Q82T=aXXXn!OB-!@1x2-|aGQL@z?|DR4|AR2jQ9#ve;a!IHS@!93XX^MBt*aacn-r1U<#yH+qNr$>v6 z(Qi3Op_o{reEZIsE{qXuLnoZ{xHk3_Ha&j9`=x~X>66dUX)B~a7pyO~uPQ?|$?iHH z$y>BQ-=e>w6?OwV8Me~hi=#PC6Xx(`y)HG)R@7#!@b8)Zgh!`n;G}ZOPVv3kF;Q`< zr=r~nW2NQp4IbJ4-IvAcYk7N&yvTNmZk_-e5t_@Ac0U7jcjNVbM+i_KD!AI`@%c4p zo*pfoe8KC!gu}1m#eJVmmn8irb9&t6 zbVAFIf4R+$S4v_g>r(%BLF^I74)U+nXv`lMAGj4&CkSmLGGg4@k@q>hPv~`mlx*1P zvO-n-vf{lWy3qH4~#t|Ie@p&h&JA zl|;y#Rkx3nwDIjlaCE(iTK}*=>ydMs(h`LJQ&(t&fUDWHBQ$zE7X;m?Rog!wA=h~U z;|g?yI~hN04{lM@#gx%HQ}^H})U4NjuAnK8B9)GJ4~Oj2v)r$+l`k`z#5?e}SI3M| zf$zvXRIn9ZuNLGaMpFz)latA!n;$-QFEx~jN}pedim6BD+}h2U=FIG5t3Dlf^8A2N zsY)VeoetM37ZI6)N00tny88fb-3f=?n2|60nv2Gtq=Gy;6myC%dU${HRtZMif1lY* zrV#JG$>IpCg$moby9Tqm_Hc#^y%A*}2op58bWUB0?nqB#;kjy-P&dw^1CmRrrSbrU zCn9hBT;DpbIy>3^c2ZyD4E)KP6EtNxl9!`1SSYy`(l>!RVELO&(%Wok=Q-mR=Dz&O z4)JIE>rh6M52p-s1SmG&bj~ID@GKUPZL4!>t_CS}4m{Mz{BZGW}1dheSF1#)vrVjkeAnYLE{VD-c!sxh0QR zat8UX?kA&gkKX#cBTJ-RgStJ3Iw#bJGcb=HOa-{dqT$2rQ8po z<56h_Yx-gt35oFET?=_}D3!ilcELKU-PKsW(&LygFS5AZreG6sf)%YA?$l@)1b8%|+{@l-z@q>Yj zB-{4x7spHp61kt`l)TYk^ns}OW=PcP9IjOOol3XKbi7r%BCvLHXkm=k`d?qXt;|V$ zG-v6$w&ZNKn)fv1-Q{fEgf7dW9hoHEE5}C#vw_FrsbXfEEaj*iEDA12Qr)pl4`EQz zZ}bXc|1%rmy?F7{N)FM*9<@Zr29PDva!`EqYaMQ@>>!zZFQM+I7GjMu=FXn>s&&Xq zhvm|nnzg`mR_r+)$E~771o~Wwkh2GKYo1s`z`s;vMFOtrbqHdtWP$$9&^^wbnwm_S zOZO&q3@Xi5D|diUVv9@}9D<+HA`teF_tFFsN=BvHZx0jD)L({1!lWeAf5;=zN}z8;62iSVxvZ%%0aRBNJj8t%wDbHR-C?x*2p% ztL~U?>hKC)RCgaI{@ZazfvL^+twlziU+|Nm$_odnN}Sr|Om>=D4&1*?{-LJyJPuFd ztNjU)hkDUVPq#b`?e`XlUE6JQRnzB1nA*sux^9+mRprKud*{!INh7fBGeJ&ok$S8& zwx5l-^sMMGo&@FNU&nYs{B}oZ`arJGX4SZnP--GsEW=U-1FHsY_pdSCQRk7D>`(=n zW=q$6dqS)y4VqB?r))I}p8Rs2&icYK>>Ho^Ny>-T!xwHrHzph5CUv(Kq;TX^ezl82 z{L-?7#E*haA%@Gc`hyj=cYFSDHyehu67-`zw`?tOTQ83^l`X=E|3vU_s74mEfWQXp z&OiP9ino|`o+G}0D)+py_&QLQM_3< z z*4SspX4UJW4ksM$;|n%jvlX)95n7~Wt?EoZ?2w>Uh4QV0^3vyE!0dp2IB3(u?W(0! zk=BeEN?Mf{mS&ewj&XWXYH(RP_eYRiV{<78ZpcDtgRMejKeviGE}7>g_;npdyc0lis&5)Qf-$hAsc$5L`@islL`mk!M{>o* zPga+V<~(`rs6-|73b-#k_T>vJB%_pV0QCY-ks$zkLQ|pZNNH6@=05G;GxnG|Tpztm zwv;$*O0$wlr2HF38BHfCx0mU07}6}ef`y9sQnB%~-N+#-7lrGvojO#ASn%7xOg(&f zF)YgeZZmz?yjd2FohmXXbXPzO59^R>r}%YSmx+1?p( zAwCQN+m{2>o|)g2@lmXiM#M*^bzb~}mgIA_N7tL(XbWEWyEE|y>OX3wJ(S9g@Bd}J z!3)=7w#&gQAUm7Astl3T^NdiJVEibm$)upe^In(M9ev}KZIhd5xRwCU0rCi@i4;$# z#GcM9T6J@SARj&%RKgoYPcL#(EO$w7FWj8HjKNFA6_eEw0S~9b%T1w7>Rfp5p-&0N z%4mO!F>Mn*S9K)vJoevW8e_D|wmy4iP&w(y5tYI5RpN3o-8lxqu~zfuK%^|L1-bRj z-_<4JjthOPAQ7Xt8h>j{pU(PHo6KsrFTP1qs8wQ(M4W9r``gED@}Aj7n&R!)Ay492 zOq-;6_Xm;x7Zp0}#EkG1jqsYt=U#bR4&wiSYpy7Z21wrYf__xh{YLoVK@DS8k-v4( zIb_X#7i9n}P1)F&zElkv1>_Si9uN7FYA*>y!;&TNr@%z3^DhP$k_IYfStV?8 zdH?MmLL&TpKAQYb`LPAQS|7Ro@OFO5%V;JF$1vor#s8~p-lyMj=-IVI*&5BJ8q#AK zsQ_eSm)on)VP}5x%97Rp=X^AMaTaZ^hxcyhb=j*aC6-Wc#=0qf#I7UD_pd43G;9|7 z(XpLAQfBbqYA(d~Bn>TKmkwlsX-T-H%ciH*)PJ9E!cV zNIsh^FL>Q?1z<*oPP>gSg6L`a_i}#CVC>5(is8QLeHlkB!4E{eNso;Jm?row%@$q$ zNEZQo^j#y-uTk;X{-ef^n`Flmr*LP~MgXW_RebFm_4~#SK}yMr6UGwzfc@2g{aXal z8a*ww0RmWhZ>7o(KlOfB4sTh>cDqhz2gdUqajm*N`TD~8Xb&)F?Y&z?8f{vL1^tox zL->^BLI(&*V9Kp_~do`*fW=3M|ez)8si7*nHzs1x4?uM_1Tw47vCa6agO!F)Z9}@uh!0Bub6z2KjCzPhR904TpgZIem z0N}IKtzhM^&j0{zUWlLfK!!P)>NjE)kd!(TDubwi*82ww9R&j(T_as*9i~+5$CF55 zJZ!a|i$c&<}I-b2Pg;989jJu2{WXZz<lO;B0ZYAOfV3IOQLNfp( zDN4Bg2su*;A%&MSh2aXiD3HS>xYnPdu z;g;&08qv#mYu?4S$5km+kzYj8qv^Y5&HCS)qb&jQAON5@vy@HX9%RYzvZCtp_bF{I z>32?ekN+()mfVE!Agw}LQN#!doeHAK5c4% zS29VS216(Sz|i8%bkP%#X}9ulXa!n5uMwNcdbd2cBreOrl!CGnITj<3DVgVO6M3E( z0RX@RHZ5xkRrXadWYK*UNLh(F;`u49pOI)0GXTXbH^7;0;w3e`9%E&5bhaSNo3xj+ z8v`#LeZXQ|p~=+Y-zfordYbV~=;nB-r!#WB89yk9>E%`zei*s2IF#md#`-ubT7u zjbw9WS}^(s8$g(#{~;JcMd~KPywf6+Q$ro?B|k3j=~iK^5ehDw^4|7>$u;@{t`IW;?Bg%G2{5>QFyWWxrF ziX09aU|LldD?v6Y1meK$i_Ba7jfoy|^DCqf3Kj#&0)p$m+jb>ICS`~hdvR%)%yj#r zfMLun{})*J;~6R?fZOV@FjoTg(vk!t|EeUKG0@^L0>GvVK_7l*d)upRU?s#-x(u2_ zZ@c``h)b!Wa-O9JbtxF)D=jCx6ZLuzP3L6zahCN106%)mrx*WqN@9hw0Xd;Bo^9CR zF&FV_w|mD=+68&y-JyhCd=?8~D{Q?eboZG*RsmXubM4;!mw8~C1c9VA7~<=Cg=3R; zMiZHo#bk5DF<|@vup0c_%Y6Q%MLc@1M~{tAbVRI0{4GPCk3ASE_(L$FJ9PbQUd8Rx z3`Gh^Oko8Z5UglX(?b$fJhQ?Njw=6!!lELKus%MTRjI@)C)3P$OnWa9))6q9g7OIs zi4Z%Mj9^m9c<0o-m~to>SszuugMrn>WP3Z!G6+Nm=vyBLQ1AVlZPGa)vJ5WVlQGx& z9DHNP50X)+Y(CO$LDnju_^@Js{c4Mc0%XvP>55Q?u+9=;W z#6PJ)9t$i7N^Pl%>jVfnS z_wr1#thE7frvSa-mn?#@M2ttXjk)IGjT;!fTIwp9)ZyBcfT@0bmle0CJ9lQmU6P*y zE9p|r5^#(SHnh_I=*H#@U2UF(|FAZHbZFR1RTjq5$&A0v+@sq%@1>8*!L0cJ>!yOj zm%9bS`*?15Zry(nM95&GO-fDk--~q{t6*-h<$1GQsF2Ij;B8KOd!i;f><6-q4kx=1 z1D=GQ$PD&5Oj$QG%|dw+w;0>WoT*#cEr{ z)xm@5GPuEP>g`CA*uPo;@Fug)C5{Y6<7NeG>>Y#mG=49To$skFmVB5VKS@^#NQ4Qt zV%EGoBPEHitLZr~n*DF120jS6K?`xdZQGudb*-qQb=54P#6V#>Fe_a~5fP8S_1VoH zy&UCCF2W#*qSq!1%Kk1QSqbAOm)?a)(u>LfD4At0oki5>GEZR0tav+G6Gm?^3qv4y zrL;IEzL}#EcSbqx*`EltfAsrHG*(kdIbc#4mo^(nI@hAZ|4|{9n_Pz9BUFOOeK991 z^iwp0fpbuH1HvhnEqhV6M&W~9;V>Au)4Q#>ZDU`=s zfhplT(Ww5A%{?dMBw9{7N%8%irnHVgTOO2T`=g!f`Q@UFVz}+0l%8S#g}Uz3kOfR| zvPIf1tYhr5kW?}{-3uRuj)p>KVW$`z=T7me$`1z!Ydx>|_~AuUC7s7{*w=KwD9KaxTb^<6$06S=st%LyoV^ zcA#g`1fTbD_}6jnUyMSrjGy-cah@BHJ1uz)1hmAtR)aDS5dz5|*+7sjIhVKR^CKOW zgQGL%&u=||P48ek*r#TyfN%hcD68;mw>v}lA5RBw9FQ_sdi`-$H7u*Ddlc2K&cwJ-uTF1A@v??)qgf)*>28(pM@Kc;(?~ zFMFf^^TK#nmzN-AUM|BFwh@RcyrYyRPY*<= zhGs7E+2rXuJf&z(HjsGwB%Z8v1O52f2DW^R;1)(om|0$uCggXw729|^7Jo9J?W6Tc zSDneDdP1w!pQ1yiIszX_22Ru%D<^&;+Ct+em6oWvkX>C-kvq{;z`PRee)qMF4kUy! zzmG8g+={Gr)wVS z{()CAe}9MTYIy-^mjujkndt;hYV$vwr_P0Ps2lZGJ6lY3$A2@n88q7szD_dB%pC}0 zMF~NS`bA}eVch)X(E7ANj7fKte!5Wx9sOUp^OMt39>=eNyObgyoB4SZ{dZgMMFNZ1 z!>yGFOh74`x*S>}5Jiq^<5-<^GFVHJwD_=UJAcE;z7HRZTP=M2_3u8oWG2!?vBX_QiwB}<7h_Q6mI(}o6O z9-y)~NE%@4zrP$&^51IZ8V0sRT|U@9|5$AEwrd;h57q{D@M~wN#DY6+o-RQzVUuMn z*Fo3_2ABgyr{9Gi`$NufXbByiAu*R12xvfuV}NN%&%?gBWT7_Im|zo_1l`nO)bhc9 zPL1dvdhQ#BK!7cXB^*K%*~=6160;j46g&9We`F+JTY`G>@OpqPD~h!s)vU6MIv!(w zMCl8R=RHZ$V&jU;+!&Mo|Izc8rRTpCk;Z}XYoiH66uMWVI9LI;+$gf}s0lJAxE-@b z3DvB-b9tMa*7*3Pv9~OQ5C$1LGE(b=J$i$$eddfEW3;Wl=;v z!Kk-{dh%Xz2x7lp(cR0%!w;UvKq;ZyLg%A(t-w4-qD8tWM2-1$BACxI23<*HdZ-2x zV#-t3P#*b=!4;s4^00tH%E~t*x}NdXeFQ#&KMQ()y!{%DSid>!je`G2On(~#hA42~Rgjkt~ literal 0 HcmV?d00001 diff --git a/apps/ios-playground/src/icons/linked.svg b/apps/ios-playground/src/icons/linked.svg new file mode 100644 index 000000000..6849e852f --- /dev/null +++ b/apps/ios-playground/src/icons/linked.svg @@ -0,0 +1,3 @@ + + + diff --git a/apps/ios-playground/src/icons/screenshot.svg b/apps/ios-playground/src/icons/screenshot.svg new file mode 100644 index 000000000..8b477bb98 --- /dev/null +++ b/apps/ios-playground/src/icons/screenshot.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/apps/ios-playground/src/icons/unlink.svg b/apps/ios-playground/src/icons/unlink.svg new file mode 100644 index 000000000..1529f0a62 --- /dev/null +++ b/apps/ios-playground/src/icons/unlink.svg @@ -0,0 +1,3 @@ + + + diff --git a/apps/ios-playground/src/index.tsx b/apps/ios-playground/src/index.tsx new file mode 100644 index 000000000..2c2eb681b --- /dev/null +++ b/apps/ios-playground/src/index.tsx @@ -0,0 +1,8 @@ +import ReactDOM from 'react-dom/client'; +import App from './App'; + +const rootEl = document.getElementById('root'); +if (rootEl) { + const root = ReactDOM.createRoot(rootEl); + root.render(); +} diff --git a/apps/ios-playground/src/ios-device/index.less b/apps/ios-playground/src/ios-device/index.less new file mode 100644 index 000000000..8b3e6a70d --- /dev/null +++ b/apps/ios-playground/src/ios-device/index.less @@ -0,0 +1,44 @@ +.ios-device-container { + .status-indicator { + width: 8px; + height: 8px; + border-radius: 50%; + display: inline-block; + + &.connected { + background-color: #52c41a; + } + + &.disconnected { + background-color: #ff4d4f; + } + } + + .connection-actions { + border: 1px dashed #d9d9d9; + border-radius: 6px; + padding: 12px; + background-color: #fafafa; + } + + .connection-info { + text-align: center; + padding: 8px; + background-color: #f6ffed; + border: 1px solid #b7eb8f; + border-radius: 6px; + } + + .server-status { + padding: 8px 12px; + background-color: #fafafa; + border-radius: 6px; + border: 1px solid #d9d9d9; + } + + .server-url { + padding: 8px 12px; + background-color: #f0f0f0; + border-radius: 6px; + } +} diff --git a/apps/ios-playground/src/ios-device/index.tsx b/apps/ios-playground/src/ios-device/index.tsx new file mode 100644 index 000000000..6fcb4eff1 --- /dev/null +++ b/apps/ios-playground/src/ios-device/index.tsx @@ -0,0 +1,129 @@ +import { Button, Card, Space, Typography } from 'antd'; +import { useEffect, useState } from 'react'; +import './index.less'; + +const { Text, Title } = Typography; + +interface IOSDeviceProps { + serverUrl?: string; + onServerStatusChange?: (connected: boolean) => void; +} + +export default function IOSDevice({ + serverUrl = 'http://localhost:1412', + onServerStatusChange, +}: IOSDeviceProps) { + const [serverConnected, setServerConnected] = useState(false); + const [checking, setChecking] = useState(false); + + // Helper function to get the appropriate URL for API calls + const getApiUrl = (endpoint: string) => { + // In development, use proxy; in production or when server is not localhost:1412, use direct URL + if (serverUrl === 'http://localhost:1412' && process.env.NODE_ENV === 'development') { + return `/api/pyautogui${endpoint}`; + } + return `${serverUrl}${endpoint}`; + }; + + const checkServerStatus = async () => { + setChecking(true); + try { + // Use proxy endpoint to avoid CORS issues + const response = await fetch(getApiUrl('/health')); + const connected = response.ok; + setServerConnected(connected); + onServerStatusChange?.(connected); + } catch (error) { + console.error('Failed to check server status:', error); + setServerConnected(false); + onServerStatusChange?.(false); + } finally { + setChecking(false); + } + }; + + const startPyAutoGUIServer = () => { + // Show instructions to user since we can't start server from frontend + const message = `Please start the PyAutoGUI server manually: + +1. Open Terminal +2. Run: npx @midscene/ios server +3. Make sure iPhone Mirroring app is open and connected`; + + alert(message); + }; + + useEffect(() => { + checkServerStatus(); + // Check server status every 3 seconds + const interval = setInterval(checkServerStatus, 3000); + return () => clearInterval(interval); + }, [serverUrl]); + + return ( +

+ + + iOS Device Connection + + + } + size="small" + > + +
+ + PyAutoGUI Server: +
+ + {serverConnected ? 'Connected' : 'Disconnected'} + + +
+ +
+ + Server URL: + {serverUrl} + +
+ + {!serverConnected && ( +
+ + + + +
+ )} + + {serverConnected && ( +
+ + ✅ Ready for iOS automation + +
+ )} + + +
+ ); +} diff --git a/apps/ios-playground/src/ios-player/index.less b/apps/ios-playground/src/ios-player/index.less new file mode 100644 index 000000000..2c3f9927a --- /dev/null +++ b/apps/ios-playground/src/ios-player/index.less @@ -0,0 +1,60 @@ +.ios-player-container { + .mirror-config { + margin-bottom: 16px; + padding: 12px; + background-color: #fafafa; + border: 1px solid #d9d9d9; + border-radius: 6px; + } + + .config-status { + margin-bottom: 12px; + padding: 8px; + border-radius: 4px; + + &.enabled { + background-color: #f6ffed; + border: 1px solid #b7eb8f; + color: #52c41a; + } + + &.disabled { + background-color: #fff7e6; + border: 1px solid #ffd591; + color: #fa8c16; + } + } + + .display-area { + min-height: 400px; + display: flex; + align-items: center; + justify-content: center; + background-color: #f5f5f5; + border: 1px solid #d9d9d9; + border-radius: 6px; + position: relative; + + .placeholder { + text-align: center; + color: #8c8c8c; + } + + .screenshot-container { + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + padding: 8px; + + .ios-screenshot { + max-width: 100%; + max-height: 500px; + border-radius: 12px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + border: 1px solid #d9d9d9; + } + } + } +} diff --git a/apps/ios-playground/src/ios-player/index.tsx b/apps/ios-playground/src/ios-player/index.tsx new file mode 100644 index 000000000..a03431f50 --- /dev/null +++ b/apps/ios-playground/src/ios-player/index.tsx @@ -0,0 +1,171 @@ +import { Card, Button, Space, Typography, message, Tooltip } from 'antd'; +import { SearchOutlined } from '@ant-design/icons'; +import { forwardRef, useImperativeHandle, useState, useEffect } from 'react'; +import './index.less'; + +const { Text } = Typography; + +export interface IOSPlayerRefMethods { + refreshDisplay: () => Promise; +} + +interface IOSPlayerProps { + serverUrl?: string; + autoConnect?: boolean; +} + +const IOSPlayer = forwardRef( + ({ serverUrl = 'http://localhost:1412', autoConnect = false }, ref) => { + const [connected, setConnected] = useState(false); + const [autoDetecting, setAutoDetecting] = useState(false); + const [messageApi, contextHolder] = message.useMessage(); + const [mirrorConfig, setMirrorConfig] = useState(null); + + // Helper function to get the appropriate URL for API calls + const getApiUrl = (endpoint: string) => { + // In development, use proxy; in production or when server is not localhost:1412, use direct URL + if (serverUrl === 'http://localhost:1412' && process.env.NODE_ENV === 'development') { + return `/api/pyautogui${endpoint}`; + } + return `${serverUrl}${endpoint}`; + }; + + const checkConnection = async () => { + try { + const response = await fetch(getApiUrl('/health')); + const isConnected = response.ok; + setConnected(isConnected); + + // If connected, also get the current config + if (isConnected) { + try { + const configResponse = await fetch(getApiUrl('/config')); + const configResult = await configResponse.json(); + if (configResult.status === 'ok') { + setMirrorConfig(configResult.config); + } + } catch (error) { + // Ignore config fetch errors + console.warn('Failed to fetch config:', error); + } + } + + return isConnected; + } catch (error) { + setConnected(false); + setMirrorConfig(null); + return false; + } + }; + + const autoDetectMirror = async () => { + if (!connected) { + messageApi.warning('Server is not connected'); + return; + } + + setAutoDetecting(true); + try { + const response = await fetch(getApiUrl('/detect'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + + const result = await response.json(); + if (result.status === 'ok') { + messageApi.success(`Auto-configured: ${result.message}`); + setMirrorConfig(result.config); + } else { + messageApi.error(`Auto-detection failed: ${result.error}`); + if (result.suggestion) { + messageApi.info(result.suggestion); + } + } + } catch (error) { + messageApi.error(`Auto-detection error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + setAutoDetecting(false); + } + }; + + useImperativeHandle(ref, () => ({ + refreshDisplay: async () => { + // Just refresh the connection status + await checkConnection(); + }, + })); + + useEffect(() => { + checkConnection(); + const interval = setInterval(checkConnection, 3000); + return () => clearInterval(interval); + }, [serverUrl]); + + useEffect(() => { + if (autoConnect && connected) { + // Try auto-detection when connected + autoDetectMirror(); + } + }, [autoConnect, connected]); + + return ( +
+ {contextHolder} + + iOS Display + {connected && ( + + + + + + )} + + } + size="small" + > + {connected && mirrorConfig && mirrorConfig.enabled && ( +
+ + ✅ Configured: {mirrorConfig.estimated_ios_width}×{mirrorConfig.estimated_ios_height} device + → {mirrorConfig.mirror_width}×{mirrorConfig.mirror_height} at ({mirrorConfig.mirror_x}, {mirrorConfig.mirror_y}) + +
+ )} + +
+ {!connected ? ( +
+ + Waiting for iOS device connection... +
+ Please ensure iPhone Mirroring is active + +
+ ) : ( +
+ + iOS device connected. Use Auto Detect to configure mirroring. + +
+ )} +
+ +
+ ); + } +); + +IOSPlayer.displayName = 'IOSPlayer'; + +export default IOSPlayer; diff --git a/apps/ios-playground/src/scripts/blank_polyfill.ts b/apps/ios-playground/src/scripts/blank_polyfill.ts new file mode 100644 index 000000000..ff8b4c563 --- /dev/null +++ b/apps/ios-playground/src/scripts/blank_polyfill.ts @@ -0,0 +1 @@ +export default {}; diff --git a/apps/ios-playground/tsconfig.json b/apps/ios-playground/tsconfig.json new file mode 100644 index 000000000..ca9a7367e --- /dev/null +++ b/apps/ios-playground/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "lib": ["DOM", "ES2020"], + "jsx": "react-jsx", + "target": "ES2020", + "skipLibCheck": true, + "useDefineForClassFields": true, + + /* modules */ + "module": "ESNext", + "isolatedModules": true, + "resolveJsonModule": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "noEmit": true, + + /* type checking */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true + }, + "include": ["src"] +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index f5f02fc2c..3115e7a90 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -8,6 +8,17 @@ import { createConfig, createFilesConfig } from './config-factory'; Promise.resolve( (async () => { + // Load .env file early, before parsing any YAML files that might use env vars + const dotEnvConfigFile = join(process.cwd(), '.env'); + if (existsSync(dotEnvConfigFile)) { + console.log(` Env file: ${dotEnvConfigFile}`); + dotenv.config({ + path: dotEnvConfigFile, + debug: false, // Will be set properly later + override: false, // Will be set properly later + }); + } + const { options, path, files: cmdFiles } = await parseProcessArgs(); const welcome = `\nWelcome to @midscene/cli v${version}\n`; @@ -65,9 +76,8 @@ Promise.resolve( process.exit(1); } - const dotEnvConfigFile = join(process.cwd(), '.env'); + // Update dotenv configuration with user-specified options if (existsSync(dotEnvConfigFile)) { - console.log(` Env file: ${dotEnvConfigFile}`); dotenv.config({ path: dotEnvConfigFile, debug: config.dotenvDebug, diff --git a/packages/ios-playground/README.md b/packages/ios-playground/README.md new file mode 100644 index 000000000..f3b91119f --- /dev/null +++ b/packages/ios-playground/README.md @@ -0,0 +1,147 @@ +# @midscene/ios-playground + +iOS playground for Midscene.js - Control iOS devices through natural language commands using screen mirroring. + +## Quick Start + +```bash +npx @midscene/ios-playground +``` + +This will: + +1. Automatically start the PyAutoGUI server on port 1412 +2. Launch the iOS playground web interface +3. Open the playground in your default browser + +## Prerequisites + +1. **macOS System**: iOS playground requires macOS (tested on macOS 11 and later). + +2. **Python 3 and Dependencies**: The playground will automatically manage the PyAutoGUI server, but you need Python 3 with required packages: + + ```bash + pip3 install pyautogui flask flask-cors + ``` + +3. **iPhone Mirroring**: Use iPhone Mirroring (macOS Sequoia) to mirror your physical iPhone to your Mac screen. + +4. **AI Model Configuration**: Set up your AI model credentials. See [Midscene documentation](https://midscenejs.com/choose-a-model) for supported models. + +## Features + +- **Automatic Server Management**: PyAutoGUI server starts and stops automatically +- **Auto-Detection**: Automatically detects iPhone Mirroring window position and size +- **Natural Language Control**: Control iOS devices using natural language commands +- **Screenshot Capture**: Takes screenshots of only the iOS mirrored region +- **Coordinate Transformation**: Automatically maps iOS coordinates to macOS screen coordinates +- **Real-time Interaction**: Direct interaction with iOS interface elements through AI + +## Usage + +1. **Start the playground**: + + ```bash + npx @midscene/ios-playground + ``` + +2. **Set up iPhone Mirroring**: Open iPhone Mirroring app on your Mac (macOS Sequoia) and connect your iPhone + +3. **Configure AI Model**: In the playground web interface, configure your AI model credentials + +4. **Auto-detect or Manual Setup**: + - Click "Auto Detect iOS Mirror" for automatic configuration, or + - Manually set the mirror region coordinates + +5. **Use natural language commands** to interact with your iOS device: + + - **Action**: "tap the Settings app" + - **Query**: "extract the battery percentage" + - **Assert**: "the home screen is visible" + +## Development + +To run the playground in development mode: + +```bash +cd packages/ios-playground +npm install +npm run dev:server +``` + +This will build the project and start the server locally. + +## Architecture + +The iOS playground architecture consists of: + +- **Frontend**: Web-based interface for AI interaction (built with React/TypeScript) +- **Playground Server**: Express.js server that bridges between frontend and iOS automation +- **PyAutoGUI Server**: Python Flask server for screen capture and input control +- **iPhone Mirroring**: macOS iPhone Mirroring for device display +- **Midscene AI Core**: AI-powered automation engine with iOS device adapter +- **Coordinate Transformation**: Automatic mapping between iOS logical coordinates and macOS screen coordinates + +## How It Works + +1. **Screen Mirroring**: iOS device screen is displayed on macOS through iPhone Mirroring +2. **Auto-Detection**: Python server detects the mirroring window position and size using AppleScript +3. **Coordinate Mapping**: iOS logical coordinates (e.g., 200, 400) are automatically transformed to macOS screen coordinates +4. **AI Processing**: Midscene AI analyzes screenshots and determines actions based on natural language commands +5. **Action Execution**: Actions are executed on the macOS screen within the iOS mirrored region + +## Troubleshooting + +### PyAutoGUI Server Issues + +If the PyAutoGUI server fails to start automatically, check: + +```bash +# Check if port 1412 is available +lsof -i :1412 +# Manually start the server +cd packages/ios +node bin/server.js 1412 +``` + +### iPhone Mirroring Detection Issues + +1. Ensure iPhone Mirroring app is open and visible on screen +2. Try clicking "Auto Detect iOS Mirror" in the playground interface +3. Manually configure mirror coordinates if auto-detection fails +4. Check that the iPhone Mirroring window is not minimized + +### Permission Issues + +On macOS, you may need to grant the following permissions: + +- **Accessibility**: System Preferences > Security & Privacy > Privacy > Accessibility +- **Screen Recording**: System Preferences > Security & Privacy > Privacy > Screen Recording + +Add Terminal, Python, or your development environment to these permission lists. + +### Python Dependencies + +If you encounter Python-related errors: + +```bash +# Install or upgrade required packages +pip3 install --upgrade pyautogui flask flask-cors + +# On macOS, you might need to install using conda or homebrew +brew install python@3.11 +``` + +### Mirror Region Configuration + +If clicks are not landing in the right place: + +1. Use the "Auto Detect iOS Mirror" feature first +2. If manual configuration is needed, measure the exact position and size of your iPhone Mirroring window +3. Account for window borders and title bars when setting coordinates + +## Related Documentation + +- [Midscene.js Documentation](https://midscenejs.com/) +- [API Reference](https://midscenejs.com/api) +- [Choosing AI Models](https://midscenejs.com/choose-a-model) diff --git a/packages/ios-playground/bin/ios-playground b/packages/ios-playground/bin/ios-playground new file mode 100755 index 000000000..9341e503e --- /dev/null +++ b/packages/ios-playground/bin/ios-playground @@ -0,0 +1,34 @@ +#!/usr/bin/env node + +const path = require('path'); +const { spawn } = require('child_process'); + +// Get the directory where this script is located +const binDir = __dirname; +// The server script should be in the same directory as this bin script +const serverScript = path.join(binDir, 'server.js'); + +console.log('Starting iOS Playground server...'); + +// Start the server +const serverProcess = spawn('node', [serverScript], { + stdio: 'inherit', + env: { ...process.env, NODE_ENV: 'production' } +}); + +// Handle process termination +process.on('SIGINT', () => { + console.log('\nShutting down iOS Playground server...'); + serverProcess.kill('SIGINT'); + process.exit(0); +}); + +process.on('SIGTERM', () => { + serverProcess.kill('SIGTERM'); + process.exit(0); +}); + +serverProcess.on('close', (code) => { + console.log(`iOS Playground server exited with code ${code}`); + process.exit(code); +}); diff --git a/packages/ios-playground/bin/server.js b/packages/ios-playground/bin/server.js new file mode 100644 index 000000000..ed405d1d9 --- /dev/null +++ b/packages/ios-playground/bin/server.js @@ -0,0 +1,162 @@ +const path = require('path'); +const { spawn } = require('child_process'); +const { iOSDevice, iOSAgent } = require('@midscene/ios'); +const { PLAYGROUND_SERVER_PORT } = require('@midscene/shared/constants'); +const PlaygroundServer = require('@midscene/web/midscene-server').default; + +const staticDir = path.join(__dirname, '..', '..', '..', 'apps', 'ios-playground', 'dist'); +const playgroundServer = new PlaygroundServer( + iOSDevice, + iOSAgent, + staticDir, +); + +// Auto server management +let autoServerProcess = null; +const AUTO_SERVER_PORT = 1412; + +/** + * Check if auto server is running on the specified port + */ +const checkAutoServerRunning = async (port = AUTO_SERVER_PORT) => { + return new Promise((resolve) => { + const net = require('net'); + const client = new net.Socket(); + + client.setTimeout(1000); + + client.on('connect', () => { + client.destroy(); + resolve(true); + }); + + client.on('timeout', () => { + client.destroy(); + resolve(false); + }); + + client.on('error', () => { + resolve(false); + }); + + client.connect(port, 'localhost'); + }); +}; + +/** + * Start the auto server if it's not running + */ +const startAutoServer = async () => { + try { + const isRunning = await checkAutoServerRunning(); + + if (isRunning) { + console.log(`✅ PyAutoGUI server is already running on port ${AUTO_SERVER_PORT}`); + return true; + } + + console.log(`🚀 Starting PyAutoGUI server on port ${AUTO_SERVER_PORT}...`); + + // Find the auto server script path + const autoServerPath = path.join(__dirname, '..', '..', 'ios', 'bin', 'server.js'); + + // Start the auto server process + autoServerProcess = spawn('node', [autoServerPath, AUTO_SERVER_PORT], { + stdio: 'pipe', + env: { + ...process.env, + NODE_ENV: 'production' + } + }); + + // Handle auto server output + autoServerProcess.stdout.on('data', (data) => { + const output = data.toString().trim(); + if (output) { + console.log(`[PyAutoGUI] ${output}`); + } + }); + + autoServerProcess.stderr.on('data', (data) => { + const output = data.toString().trim(); + if (output) { + console.error(`[PyAutoGUI Error] ${output}`); + } + }); + + autoServerProcess.on('error', (error) => { + console.error('Failed to start PyAutoGUI server:', error); + }); + + autoServerProcess.on('close', (code) => { + if (code !== 0) { + console.error(`PyAutoGUI server exited with code ${code}`); + } + autoServerProcess = null; + }); + + // Wait a bit for the server to start + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Verify it's running + const isNowRunning = await checkAutoServerRunning(); + if (isNowRunning) { + console.log(`✅ PyAutoGUI server started successfully on port ${AUTO_SERVER_PORT}`); + return true; + } else { + console.error(`❌ Failed to start PyAutoGUI server on port ${AUTO_SERVER_PORT}`); + return false; + } + + } catch (error) { + console.error('Error starting auto server:', error); + return false; + } +}; + +const main = async () => { + try { + // Start auto server first + await startAutoServer(); + + await playgroundServer.launch(PLAYGROUND_SERVER_PORT); + console.log( + `Midscene iOS Playground server is running on http://localhost:${playgroundServer.port}`, + ); + + // Automatically open browser + if (process.env.NODE_ENV !== 'test') { + try { + const { default: open } = await import('open'); + await open(`http://localhost:${playgroundServer.port}`); + } catch (error) { + console.log('Could not open browser automatically. Please visit the URL manually.'); + } + } + } catch (error) { + console.error('Failed to start iOS playground server:', error); + process.exit(1); + } +}; + +// Handle graceful shutdown +const cleanup = () => { + console.log('Shutting down gracefully...'); + + if (playgroundServer) { + playgroundServer.close(); + } + + if (autoServerProcess) { + console.log('Stopping PyAutoGUI server...'); + autoServerProcess.kill('SIGTERM'); + autoServerProcess = null; + } + + process.exit(0); +}; + +process.on('SIGTERM', cleanup); +process.on('SIGINT', cleanup); + +main(); diff --git a/packages/ios-playground/modern.config.ts b/packages/ios-playground/modern.config.ts new file mode 100644 index 000000000..ff87335a3 --- /dev/null +++ b/packages/ios-playground/modern.config.ts @@ -0,0 +1,13 @@ +import { moduleTools, defineConfig } from '@modern-js/module-tools'; + +export default defineConfig({ + plugins: [moduleTools()], + buildConfig: { + buildType: 'bundle', + format: 'cjs', + target: 'es2019', + outDir: './dist', + dts: false, + externals: ['express', 'cors', 'open'], + }, +}); diff --git a/packages/ios-playground/package.json b/packages/ios-playground/package.json new file mode 100644 index 000000000..622eaab01 --- /dev/null +++ b/packages/ios-playground/package.json @@ -0,0 +1,34 @@ +{ + "name": "@midscene/ios-playground", + "version": "0.25.3", + "description": "iOS playground for Midscene", + "main": "./dist/lib/index.js", + "types": "./dist/types/index.d.ts", + "files": ["dist", "static", "bin", "README.md"], + "bin": { + "midscene-ios-playground": "./bin/ios-playground", + "@midscene/ios-playground": "./bin/ios-playground" + }, + "scripts": { + "dev": "modern dev", + "dev:server": "npm run build && ./bin/ios-playground", + "build": "modern build -c ./modern.config.ts", + "build:watch": "modern build -w -c ./modern.config.ts --no-clear" + }, + "dependencies": { + "@midscene/ios": "workspace:*", + "@midscene/shared": "workspace:*", + "@midscene/web": "workspace:*", + "cors": "2.8.5", + "express": "^4.21.2", + "open": "10.1.0" + }, + "devDependencies": { + "@modern-js/module-tools": "2.60.6", + "@types/cors": "2.8.12", + "@types/express": "^4.17.21", + "@types/node": "^18.0.0", + "typescript": "^5.8.3" + }, + "license": "MIT" +} diff --git a/packages/ios-playground/test-health.js b/packages/ios-playground/test-health.js new file mode 100644 index 000000000..88618bbc8 --- /dev/null +++ b/packages/ios-playground/test-health.js @@ -0,0 +1,28 @@ +const fetch = require('node-fetch'); + +async function testHealth() { + try { + console.log('Testing health check...'); + const response = await fetch('http://localhost:5800/status'); + const data = await response.json(); + console.log('Status:', response.status); + console.log('Data:', data); + console.log('Success!'); + } catch (error) { + console.error('Error:', error.message); + } +} + +// Test multiple times like the frontend does +let counter = 0; +const interval = setInterval(async () => { + counter++; + console.log(`\n--- Test ${counter} ---`); + await testHealth(); + + if (counter >= 5) { + clearInterval(interval); + console.log('\nTest completed'); + process.exit(0); + } +}, 2000); diff --git a/packages/ios-playground/tsconfig.json b/packages/ios-playground/tsconfig.json new file mode 100644 index 000000000..3534ba4c7 --- /dev/null +++ b/packages/ios-playground/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "moduleResolution": "node", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true + }, + "include": ["bin/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ios/bin/server.js b/packages/ios/bin/server.js index 3986753ee..e84645c02 100755 --- a/packages/ios/bin/server.js +++ b/packages/ios/bin/server.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { spawn } from 'node:child_process'; +import { spawn, execSync } from 'node:child_process'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -11,6 +11,14 @@ const port = process.argv[2] || '1412'; console.log(`Starting PyAutoGUI server on port ${port}...`); +// kill process on port 1412 first +try { + execSync(`lsof -ti:${port} | xargs kill -9`, { stdio: 'ignore' }); + console.log(`Killed existing process on port ${port}`); +} catch (error) { + console.error(`Failed to kill process on port ${port}:`, error); +} + const server = spawn('python3', [serverPath, port], { stdio: 'inherit', env: { @@ -38,4 +46,4 @@ process.on('SIGINT', () => { process.on('SIGTERM', () => { console.log('\nShutting down PyAutoGUI server...'); server.kill('SIGTERM'); -}); +}); \ No newline at end of file diff --git a/packages/ios/examples/ios-input-example.yaml b/packages/ios/examples/ios-input-example.yaml index 044436dad..fce09e192 100644 --- a/packages/ios/examples/ios-input-example.yaml +++ b/packages/ios/examples/ios-input-example.yaml @@ -10,11 +10,6 @@ ios: # iOS device mirroring configuration for precise location targeting # These values define the position and size of the mirrored device screen - mirrorConfig: - mirrorX: 794 # X position of iOS mirror on computer screen - mirrorY: 171 # Y position of iOS mirror on computer screen - mirrorWidth: 332 # Width of the mirrored iOS screen - mirrorHeight: 720 # Height of the mirrored iOS screen (iPhone 11 Pro size) # Output file for aiQuery/aiAssert results (optional) output: "./results.json" @@ -35,4 +30,4 @@ tasks: - aiWaitFor: "Search results are displayed" - aiAction: "播放第一首歌曲" - sleep: 3000 - - aiAction: "返回Home" \ No newline at end of file + - aiAction: "返回Home" diff --git a/packages/ios/examples/ios-mirroring-demo.js b/packages/ios/examples/ios-mirroring-demo.js index e8092b2b3..1e8af4ef1 100644 --- a/packages/ios/examples/ios-mirroring-demo.js +++ b/packages/ios/examples/ios-mirroring-demo.js @@ -55,18 +55,8 @@ async function demonstrateIOSMirroring() { // Step 1: Configure iOS device mirroring console.log('📱 Step 1: Setting up iOS device mirroring...'); - // Example configuration for iOS device mirrored via macOS screen sharing - // {"iPhone Mirroring", {692, 161}, {344, 764}} - const mirrorConfig = { - mirrorX: 692, // X position of iOS mirror on macOS screen - mirrorY: 161, // Y position of iOS mirror on macOS screen - mirrorWidth: 344, // Width of iOS mirror on macOS screen - mirrorHeight: 764, // Height of iOS mirror on macOS screen - }; - const device = new iOSDevice({ - serverPort: 1412, - mirrorConfig: mirrorConfig, + serverPort: 1412 }); try { @@ -85,38 +75,13 @@ async function demonstrateIOSMirroring() { const agent = new iOSAgent(device); console.log('✅ AI agent ready!\n'); - // Step 3: Demonstrate coordinate transformation - console.log('🎯 Step 3: Testing coordinate transformation...'); - // sleep 5 seconds to allow user to make the mirror app foreground - await new Promise((resolve) => setTimeout(resolve, 5000)); - - // Test tap at various iOS coordinates - const testPoints = [ - { left: 100, top: 200, description: 'Upper left area' }, - { left: 196, top: 426, description: 'Center of screen' }, - { left: 300, top: 700, description: 'Lower right area' }, - ]; - - for (const point of testPoints) { - console.log( - ` 📍 Tapping at iOS coordinates (${point.left}, ${point.top}) - ${point.description}`, - ); - await device.tap(point); - await new Promise((resolve) => setTimeout(resolve, 500)); // Brief pause - } - console.log('✅ Coordinate transformation test completed!\n'); - - // Step 4: Take iOS region screenshot - console.log('📸 Step 4: Taking iOS region screenshot...'); + // Step 3: Take iOS region screenshot + console.log('📸 Step 3: Taking iOS region screenshot...'); const screenshot = await device.screenshotBase64(); console.log(`✅ Screenshot captured (${screenshot.length} bytes)`); console.log(' 💾 Screenshot contains only the iOS mirrored area\n'); - // Step 5: Test enhanced scrolling functionality (now uses intelligent distance mapping) - console.log( - '🖱️ Step 5: Testing enhanced trackpad scrolling with intelligent distance mapping...', - ); - + // Step 4: Test scrolling console.log( ' 🔄 Testing horizontal scroll right (300px) - should scroll horizontally to the right:', ); @@ -131,8 +96,8 @@ async function demonstrateIOSMirroring() { console.log('✅ Enhanced horizontal scrolling test completed!\n'); - // Step 6: Demonstrate AI automation - console.log('🧠 Step 6: AI automation example...'); + // Step 5: Demonstrate AI automation + console.log('🧠 Step 5: AI automation example...'); console.log(' (This would work with actual iOS app content)'); // Example AI operations (commented out as they need actual iOS app content) @@ -142,8 +107,8 @@ async function demonstrateIOSMirroring() { console.log('✅ Demo completed successfully!\n'); - // Step 7: Show usage summary - console.log('📋 Usage Summary:'); + // Step 6: Show usage summary + console.log('📋 Step 6: Usage Summary:'); console.log('================'); console.log( '• iOS coordinates are automatically transformed to macOS coordinates', diff --git a/packages/ios/idb/auto_server.py b/packages/ios/idb/auto_server.py index 8b99a3b7d..89f1578e8 100644 --- a/packages/ios/idb/auto_server.py +++ b/packages/ios/idb/auto_server.py @@ -1,18 +1,184 @@ from flask import Flask, request, jsonify +from flask_cors import CORS import pyautogui import time import traceback -import json import sys import subprocess -import os +import base64 +import io app = Flask(__name__) +CORS(app) # Enable CORS for all routes # Configure pyautogui pyautogui.FAILSAFE = True pyautogui.PAUSE = 0.1 +def detect_and_configure_ios_mirror(): + """Automatically detect iPhone Mirroring app window and configure mapping""" + try: + # AppleScript to get window information for iPhone Mirroring app + applescript = ''' + tell application "System Events" + try + set mirrorApp to first application process whose name contains "iPhone Mirroring" + set mirrorWindow to first window of mirrorApp + set windowPosition to position of mirrorWindow + set windowSize to size of mirrorWindow + + -- Get window frame information + set windowX to item 1 of windowPosition + set windowY to item 2 of windowPosition + set windowWidth to item 1 of windowSize + set windowHeight to item 2 of windowSize + + -- Try to get the actual visible frame (content area) + try + set appName to name of mirrorApp + set bundleId to bundle identifier of mirrorApp + set visibleFrame to "{" & quote & "found" & quote & ":true," & quote & "x" & quote & ":" & windowX & "," & quote & "y" & quote & ":" & windowY & "," & quote & "width" & quote & ":" & windowWidth & "," & quote & "height" & quote & ":" & windowHeight & "," & quote & "app" & quote & ":" & quote & appName & quote & "," & quote & "bundle" & quote & ":" & quote & bundleId & quote & "}" + return visibleFrame + on error + return "{" & quote & "found" & quote & ":true," & quote & "x" & quote & ":" & windowX & "," & quote & "y" & quote & ":" & windowY & "," & quote & "width" & quote & ":" & windowWidth & "," & quote & "height" & quote & ":" & windowHeight & "}" + end try + + on error errMsg + return "{" & quote & "found" & quote & ":false," & quote & "error" & quote & ":" & quote & errMsg & quote & "}" + end try + end tell + ''' + + success, stdout, stderr = execute_applescript(applescript) + + if not success: + return { + "status": "error", + "error": "Failed to execute AppleScript", + "details": stderr + } + + # Parse the JSON-like response + import re + result_str = stdout.strip() + + # Extract values using regex since it's a simple JSON-like format + found_match = re.search(r'"found":(\w+)', result_str) + if not found_match or found_match.group(1) != 'true': + error_match = re.search(r'"error":"([^"]*)"', result_str) + error_msg = error_match.group(1) if error_match else "iPhone Mirroring app not found" + return { + "status": "error", + "error": "iPhone Mirroring app not found or not active", + "details": error_msg, + "suggestion": "Please make sure iPhone Mirroring app is open and visible" + } + + # Extract window coordinates and size + x_match = re.search(r'"x":(\d+)', result_str) + y_match = re.search(r'"y":(\d+)', result_str) + width_match = re.search(r'"width":(\d+)', result_str) + height_match = re.search(r'"height":(\d+)', result_str) + + if not all([x_match, y_match, width_match, height_match]): + return { + "status": "error", + "error": "Failed to parse window dimensions", + "raw_output": result_str + } + + window_x = int(x_match.group(1)) + window_y = int(y_match.group(1)) + window_width = int(width_match.group(1)) + window_height = int(height_match.group(1)) + + # Extract app info if available + app_match = re.search(r'"app":"([^"]*)"', result_str) + bundle_match = re.search(r'"bundle":"([^"]*)"', result_str) + app_name = app_match.group(1) if app_match else "Unknown" + bundle_id = bundle_match.group(1) if bundle_match else "Unknown" + + print(f"🔍 Detected {app_name} window: {window_width}x{window_height} at ({window_x}, {window_y})") + print(f" Bundle ID: {bundle_id}") + + # Calculate device content area with smart detection based on window size + # Different calculation strategies based on window size + if window_width < 500 and window_height < 1000: + # Small window - minimal padding + title_bar_height = 28 + content_padding_h = 20 # horizontal padding + content_padding_v = 20 # vertical padding + elif window_width < 800 and window_height < 1400: + # Medium window - moderate padding + title_bar_height = 28 + content_padding_h = 40 + content_padding_v = 50 + else: + # Large window - more padding + title_bar_height = 28 + content_padding_h = 80 + content_padding_v = 100 + + # Calculate the actual iOS device screen area within the window + content_x = window_x + content_padding_h // 2 + content_y = window_y + title_bar_height + content_padding_v // 2 + content_width = window_width - content_padding_h + content_height = window_height - title_bar_height - content_padding_v + + # Ensure minimum viable dimensions + if content_width < 200 or content_height < 400: + # Try with minimal padding if initial calculation is too small + content_x = window_x + 10 + content_y = window_y + title_bar_height + 10 + content_width = window_width - 20 + content_height = window_height - title_bar_height - 20 + + if content_width < 200 or content_height < 400: + return { + "status": "error", + "error": "Detected window seems too small for iPhone content", + "window_size": [window_width, window_height], + "calculated_content": [content_width, content_height], + "suggestion": "Try making the iPhone Mirroring window larger" + } + + # Auto-configure the mapping + setup_ios_mapping(content_x, content_y, content_width, content_height) + + # Verify configuration was set + print(f"✅ iOS mapping auto-configured successfully!") + print(f" Enabled: {ios_config['enabled']}") + print(f" Device estimation: {ios_config['estimated_ios_width']}x{ios_config['estimated_ios_height']}") + print(f" Mirror area: {ios_config['mirror_width']}x{ios_config['mirror_height']} at ({ios_config['mirror_x']}, {ios_config['mirror_y']})") + + return { + "status": "ok", + "action": "detect_ios_mirror", + "window_detected": { + "x": window_x, + "y": window_y, + "width": window_width, + "height": window_height, + "app_name": app_name, + "bundle_id": bundle_id + }, + "content_area": { + "x": content_x, + "y": content_y, + "width": content_width, + "height": content_height + }, + "config": ios_config, + "message": f"Successfully auto-configured for {ios_config['estimated_ios_width']}x{ios_config['estimated_ios_height']} device" + } + + except Exception as e: + return { + "status": "error", + "error": f"Exception during auto-detection: {str(e)}", + "traceback": traceback.format_exc() + } + def execute_applescript(script): """Execute AppleScript command""" try: @@ -66,8 +232,9 @@ def setup_ios_mapping(mirror_x, mirror_y, mirror_width, mirror_height): "estimated_ios_height": best_match["height"] }) - print(f"iOS mapping configured: Estimated {best_match['name']} ({best_match['width']}x{best_match['height']}) -> {mirror_width}x{mirror_height} at ({mirror_x},{mirror_y})") - print(f"Aspect ratio: {mirror_aspect_ratio:.3f}, Device: {best_match['name']}") + print(f"📱 iOS mapping configured: Estimated {best_match['name']} ({best_match['width']}x{best_match['height']}) -> {mirror_width}x{mirror_height} at ({mirror_x},{mirror_y})") + print(f" Aspect ratio: {mirror_aspect_ratio:.3f}, Device: {best_match['name']}") + print(f" ✅ iOS coordinate transformation is now ENABLED") def transform_ios_coordinates(ios_x, ios_y): """Transform iOS coordinates to macOS screen coordinates""" @@ -284,14 +451,40 @@ def handle_action(action): elif act == "screenshot": # Take screenshot of iOS region if mapping is enabled region = get_ios_screenshot_region() + print(f"📸 Taking screenshot with region: {region}") + if region: + print(f" 📱 iOS screenshot region: x={region[0]}, y={region[1]}, w={region[2]}, h={region[3]}") screenshot = pyautogui.screenshot(region=region) else: + print(f" 🖥️ Full screen screenshot (iOS config not enabled)") screenshot = pyautogui.screenshot() - # Save to temporary file and return path + + # Convert screenshot to base64 for web frontend + buffer = io.BytesIO() + screenshot.save(buffer, format='PNG') + buffer.seek(0) + + # Create base64 data URL + img_base64 = base64.b64encode(buffer.read()).decode('utf-8') + data_url = f"data:image/png;base64,{img_base64}" + + # Also save to temporary file as backup temp_path = f"/tmp/screenshot_{int(time.time())}.png" screenshot.save(temp_path) - return {"status": "ok", "action": "screenshot", "path": temp_path, "ios_region": region is not None} + + print(f" ✅ Screenshot saved: {temp_path}") + print(f" 📊 Screenshot size: {screenshot.size[0]}x{screenshot.size[1]}") + + return { + "status": "ok", + "action": "screenshot", + "path": temp_path, + "data_url": data_url, + "ios_region": region is not None, + "region_info": region if region else None, + "screenshot_size": {"width": screenshot.size[0], "height": screenshot.size[1]} + } elif act == "get_screen_size": if ios_config["enabled"]: @@ -308,6 +501,11 @@ def handle_action(action): setup_ios_mapping(mirror_x, mirror_y, mirror_width, mirror_height) return {"status": "ok", "action": "configure_ios", "config": ios_config} + elif act == "detect_ios_mirror": + """Automatically detect iPhone Mirroring app window and configure mapping""" + result = detect_and_configure_ios_mirror() + return result + elif act == "sleep": seconds = float(action["seconds"]) time.sleep(seconds) @@ -340,6 +538,19 @@ def health_check(): "error": str(e) }), 500 +@app.route("/detect", methods=["POST"]) +def detect_ios_mirror(): + """Auto-detect and configure iOS device mapping""" + try: + result = handle_action({"action": "detect_ios_mirror"}) + return jsonify(result) + except Exception as e: + return jsonify({ + "status": "error", + "error": str(e), + "traceback": traceback.format_exc() + }) + @app.route("/configure", methods=["POST"]) def configure_ios(): """Configure iOS device mapping""" diff --git a/packages/ios/setup.sh b/packages/ios/setup.sh index 9cd0e87e9..5a75b1cc2 100755 --- a/packages/ios/setup.sh +++ b/packages/ios/setup.sh @@ -31,5 +31,4 @@ echo "✅ iOS package setup completed!" echo "" echo "To test the setup:" echo "1. Start the server: npm run server" -echo "2. Test the server: python3 idb/test.py" -echo "3. Run example: npm run example" +echo "2. Run example: npm run example" diff --git a/packages/ios/src/agent/index.ts b/packages/ios/src/agent/index.ts index 0526ad67e..2ebf5b9c4 100644 --- a/packages/ios/src/agent/index.ts +++ b/packages/ios/src/agent/index.ts @@ -8,23 +8,20 @@ type iOSAgentOpt = PageAgentOpt; export class iOSAgent extends PageAgent { declare page: iOSDevice; + private connectionPromise: Promise | null = null; - async launch(uri: string): Promise { - const device = this.page; - await device.launch(uri); + constructor(page: iOSDevice, opts?: iOSAgentOpt) { + super(page, opts); + this.ensureConnected(); } - async back(): Promise { - await this.page.back(); + private ensureConnected(): Promise { + if (!this.connectionPromise) { + this.connectionPromise = this.page.connect(); + } + return this.connectionPromise; } - async home(): Promise { - await this.page.home(); - } - - async recentApps(): Promise { - await this.page.recentApps(); - } } export async function agentFromPyAutoGUI(opts?: iOSAgentOpt & iOSDeviceOpt) { diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 8cb8e310b..657b99c1c 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -80,11 +80,14 @@ export class iOSDevice implements AndroidDevicePage { uri: string | undefined; options?: iOSDeviceOpt; private serverUrl: string; + private serverPort: number; + private serverProcess?: any; // Store reference to server process constructor(options?: iOSDeviceOpt) { this.options = options; + this.serverPort = options?.serverPort || 1412; this.serverUrl = - options?.serverUrl || `http://localhost:${options?.serverPort || 1412}`; + options?.serverUrl || `http://localhost:${this.serverPort}`; } public async connect(): Promise { @@ -102,36 +105,131 @@ export class iOSDevice implements AndroidDevicePage { } const healthData = await response.json(); debugPage(`Python server is running: ${JSON.stringify(healthData)}`); - - // Make iPhone mirroring app foreground + } catch (error: any) { + debugPage(`Python server connection failed: ${error.message}`); + + // Try to start server automatically + debugPage('Attempting to start Python server automatically...'); + try { - // Use fixed mirroring app name for iOS device screen mirroring - const mirroringAppName = 'iPhone Mirroring'; - - const { exec } = await import('node:child_process'); - const { promisify } = await import('node:util'); - const execAsync = promisify(exec); - - // Activate the mirroring application using AppleScript - await execAsync( - `osascript -e 'tell application "${mirroringAppName}" to activate'`, - ); - debugPage(`Activated iOS mirroring app: ${mirroringAppName}`); - } catch (mirrorError: any) { - debugPage( - `Warning: Failed to bring iOS mirroring app to foreground: ${mirrorError.message}`, + await this.startPyAutoGUIServer(); + debugPage('Python server started successfully'); + + // Verify server is now running + const response = await fetch(`${this.serverUrl}/health`); + if (!response.ok) { + throw new Error(`Server still not responding after startup: ${response.status}`); + } + + const healthData = await response.json(); + debugPage(`Python server is now running: ${JSON.stringify(healthData)}`); + } catch (startError: any) { + throw new Error( + `Failed to auto-start Python server: ${startError.message}. ` + + `Please manually start the server by running: node packages/ios/bin/server.js ${this.serverPort}` ); - // Continue execution even if this fails - it's not critical } - } catch (error: any) { - throw new Error( - `Failed to connect to Python server at ${this.serverUrl}: ${error.message}`, + } + + // Make iPhone mirroring app foreground + try { + // Use fixed mirroring app name for iOS device screen mirroring + const mirroringAppName = 'iPhone Mirroring'; + + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + // Activate the mirroring application using AppleScript + await execAsync( + `osascript -e 'tell application "${mirroringAppName}" to activate'`, + ); + debugPage(`Activated iOS mirroring app: ${mirroringAppName}`); + } catch (mirrorError: any) { + debugPage( + `Warning: Failed to bring iOS mirroring app to foreground: ${mirrorError.message}`, ); + // Continue execution even if this fails - it's not critical } // Configure iOS mirroring if provided + await this.initializeMirrorConfiguration(); + + } + + private async startPyAutoGUIServer(): Promise { + try { + const { spawn } = await import('node:child_process'); + const serverScriptPath = path.resolve(__dirname, '../../bin/server.js'); + + debugPage(`Starting PyAutoGUI server using: node ${serverScriptPath} ${this.serverPort}`); + + // Start server process in background (similar to server.js background mode) + this.serverProcess = spawn('node', [serverScriptPath, this.serverPort.toString()], { + detached: true, + stdio: 'pipe', // Capture output + env: { + ...process.env, + }, + }); + + // Handle server process events + this.serverProcess.on('error', (error: any) => { + debugPage(`Server process error: ${error.message}`); + }); + + this.serverProcess.on('exit', (code: number, signal: string) => { + debugPage(`Server process exited with code ${code}, signal ${signal}`); + }); + + // Capture and log server output + if (this.serverProcess.stdout) { + this.serverProcess.stdout.on('data', (data: Buffer) => { + debugPage(`Server stdout: ${data.toString().trim()}`); + }); + } + + if (this.serverProcess.stderr) { + this.serverProcess.stderr.on('data', (data: Buffer) => { + debugPage(`Server stderr: ${data.toString().trim()}`); + }); + } + + debugPage(`Started PyAutoGUI server process with PID: ${this.serverProcess.pid}`); + + // Wait for server to start up (similar to server.js timeout) + await sleep(3000); + + } catch (error: any) { + throw new Error(`Failed to start PyAutoGUI server: ${error.message}`); + } + } + + private async initializeMirrorConfiguration() { if (this.options?.mirrorConfig) { await this.configureIOSMirror(this.options.mirrorConfig); + } else { + try { + // Auto-detect iPhone Mirroring app window using AppleScript + const mirrorConfig = await this.detectAndConfigureIOSMirror(); + if (mirrorConfig) { + if (!this.options || typeof this.options.mirrorConfig !== 'object') { + this.options = {}; + } + this.options.mirrorConfig = mirrorConfig; + + debugPage( + `Auto-detected iOS mirror config: ${mirrorConfig.mirrorWidth}x${mirrorConfig.mirrorHeight} at (${mirrorConfig.mirrorX}, ${mirrorConfig.mirrorY})` + ); + + // Configure the detected mirror settings + await this.configureIOSMirror(mirrorConfig); + } else { + debugPage('No iPhone Mirroring app found or auto-detection failed'); + } + } catch (error: any) { + debugPage(`Failed to auto-detect iPhone Mirroring app: ${error.message}`); + } } // Get screen information (will use iOS dimensions if configured) @@ -173,6 +271,128 @@ export class iOSDevice implements AndroidDevicePage { } } + private async detectAndConfigureIOSMirror(): Promise<{ + mirrorX: number; + mirrorY: number; + mirrorWidth: number; + mirrorHeight: number; + } | null> { + try { + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); + + // AppleScript to get window information for iPhone Mirroring app + const applescript = ` + tell application "System Events" + try + set mirrorApp to first application process whose name contains "iPhone Mirroring" + set mirrorWindow to first window of mirrorApp + set windowPosition to position of mirrorWindow + set windowSize to size of mirrorWindow + + -- Get window frame information + set windowX to item 1 of windowPosition + set windowY to item 2 of windowPosition + set windowWidth to item 1 of windowSize + set windowHeight to item 2 of windowSize + + -- Try to get the actual visible frame (content area) + try + set appName to name of mirrorApp + set bundleId to bundle identifier of mirrorApp + set visibleFrame to "{\\"found\\":true,\\"x\\":" & windowX & ",\\"y\\":" & windowY & ",\\"width\\":" & windowWidth & ",\\"height\\":" & windowHeight & ",\\"app\\":\\"" & appName & "\\",\\"bundle\\":\\"" & bundleId & "\\"}" + return visibleFrame + on error + return "{\\"found\\":true,\\"x\\":" & windowX & ",\\"y\\":" & windowY & ",\\"width\\":" & windowWidth & ",\\"height\\":" & windowHeight & "}" + end try + + on error errMsg + return "{\\"found\\":false,\\"error\\":\\"" & errMsg & "\\"}" + end try + end tell + `; + + const { stdout, stderr } = await execAsync(`osascript -e '${applescript}'`); + + if (stderr) { + debugPage(`AppleScript error: ${stderr}`); + return null; + } + + const result = JSON.parse(stdout.trim()); + + if (!result.found) { + debugPage(`iPhone Mirroring app not found: ${result.error || 'Unknown error'}`); + return null; + } + + const windowX = result.x; + const windowY = result.y; + const windowWidth = result.width; + const windowHeight = result.height; + + debugPage(`Detected iPhone Mirroring window: ${windowWidth}x${windowHeight} at (${windowX}, ${windowY})`); + + // Calculate device content area with smart detection based on window size + let titleBarHeight = 28; + let contentPaddingH, contentPaddingV; + + if (windowWidth < 500 && windowHeight < 1000) { + // Small window - minimal padding + contentPaddingH = 20; + contentPaddingV = 20; + } else if (windowWidth < 800 && windowHeight < 1400) { + // Medium window - moderate padding + contentPaddingH = 40; + contentPaddingV = 50; + } else { + // Large window - more padding + contentPaddingH = 80; + contentPaddingV = 100; + } + + // Calculate the actual iOS device screen area within the window + const contentX = windowX + Math.floor(contentPaddingH / 2); + const contentY = windowY + titleBarHeight + Math.floor(contentPaddingV / 2); + const contentWidth = windowWidth - contentPaddingH; + const contentHeight = windowHeight - titleBarHeight - contentPaddingV; + + // Ensure minimum viable dimensions + if (contentWidth < 200 || contentHeight < 400) { + // Try with minimal padding if initial calculation is too small + const minimalContentX = windowX + 10; + const minimalContentY = windowY + titleBarHeight + 10; + const minimalContentWidth = windowWidth - 20; + const minimalContentHeight = windowHeight - titleBarHeight - 20; + + if (minimalContentWidth < 200 || minimalContentHeight < 400) { + debugPage(`Detected window seems too small for iPhone content: ${windowWidth}x${windowHeight}`); + return null; + } + + return { + mirrorX: minimalContentX, + mirrorY: minimalContentY, + mirrorWidth: minimalContentWidth, + mirrorHeight: minimalContentHeight, + }; + } + + debugPage(`Calculated content area: ${contentWidth}x${contentHeight} at (${contentX}, ${contentY})`); + + return { + mirrorX: contentX, + mirrorY: contentY, + mirrorWidth: contentWidth, + mirrorHeight: contentHeight, + }; + } catch (error: any) { + debugPage(`Exception during iPhone Mirroring app detection: ${error.message}`); + return null; + } + } + async getConfiguration(): Promise { const response = await fetch(`${this.serverUrl}/config`); if (!response.ok) { @@ -866,6 +1086,17 @@ export class iOSDevice implements AndroidDevicePage { async destroy(): Promise { debugPage('destroy iOS device'); this.destroyed = true; + + // Clean up server process if we started it + if (this.serverProcess) { + try { + debugPage('Terminating PyAutoGUI server process'); + this.serverProcess.kill('SIGTERM'); + this.serverProcess = undefined; + } catch (error) { + debugPage('Error terminating server process:', error); + } + } } // Additional abstract methods from AbstractPage diff --git a/packages/ios/src/utils/index.ts b/packages/ios/src/utils/index.ts index b25b9c8d9..52e8ab780 100644 --- a/packages/ios/src/utils/index.ts +++ b/packages/ios/src/utils/index.ts @@ -81,10 +81,17 @@ export async function getScreenSize(): Promise { export async function startPyAutoGUIServer(port = 1412): Promise { const { spawn } = await import('node:child_process'); const path = await import('node:path'); - const { fileURLToPath } = await import('node:url'); - - const __dirname = path.dirname(fileURLToPath(import.meta.url)); - const serverPath = path.join(__dirname, '../../idb/auto_server.py'); + + // Use __dirname in a way that works for both ESM and CommonJS + let currentDir: string; + if (typeof __dirname !== 'undefined') { + currentDir = __dirname; + } else { + const { fileURLToPath } = await import('node:url'); + currentDir = path.dirname(fileURLToPath(import.meta.url)); + } + + const serverPath = path.join(currentDir, '../../idb/auto_server.py'); const server = spawn('python3', [serverPath], { stdio: 'inherit', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ebc04cbc0..eaf76ea2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -239,6 +239,70 @@ importers: specifier: ^5.8.3 version: 5.8.3 + apps/ios-playground: + dependencies: + '@ant-design/icons': + specifier: ^5.3.1 + version: 5.5.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@midscene/core': + specifier: workspace:* + version: link:../../packages/core + '@midscene/ios': + specifier: workspace:* + version: link:../../packages/ios + '@midscene/shared': + specifier: workspace:* + version: link:../../packages/shared + '@midscene/visualizer': + specifier: workspace:* + version: link:../../packages/visualizer + '@midscene/web': + specifier: workspace:* + version: link:../../packages/web-integration + antd: + specifier: ^5.21.6 + version: 5.21.6(moment@2.30.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + dayjs: + specifier: ^1.11.11 + version: 1.11.13 + react: + specifier: 18.3.1 + version: 18.3.1 + react-dom: + specifier: 18.3.1 + version: 18.3.1(react@18.3.1) + devDependencies: + '@rsbuild/core': + specifier: ^1.3.22 + version: 1.4.13 + '@rsbuild/plugin-less': + specifier: ^1.2.4 + version: 1.2.4(@rsbuild/core@1.4.13) + '@rsbuild/plugin-node-polyfill': + specifier: 1.3.0 + version: 1.3.0(@rsbuild/core@1.4.13) + '@rsbuild/plugin-react': + specifier: ^1.3.1 + version: 1.3.4(@rsbuild/core@1.4.13) + '@rsbuild/plugin-svgr': + specifier: ^1.1.1 + version: 1.2.0(@rsbuild/core@1.4.13)(typescript@5.8.3) + '@types/react': + specifier: ^18.3.1 + version: 18.3.23 + '@types/react-dom': + specifier: ^18.3.1 + version: 18.3.7(@types/react@18.3.23) + archiver: + specifier: ^6.0.0 + version: 6.0.2 + less: + specifier: ^4.2.0 + version: 4.3.0 + typescript: + specifier: ^5.8.3 + version: 5.8.3 + apps/recorder-form: dependencies: '@midscene/recorder': @@ -665,6 +729,43 @@ importers: specifier: ^2.1.8 version: 2.1.9(@types/node@22.15.3)(jsdom@26.1.0)(less@4.3.0)(lightningcss@1.30.1)(sass-embedded@1.86.3)(terser@5.43.1) + packages/ios-playground: + dependencies: + '@midscene/ios': + specifier: workspace:* + version: link:../ios + '@midscene/shared': + specifier: workspace:* + version: link:../shared + '@midscene/web': + specifier: workspace:* + version: link:../web-integration + cors: + specifier: 2.8.5 + version: 2.8.5 + express: + specifier: ^4.21.2 + version: 4.21.2 + open: + specifier: 10.1.0 + version: 10.1.0 + devDependencies: + '@modern-js/module-tools': + specifier: 2.60.6 + version: 2.60.6(debug@4.4.0)(typescript@5.8.3) + '@types/cors': + specifier: 2.8.12 + version: 2.8.12 + '@types/express': + specifier: ^4.17.21 + version: 4.17.21 + '@types/node': + specifier: ^18.0.0 + version: 18.19.62 + typescript: + specifier: ^5.8.3 + version: 5.8.3 + packages/mcp: dependencies: puppeteer-core: @@ -7713,10 +7814,6 @@ packages: resolution: {integrity: sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==} engines: {node: '>=10.13.0'} - enhanced-resolve@5.18.0: - resolution: {integrity: sha512-0/r0MySGYG8YqlayBZ6MuCfECmHFdJ5qyPh8s8wa5Hnm6SaFLSK1VYCbj+NKp090Nm1caZhD+QTnmxO7esYGyQ==} - engines: {node: '>=10.13.0'} - enhanced-resolve@5.18.1: resolution: {integrity: sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==} engines: {node: '>=10.13.0'} @@ -8149,9 +8246,6 @@ packages: debug: optional: true - for-each@0.3.3: - resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} - for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -8964,10 +9058,6 @@ packages: is-function@1.0.2: resolution: {integrity: sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==} - is-generator-function@1.0.10: - resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} - engines: {node: '>= 0.4'} - is-generator-function@1.1.0: resolution: {integrity: sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==} engines: {node: '>= 0.4'} @@ -9083,10 +9173,6 @@ packages: resolution: {integrity: sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw==} engines: {node: '>=8'} - is-typed-array@1.1.13: - resolution: {integrity: sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==} - engines: {node: '>= 0.4'} - is-typed-array@1.1.15: resolution: {integrity: sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==} engines: {node: '>= 0.4'} @@ -13524,10 +13610,6 @@ packages: resolution: {integrity: sha512-MOiaDbA5ZZgUjkeMWM5EkJp4loW5ZRoa5bc3/aeMox/PJelMhE6t7S/mLuiY43DBupyxH+S0U1bTui9kWUlmsw==} engines: {node: '>=8.15'} - which-typed-array@1.1.15: - resolution: {integrity: sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==} - engines: {node: '>= 0.4'} - which-typed-array@1.1.19: resolution: {integrity: sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==} engines: {node: '>= 0.4'} @@ -17971,8 +18053,8 @@ snapshots: dependencies: '@ampproject/remapping': 2.3.0 '@ast-grep/napi': 0.16.0 - '@babel/core': 7.26.0 - '@babel/types': 7.26.8 + '@babel/core': 7.26.10 + '@babel/types': 7.28.1 '@modern-js/core': 2.60.6 '@modern-js/plugin': 2.60.6 '@modern-js/plugin-changeset': 2.60.6(debug@4.4.0) @@ -17993,7 +18075,7 @@ snapshots: style-inject: 0.3.0 sucrase: 3.29.0 tapable: 2.2.1 - terser: 5.36.0 + terser: 5.43.1 tsconfig-paths-webpack-plugin: 4.1.0 optionalDependencies: typescript: 5.8.3 @@ -19237,6 +19319,12 @@ snapshots: deepmerge: 4.3.1 reduce-configs: 1.1.0 + '@rsbuild/plugin-less@1.2.4(@rsbuild/core@1.4.13)': + dependencies: + '@rsbuild/core': 1.4.13 + deepmerge: 4.3.1 + reduce-configs: 1.1.0 + '@rsbuild/plugin-node-polyfill@1.3.0(@rsbuild/core@1.3.22)': dependencies: assert: 2.1.0 @@ -19265,6 +19353,34 @@ snapshots: optionalDependencies: '@rsbuild/core': 1.3.22 + '@rsbuild/plugin-node-polyfill@1.3.0(@rsbuild/core@1.4.13)': + dependencies: + assert: 2.1.0 + browserify-zlib: 0.2.0 + buffer: 5.7.1 + console-browserify: 1.2.0 + constants-browserify: 1.0.0 + crypto-browserify: 3.12.1 + domain-browser: 5.7.0 + events: 3.3.0 + https-browserify: 1.0.0 + os-browserify: 0.3.0 + path-browserify: 1.0.1 + process: 0.11.10 + punycode: 2.3.1 + querystring-es3: 0.2.1 + readable-stream: 4.7.0 + stream-browserify: 3.0.0 + stream-http: 3.2.0 + string_decoder: 1.3.0 + timers-browserify: 2.0.12 + tty-browserify: 0.0.1 + url: 0.11.4 + util: 0.12.5 + vm-browserify: 1.1.2 + optionalDependencies: + '@rsbuild/core': 1.4.13 + '@rsbuild/plugin-react@1.3.1(@rsbuild/core@1.3.22)': dependencies: '@rsbuild/core': 1.3.22 @@ -19303,6 +19419,20 @@ snapshots: - typescript - webpack-hot-middleware + '@rsbuild/plugin-svgr@1.2.0(@rsbuild/core@1.4.13)(typescript@5.8.3)': + dependencies: + '@rsbuild/core': 1.4.13 + '@rsbuild/plugin-react': 1.3.1(@rsbuild/core@1.4.13) + '@svgr/core': 8.1.0(typescript@5.8.3) + '@svgr/plugin-jsx': 8.1.0(@svgr/core@8.1.0(typescript@5.8.3)) + '@svgr/plugin-svgo': 8.1.0(@svgr/core@8.1.0(typescript@5.8.3))(typescript@5.8.3) + deepmerge: 4.3.1 + loader-utils: 3.3.1 + transitivePeerDependencies: + - supports-color + - typescript + - webpack-hot-middleware + '@rsbuild/plugin-type-check@1.2.3(@rsbuild/core@1.3.22)(@rspack/core@1.4.11)(typescript@5.8.3)': dependencies: deepmerge: 4.3.1 @@ -22655,11 +22785,6 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.2.1 - enhanced-resolve@5.18.0: - dependencies: - graceful-fs: 4.2.11 - tapable: 2.2.1 - enhanced-resolve@5.18.1: dependencies: graceful-fs: 4.2.11 @@ -23315,10 +23440,6 @@ snapshots: optionalDependencies: debug: 4.4.0(supports-color@5.5.0) - for-each@0.3.3: - dependencies: - is-callable: 1.2.7 - for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -24332,7 +24453,7 @@ snapshots: is-arguments@1.1.1: dependencies: - call-bind: 1.0.7 + call-bind: 1.0.8 has-tostringtag: 1.0.2 is-array-buffer@3.0.5: @@ -24415,10 +24536,6 @@ snapshots: is-function@1.0.2: {} - is-generator-function@1.0.10: - dependencies: - has-tostringtag: 1.0.2 - is-generator-function@1.1.0: dependencies: call-bound: 1.0.4 @@ -24520,10 +24637,6 @@ snapshots: dependencies: text-extensions: 2.4.0 - is-typed-array@1.1.13: - dependencies: - which-typed-array: 1.1.15 - is-typed-array@1.1.15: dependencies: which-typed-array: 1.1.19 @@ -24877,7 +24990,6 @@ snapshots: mime: 1.6.0 needle: 3.3.1 source-map: 0.6.1 - optional: true levdist@1.0.0: {} @@ -25097,7 +25209,7 @@ snapshots: magic-string@0.30.12: dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.4 magic-string@0.30.17: dependencies: @@ -29058,7 +29170,7 @@ snapshots: glob: 7.1.6 lines-and-columns: 1.2.4 mz: 2.7.0 - pirates: 4.0.6 + pirates: 4.0.7 ts-interface-checker: 0.1.13 supports-color@5.5.0: @@ -29385,7 +29497,7 @@ snapshots: tsconfig-paths-webpack-plugin@4.1.0: dependencies: chalk: 4.1.2 - enhanced-resolve: 5.18.0 + enhanced-resolve: 5.18.2 tsconfig-paths: 4.2.0 tsconfig-paths@4.2.0: @@ -29686,9 +29798,9 @@ snapshots: dependencies: inherits: 2.0.4 is-arguments: 1.1.1 - is-generator-function: 1.0.10 - is-typed-array: 1.1.13 - which-typed-array: 1.1.15 + is-generator-function: 1.1.0 + is-typed-array: 1.1.15 + which-typed-array: 1.1.19 utils-merge@1.0.1: {} @@ -30162,14 +30274,6 @@ snapshots: load-yaml-file: 0.2.0 path-exists: 4.0.0 - which-typed-array@1.1.15: - dependencies: - available-typed-arrays: 1.0.7 - call-bind: 1.0.7 - for-each: 0.3.3 - gopd: 1.2.0 - has-tostringtag: 1.0.2 - which-typed-array@1.1.19: dependencies: available-typed-arrays: 1.0.7 From 1908f1d1b75367f8bf64b5f9913ac64527ebf00d Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Fri, 15 Aug 2025 00:23:24 +0800 Subject: [PATCH 13/17] feat(ios): implement action space for iOS devices --- packages/ios/src/page/index.ts | 157 +++++++++++++++++++++------------ 1 file changed, 103 insertions(+), 54 deletions(-) diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 2474d47e8..6205fdeba 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import type { Point, Size } from '@midscene/core'; -import type { PageType } from '@midscene/core'; +import type { DeviceAction, ExecutorContext, PageType } from '@midscene/core'; import { getTmpFile, sleep } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; import { resizeImg } from '@midscene/shared/img'; @@ -9,62 +9,11 @@ import { getDebug } from '@midscene/shared/logger'; import { type AndroidDeviceInputOpt, type AndroidDevicePage, - commonWebActions, } from '@midscene/web'; +import { commonWebActionsForWebPage } from '@midscene/web/utils'; import { type ScreenInfo, getScreenSize } from '../utils'; export const debugPage = getDebug('ios:device'); - -// Temporary DeviceAction interface definition - should match @midscene/core types -interface DeviceAction { - name: string; - description?: string; - paramSchema?: string; - paramDescription?: string; - location?: 'required' | 'optional' | false; - whatToLocate?: string; // what to locate if location is required or optional - call: (param: ParamType) => Promise | void; -} - -const asyncNoop = async () => {}; -const iOSActions: DeviceAction[] = [ - { - name: 'AndroidBackButton', - description: 'Trigger the system "back" operation on iOS devices using CMD+[', - location: false, - call: asyncNoop, - }, - { - name: 'AndroidHomeButton', - description: 'Trigger the system "home" operation on iOS devices using CMD+1', - location: false, - call: asyncNoop, - }, - { - name: 'AndroidRecentAppsButton', - description: 'Trigger the system "recent apps" operation on iOS devices using CMD+2', - location: false, - call: asyncNoop, - }, - { - name: 'AndroidLongPress', - description: 'Long press operation on iOS devices', - paramSchema: '{ duration?: number }', - paramDescription: 'The duration of the long press', - location: 'optional', - whatToLocate: 'The element to be long pressed', - call: asyncNoop, - }, - { - name: 'AndroidPull', - description: 'Pull gesture (swipe) operation on iOS devices, can be pull up or pull down', - location: false, - paramSchema: '{ direction: "up" | "down" }', - paramDescription: 'Pull direction: up or down', - call: asyncNoop, - }, -]; - export interface iOSDeviceOpt extends AndroidDeviceInputOpt { serverUrl?: string; serverPort?: number; @@ -143,9 +92,109 @@ export class iOSDevice implements AndroidDevicePage { } actionSpace(): DeviceAction[] { - return commonWebActions.concat(iOSActions); + const commonActions = commonWebActionsForWebPage(this); + commonActions.forEach((action) => { + if (action.name === 'Input') { + action.call = async (context, param) => { + const { element } = context; + if (element) { + await this.clearInput(element as unknown as ElementInfo); + + if (!param || !param.value) { + return; + } + } + + await this.keyboard.type(param.value, { + autoDismissKeyboard: + param.autoDismissKeyboard ?? this.options?.autoDismissKeyboard, + }); + }; + } + }); + + const allActions: DeviceAction[] = [ + ...commonWebActionsForWebPage(this), + { + name: 'IOSBackButton', + description: 'Trigger the system "back" operation on iOS devices', + location: false, + call: async (context, param) => { + await this.back(); + }, + }, + { + name: 'IOSHomeButton', + description: 'Trigger the system "home" operation on iOS devices', + location: false, + call: async (context, param) => { + await this.home(); + }, + }, + { + name: 'IOSRecentAppsButton', + description: + 'Trigger the system "recent apps" operation on iOS devices', + location: false, + call: async (context, param) => { + await this.recentApps(); + }, + }, + { + name: 'IOSLongPress', + description: + 'Trigger a long press on the screen at specified coordinates on iOS devices', + paramSchema: '{ duration?: number }', + paramDescription: 'The duration of the long press in milliseconds', + location: 'required', + whatToLocate: 'The element to be long pressed', + call: async (context, param) => { + const { element } = context; + if (!element) { + throw new Error( + 'IOSLongPress requires an element to be located', + ); + } + const [x, y] = element.center; + await this.longPress(x, y, param?.duration); + }, + } as DeviceAction<{ duration?: number }>, + { + name: 'IOSPull', + description: + 'Trigger pull down to refresh or pull up actions on iOS devices', + paramSchema: + '{ direction: "up" | "down", distance?: number, duration?: number }', + paramDescription: + 'The direction to pull, the distance to pull (in pixels), and the duration of the pull (in milliseconds).', + location: 'optional', + whatToLocate: 'The element to be pulled', + call: async (context, param) => { + const { element } = context; + const startPoint = element + ? { left: element.center[0], top: element.center[1] } + : undefined; + if (!param || !param.direction) { + throw new Error('IOSPull requires a direction parameter'); + } + if (param.direction === 'down') { + await this.pullDown(startPoint, param.distance, param.duration); + } else if (param.direction === 'up') { + await this.pullUp(startPoint, param.distance, param.duration); + } else { + throw new Error(`Unknown pull direction: ${param.direction}`); + } + }, + } as DeviceAction<{ + direction: 'up' | 'down'; + distance?: number; + duration?: number; + }>, + ]; + return allActions; } + public async connect(): Promise { if (this.destroyed) { throw new Error('iOSDevice has been destroyed and cannot be used'); From f175408e01c0dfc17c93638d46865b9c685e8f7d Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Fri, 15 Aug 2025 21:47:50 +0800 Subject: [PATCH 14/17] refactor(ios): use resizeImgBuffer instead of resizeImg --- packages/ios/src/page/index.ts | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index 6205fdeba..f3b4eb5fc 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -4,7 +4,7 @@ import type { Point, Size } from '@midscene/core'; import type { DeviceAction, ExecutorContext, PageType } from '@midscene/core'; import { getTmpFile, sleep } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; -import { resizeImg } from '@midscene/shared/img'; +import { resizeImgBuffer } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { type AndroidDeviceInputOpt, @@ -607,10 +607,14 @@ export class iOSDevice implements AndroidDevicePage { const { width, height } = await this.size(); // Resize to match iOS device dimensions - const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { - width, - height, - }); + const { buffer: resizedScreenshotBuffer } = await resizeImgBuffer( + 'png', + screenshotBuffer, + { + width, + height, + } + ); // Clean up temporary file try { @@ -639,10 +643,14 @@ export class iOSDevice implements AndroidDevicePage { const screenshotBuffer = await fs.promises.readFile(tempPath); const { width, height } = await this.size(); - const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { - width, - height, - }); + const { buffer: resizedScreenshotBuffer } = await resizeImgBuffer( + 'png', + screenshotBuffer, + { + width, + height, + } + ); debugPage('screenshotBase64 end (via screencapture)'); return `data:image/png;base64,${resizedScreenshotBuffer.toString('base64')}`; From 9fecc8025c3ac86bac8afc6fd7d2e792b135bc61 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Sat, 16 Aug 2025 00:11:11 +0800 Subject: [PATCH 15/17] refactor(ios): improve code quality and formatting --- apps/ios-playground/rsbuild.config.ts | 8 +- apps/ios-playground/src/App.tsx | 27 +- apps/ios-playground/src/ios-device/index.tsx | 214 ++++++------ apps/ios-playground/src/ios-player/index.tsx | 319 +++++++++--------- .../src/scripts/blank_polyfill.ts | 2 +- packages/ios-playground/bin/server.js | 252 +++++++------- packages/ios-playground/modern.config.ts | 2 +- packages/ios-playground/test-health.js | 36 +- packages/ios/bin/server.js | 4 +- packages/ios/examples/ios-mirroring-demo.js | 2 +- packages/ios/src/agent/index.ts | 1 - packages/ios/src/page/index.ts | 116 ++++--- packages/ios/src/utils/index.ts | 4 +- 13 files changed, 519 insertions(+), 468 deletions(-) diff --git a/apps/ios-playground/rsbuild.config.ts b/apps/ios-playground/rsbuild.config.ts index 1449447f1..bf8a29d6b 100644 --- a/apps/ios-playground/rsbuild.config.ts +++ b/apps/ios-playground/rsbuild.config.ts @@ -61,10 +61,10 @@ export default defineConfig({ target: 'http://localhost:1412', changeOrigin: true, pathRewrite: { - '^/api/pyautogui': '' - } - } - } + '^/api/pyautogui': '', + }, + }, + }, }, resolve: { alias: { diff --git a/apps/ios-playground/src/App.tsx b/apps/ios-playground/src/App.tsx index d44e74a28..51c6aac33 100644 --- a/apps/ios-playground/src/App.tsx +++ b/apps/ios-playground/src/App.tsx @@ -115,15 +115,10 @@ export default function App() { try { // Use a fixed context string for iOS since we don't have device selection - const res = await requestPlaygroundServer( - 'ios-device', - type, - prompt, - { - requestId: thisRunningId, - deepThink, - }, - ); + const res = await requestPlaygroundServer('ios-device', type, prompt, { + requestId: thisRunningId, + deepThink, + }); // stop polling clearPollingInterval(); @@ -144,7 +139,6 @@ export default function App() { setReplayScriptsInfo(null); } messageApi.success('Command executed'); - } catch (error) { clearPollingInterval(); setLoading(false); @@ -205,9 +199,7 @@ export default function App() {

- And make sure PyAutoGUI server is running on port 1412 + And make sure PyAutoGUI server is running on port + 1412 } /> diff --git a/apps/ios-playground/src/ios-device/index.tsx b/apps/ios-playground/src/ios-device/index.tsx index 6fcb4eff1..073ca61ad 100644 --- a/apps/ios-playground/src/ios-device/index.tsx +++ b/apps/ios-playground/src/ios-device/index.tsx @@ -5,125 +5,131 @@ import './index.less'; const { Text, Title } = Typography; interface IOSDeviceProps { - serverUrl?: string; - onServerStatusChange?: (connected: boolean) => void; + serverUrl?: string; + onServerStatusChange?: (connected: boolean) => void; } export default function IOSDevice({ - serverUrl = 'http://localhost:1412', - onServerStatusChange, + serverUrl = 'http://localhost:1412', + onServerStatusChange, }: IOSDeviceProps) { - const [serverConnected, setServerConnected] = useState(false); - const [checking, setChecking] = useState(false); + const [serverConnected, setServerConnected] = useState(false); + const [checking, setChecking] = useState(false); - // Helper function to get the appropriate URL for API calls - const getApiUrl = (endpoint: string) => { - // In development, use proxy; in production or when server is not localhost:1412, use direct URL - if (serverUrl === 'http://localhost:1412' && process.env.NODE_ENV === 'development') { - return `/api/pyautogui${endpoint}`; - } - return `${serverUrl}${endpoint}`; - }; + // Helper function to get the appropriate URL for API calls + const getApiUrl = (endpoint: string) => { + // In development, use proxy; in production or when server is not localhost:1412, use direct URL + if ( + serverUrl === 'http://localhost:1412' && + process.env.NODE_ENV === 'development' + ) { + return `/api/pyautogui${endpoint}`; + } + return `${serverUrl}${endpoint}`; + }; - const checkServerStatus = async () => { - setChecking(true); - try { - // Use proxy endpoint to avoid CORS issues - const response = await fetch(getApiUrl('/health')); - const connected = response.ok; - setServerConnected(connected); - onServerStatusChange?.(connected); - } catch (error) { - console.error('Failed to check server status:', error); - setServerConnected(false); - onServerStatusChange?.(false); - } finally { - setChecking(false); - } - }; + const checkServerStatus = async () => { + setChecking(true); + try { + // Use proxy endpoint to avoid CORS issues + const response = await fetch(getApiUrl('/health')); + const connected = response.ok; + setServerConnected(connected); + onServerStatusChange?.(connected); + } catch (error) { + console.error('Failed to check server status:', error); + setServerConnected(false); + onServerStatusChange?.(false); + } finally { + setChecking(false); + } + }; - const startPyAutoGUIServer = () => { - // Show instructions to user since we can't start server from frontend - const message = `Please start the PyAutoGUI server manually: + const startPyAutoGUIServer = () => { + // Show instructions to user since we can't start server from frontend + const message = `Please start the PyAutoGUI server manually: 1. Open Terminal 2. Run: npx @midscene/ios server 3. Make sure iPhone Mirroring app is open and connected`; - alert(message); - }; + alert(message); + }; - useEffect(() => { - checkServerStatus(); - // Check server status every 3 seconds - const interval = setInterval(checkServerStatus, 3000); - return () => clearInterval(interval); - }, [serverUrl]); + useEffect(() => { + checkServerStatus(); + // Check server status every 3 seconds + const interval = setInterval(checkServerStatus, 3000); + return () => clearInterval(interval); + }, [serverUrl]); - return ( -
- - - iOS Device Connection - - - } - size="small" - > - -
- - PyAutoGUI Server: -
- - {serverConnected ? 'Connected' : 'Disconnected'} - - -
+ return ( +
+ + + iOS Device Connection + + + } + size="small" + > + +
+ + PyAutoGUI Server: +
+ + {serverConnected ? 'Connected' : 'Disconnected'} + + +
-
- - Server URL: - {serverUrl} - -
+
+ + Server URL: + {serverUrl} + +
- {!serverConnected && ( -
- - - - -
- )} + {!serverConnected && ( +
+ + + + +
+ )} - {serverConnected && ( -
- - ✅ Ready for iOS automation - -
- )} - - -
- ); + {serverConnected && ( +
+ ✅ Ready for iOS automation +
+ )} + + +
+ ); } diff --git a/apps/ios-playground/src/ios-player/index.tsx b/apps/ios-playground/src/ios-player/index.tsx index a03431f50..82253333a 100644 --- a/apps/ios-playground/src/ios-player/index.tsx +++ b/apps/ios-playground/src/ios-player/index.tsx @@ -1,169 +1,184 @@ -import { Card, Button, Space, Typography, message, Tooltip } from 'antd'; import { SearchOutlined } from '@ant-design/icons'; -import { forwardRef, useImperativeHandle, useState, useEffect } from 'react'; +import { Button, Card, Space, Tooltip, Typography, message } from 'antd'; +import { forwardRef, useEffect, useImperativeHandle, useState } from 'react'; import './index.less'; const { Text } = Typography; export interface IOSPlayerRefMethods { - refreshDisplay: () => Promise; + refreshDisplay: () => Promise; } interface IOSPlayerProps { - serverUrl?: string; - autoConnect?: boolean; + serverUrl?: string; + autoConnect?: boolean; } const IOSPlayer = forwardRef( - ({ serverUrl = 'http://localhost:1412', autoConnect = false }, ref) => { - const [connected, setConnected] = useState(false); - const [autoDetecting, setAutoDetecting] = useState(false); - const [messageApi, contextHolder] = message.useMessage(); - const [mirrorConfig, setMirrorConfig] = useState(null); - - // Helper function to get the appropriate URL for API calls - const getApiUrl = (endpoint: string) => { - // In development, use proxy; in production or when server is not localhost:1412, use direct URL - if (serverUrl === 'http://localhost:1412' && process.env.NODE_ENV === 'development') { - return `/api/pyautogui${endpoint}`; + ({ serverUrl = 'http://localhost:1412', autoConnect = false }, ref) => { + const [connected, setConnected] = useState(false); + const [autoDetecting, setAutoDetecting] = useState(false); + const [messageApi, contextHolder] = message.useMessage(); + const [mirrorConfig, setMirrorConfig] = useState(null); + + // Helper function to get the appropriate URL for API calls + const getApiUrl = (endpoint: string) => { + // In development, use proxy; in production or when server is not localhost:1412, use direct URL + if ( + serverUrl === 'http://localhost:1412' && + process.env.NODE_ENV === 'development' + ) { + return `/api/pyautogui${endpoint}`; + } + return `${serverUrl}${endpoint}`; + }; + + const checkConnection = async () => { + try { + const response = await fetch(getApiUrl('/health')); + const isConnected = response.ok; + setConnected(isConnected); + + // If connected, also get the current config + if (isConnected) { + try { + const configResponse = await fetch(getApiUrl('/config')); + const configResult = await configResponse.json(); + if (configResult.status === 'ok') { + setMirrorConfig(configResult.config); } - return `${serverUrl}${endpoint}`; - }; - - const checkConnection = async () => { - try { - const response = await fetch(getApiUrl('/health')); - const isConnected = response.ok; - setConnected(isConnected); - - // If connected, also get the current config - if (isConnected) { - try { - const configResponse = await fetch(getApiUrl('/config')); - const configResult = await configResponse.json(); - if (configResult.status === 'ok') { - setMirrorConfig(configResult.config); - } - } catch (error) { - // Ignore config fetch errors - console.warn('Failed to fetch config:', error); - } - } - - return isConnected; - } catch (error) { - setConnected(false); - setMirrorConfig(null); - return false; - } - }; - - const autoDetectMirror = async () => { - if (!connected) { - messageApi.warning('Server is not connected'); - return; - } - - setAutoDetecting(true); - try { - const response = await fetch(getApiUrl('/detect'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - }); - - const result = await response.json(); - if (result.status === 'ok') { - messageApi.success(`Auto-configured: ${result.message}`); - setMirrorConfig(result.config); - } else { - messageApi.error(`Auto-detection failed: ${result.error}`); - if (result.suggestion) { - messageApi.info(result.suggestion); - } - } - } catch (error) { - messageApi.error(`Auto-detection error: ${error instanceof Error ? error.message : 'Unknown error'}`); - } finally { - setAutoDetecting(false); - } - }; - - useImperativeHandle(ref, () => ({ - refreshDisplay: async () => { - // Just refresh the connection status - await checkConnection(); - }, - })); - - useEffect(() => { - checkConnection(); - const interval = setInterval(checkConnection, 3000); - return () => clearInterval(interval); - }, [serverUrl]); - - useEffect(() => { - if (autoConnect && connected) { - // Try auto-detection when connected - autoDetectMirror(); - } - }, [autoConnect, connected]); - - return ( -
- {contextHolder} - - iOS Display - {connected && ( - - - - - - )} - - } - size="small" - > - {connected && mirrorConfig && mirrorConfig.enabled && ( -
- - ✅ Configured: {mirrorConfig.estimated_ios_width}×{mirrorConfig.estimated_ios_height} device - → {mirrorConfig.mirror_width}×{mirrorConfig.mirror_height} at ({mirrorConfig.mirror_x}, {mirrorConfig.mirror_y}) - -
- )} - -
- {!connected ? ( -
- - Waiting for iOS device connection... -
- Please ensure iPhone Mirroring is active - -
- ) : ( -
- - iOS device connected. Use Auto Detect to configure mirroring. - -
- )} -
- -
+ } catch (error) { + // Ignore config fetch errors + console.warn('Failed to fetch config:', error); + } + } + + return isConnected; + } catch (error) { + setConnected(false); + setMirrorConfig(null); + return false; + } + }; + + const autoDetectMirror = async () => { + if (!connected) { + messageApi.warning('Server is not connected'); + return; + } + + setAutoDetecting(true); + try { + const response = await fetch(getApiUrl('/detect'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + + const result = await response.json(); + if (result.status === 'ok') { + messageApi.success(`Auto-configured: ${result.message}`); + setMirrorConfig(result.config); + } else { + messageApi.error(`Auto-detection failed: ${result.error}`); + if (result.suggestion) { + messageApi.info(result.suggestion); + } + } + } catch (error) { + messageApi.error( + `Auto-detection error: ${error instanceof Error ? error.message : 'Unknown error'}`, ); - } + } finally { + setAutoDetecting(false); + } + }; + + useImperativeHandle(ref, () => ({ + refreshDisplay: async () => { + // Just refresh the connection status + await checkConnection(); + }, + })); + + useEffect(() => { + checkConnection(); + const interval = setInterval(checkConnection, 3000); + return () => clearInterval(interval); + }, [serverUrl]); + + useEffect(() => { + if (autoConnect && connected) { + // Try auto-detection when connected + autoDetectMirror(); + } + }, [autoConnect, connected]); + + return ( +
+ {contextHolder} + + iOS Display + {connected && ( + + + + + + )} + + } + size="small" + > + {connected && mirrorConfig && mirrorConfig.enabled && ( +
+ + ✅ Configured: {mirrorConfig.estimated_ios_width}× + {mirrorConfig.estimated_ios_height} device →{' '} + {mirrorConfig.mirror_width}×{mirrorConfig.mirror_height} at ( + {mirrorConfig.mirror_x}, {mirrorConfig.mirror_y}) + +
+ )} + +
+ {!connected ? ( +
+ + Waiting for iOS device connection... +
+ Please ensure iPhone Mirroring is active + +
+ ) : ( +
+ + iOS device connected. Use Auto Detect to configure mirroring. + +
+ )} +
+ +
+ ); + }, ); IOSPlayer.displayName = 'IOSPlayer'; diff --git a/apps/ios-playground/src/scripts/blank_polyfill.ts b/apps/ios-playground/src/scripts/blank_polyfill.ts index d1eb7e91a..f76a2469b 100644 --- a/apps/ios-playground/src/scripts/blank_polyfill.ts +++ b/apps/ios-playground/src/scripts/blank_polyfill.ts @@ -1,2 +1,2 @@ const AsyncLocalStorage = {}; -export { AsyncLocalStorage }; \ No newline at end of file +export { AsyncLocalStorage }; diff --git a/packages/ios-playground/bin/server.js b/packages/ios-playground/bin/server.js index ed405d1d9..301be59fd 100644 --- a/packages/ios-playground/bin/server.js +++ b/packages/ios-playground/bin/server.js @@ -1,15 +1,19 @@ -const path = require('path'); -const { spawn } = require('child_process'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); const { iOSDevice, iOSAgent } = require('@midscene/ios'); const { PLAYGROUND_SERVER_PORT } = require('@midscene/shared/constants'); const PlaygroundServer = require('@midscene/web/midscene-server').default; -const staticDir = path.join(__dirname, '..', '..', '..', 'apps', 'ios-playground', 'dist'); -const playgroundServer = new PlaygroundServer( - iOSDevice, - iOSAgent, - staticDir, +const staticDir = path.join( + __dirname, + '..', + '..', + '..', + 'apps', + 'ios-playground', + 'dist', ); +const playgroundServer = new PlaygroundServer(iOSDevice, iOSAgent, staticDir); // Auto server management let autoServerProcess = null; @@ -19,141 +23,155 @@ const AUTO_SERVER_PORT = 1412; * Check if auto server is running on the specified port */ const checkAutoServerRunning = async (port = AUTO_SERVER_PORT) => { - return new Promise((resolve) => { - const net = require('net'); - const client = new net.Socket(); + return new Promise((resolve) => { + const net = require('node:net'); + const client = new net.Socket(); - client.setTimeout(1000); + client.setTimeout(1000); - client.on('connect', () => { - client.destroy(); - resolve(true); - }); - - client.on('timeout', () => { - client.destroy(); - resolve(false); - }); + client.on('connect', () => { + client.destroy(); + resolve(true); + }); - client.on('error', () => { - resolve(false); - }); + client.on('timeout', () => { + client.destroy(); + resolve(false); + }); - client.connect(port, 'localhost'); + client.on('error', () => { + resolve(false); }); + + client.connect(port, 'localhost'); + }); }; /** * Start the auto server if it's not running */ const startAutoServer = async () => { - try { - const isRunning = await checkAutoServerRunning(); - - if (isRunning) { - console.log(`✅ PyAutoGUI server is already running on port ${AUTO_SERVER_PORT}`); - return true; - } - - console.log(`🚀 Starting PyAutoGUI server on port ${AUTO_SERVER_PORT}...`); - - // Find the auto server script path - const autoServerPath = path.join(__dirname, '..', '..', 'ios', 'bin', 'server.js'); - - // Start the auto server process - autoServerProcess = spawn('node', [autoServerPath, AUTO_SERVER_PORT], { - stdio: 'pipe', - env: { - ...process.env, - NODE_ENV: 'production' - } - }); - - // Handle auto server output - autoServerProcess.stdout.on('data', (data) => { - const output = data.toString().trim(); - if (output) { - console.log(`[PyAutoGUI] ${output}`); - } - }); - - autoServerProcess.stderr.on('data', (data) => { - const output = data.toString().trim(); - if (output) { - console.error(`[PyAutoGUI Error] ${output}`); - } - }); - - autoServerProcess.on('error', (error) => { - console.error('Failed to start PyAutoGUI server:', error); - }); - - autoServerProcess.on('close', (code) => { - if (code !== 0) { - console.error(`PyAutoGUI server exited with code ${code}`); - } - autoServerProcess = null; - }); - - // Wait a bit for the server to start - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Verify it's running - const isNowRunning = await checkAutoServerRunning(); - if (isNowRunning) { - console.log(`✅ PyAutoGUI server started successfully on port ${AUTO_SERVER_PORT}`); - return true; - } else { - console.error(`❌ Failed to start PyAutoGUI server on port ${AUTO_SERVER_PORT}`); - return false; - } - - } catch (error) { - console.error('Error starting auto server:', error); - return false; + try { + const isRunning = await checkAutoServerRunning(); + + if (isRunning) { + console.log( + `✅ PyAutoGUI server is already running on port ${AUTO_SERVER_PORT}`, + ); + return true; + } + + console.log(`🚀 Starting PyAutoGUI server on port ${AUTO_SERVER_PORT}...`); + + // Find the auto server script path + const autoServerPath = path.join( + __dirname, + '..', + '..', + 'ios', + 'bin', + 'server.js', + ); + + // Start the auto server process + autoServerProcess = spawn('node', [autoServerPath, AUTO_SERVER_PORT], { + stdio: 'pipe', + env: { + ...process.env, + NODE_ENV: 'production', + }, + }); + + // Handle auto server output + autoServerProcess.stdout.on('data', (data) => { + const output = data.toString().trim(); + if (output) { + console.log(`[PyAutoGUI] ${output}`); + } + }); + + autoServerProcess.stderr.on('data', (data) => { + const output = data.toString().trim(); + if (output) { + console.error(`[PyAutoGUI Error] ${output}`); + } + }); + + autoServerProcess.on('error', (error) => { + console.error('Failed to start PyAutoGUI server:', error); + }); + + autoServerProcess.on('close', (code) => { + if (code !== 0) { + console.error(`PyAutoGUI server exited with code ${code}`); + } + autoServerProcess = null; + }); + + // Wait a bit for the server to start + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Verify it's running + const isNowRunning = await checkAutoServerRunning(); + if (isNowRunning) { + console.log( + `✅ PyAutoGUI server started successfully on port ${AUTO_SERVER_PORT}`, + ); + return true; + } else { + console.error( + `❌ Failed to start PyAutoGUI server on port ${AUTO_SERVER_PORT}`, + ); + return false; } + } catch (error) { + console.error('Error starting auto server:', error); + return false; + } }; const main = async () => { - try { - // Start auto server first - await startAutoServer(); - - await playgroundServer.launch(PLAYGROUND_SERVER_PORT); + try { + // Start auto server first + await startAutoServer(); + + await playgroundServer.launch(PLAYGROUND_SERVER_PORT); + console.log( + `Midscene iOS Playground server is running on http://localhost:${playgroundServer.port}`, + ); + + // Automatically open browser + if (process.env.NODE_ENV !== 'test') { + try { + const { default: open } = await import('open'); + await open(`http://localhost:${playgroundServer.port}`); + } catch (error) { console.log( - `Midscene iOS Playground server is running on http://localhost:${playgroundServer.port}`, + 'Could not open browser automatically. Please visit the URL manually.', ); - - // Automatically open browser - if (process.env.NODE_ENV !== 'test') { - try { - const { default: open } = await import('open'); - await open(`http://localhost:${playgroundServer.port}`); - } catch (error) { - console.log('Could not open browser automatically. Please visit the URL manually.'); - } - } - } catch (error) { - console.error('Failed to start iOS playground server:', error); - process.exit(1); + } } + } catch (error) { + console.error('Failed to start iOS playground server:', error); + process.exit(1); + } }; // Handle graceful shutdown const cleanup = () => { - console.log('Shutting down gracefully...'); + console.log('Shutting down gracefully...'); - if (playgroundServer) { - playgroundServer.close(); - } + if (playgroundServer) { + playgroundServer.close(); + } - if (autoServerProcess) { - console.log('Stopping PyAutoGUI server...'); - autoServerProcess.kill('SIGTERM'); - autoServerProcess = null; - } + if (autoServerProcess) { + console.log('Stopping PyAutoGUI server...'); + autoServerProcess.kill('SIGTERM'); + autoServerProcess = null; + } - process.exit(0); + process.exit(0); }; process.on('SIGTERM', cleanup); diff --git a/packages/ios-playground/modern.config.ts b/packages/ios-playground/modern.config.ts index ff87335a3..059fb95dc 100644 --- a/packages/ios-playground/modern.config.ts +++ b/packages/ios-playground/modern.config.ts @@ -1,4 +1,4 @@ -import { moduleTools, defineConfig } from '@modern-js/module-tools'; +import { defineConfig, moduleTools } from '@modern-js/module-tools'; export default defineConfig({ plugins: [moduleTools()], diff --git a/packages/ios-playground/test-health.js b/packages/ios-playground/test-health.js index 88618bbc8..9a324fb4a 100644 --- a/packages/ios-playground/test-health.js +++ b/packages/ios-playground/test-health.js @@ -1,28 +1,28 @@ const fetch = require('node-fetch'); async function testHealth() { - try { - console.log('Testing health check...'); - const response = await fetch('http://localhost:5800/status'); - const data = await response.json(); - console.log('Status:', response.status); - console.log('Data:', data); - console.log('Success!'); - } catch (error) { - console.error('Error:', error.message); - } + try { + console.log('Testing health check...'); + const response = await fetch('http://localhost:5800/status'); + const data = await response.json(); + console.log('Status:', response.status); + console.log('Data:', data); + console.log('Success!'); + } catch (error) { + console.error('Error:', error.message); + } } // Test multiple times like the frontend does let counter = 0; const interval = setInterval(async () => { - counter++; - console.log(`\n--- Test ${counter} ---`); - await testHealth(); + counter++; + console.log(`\n--- Test ${counter} ---`); + await testHealth(); - if (counter >= 5) { - clearInterval(interval); - console.log('\nTest completed'); - process.exit(0); - } + if (counter >= 5) { + clearInterval(interval); + console.log('\nTest completed'); + process.exit(0); + } }, 2000); diff --git a/packages/ios/bin/server.js b/packages/ios/bin/server.js index e84645c02..71e150832 100755 --- a/packages/ios/bin/server.js +++ b/packages/ios/bin/server.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { spawn, execSync } from 'node:child_process'; +import { execSync, spawn } from 'node:child_process'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -46,4 +46,4 @@ process.on('SIGINT', () => { process.on('SIGTERM', () => { console.log('\nShutting down PyAutoGUI server...'); server.kill('SIGTERM'); -}); \ No newline at end of file +}); diff --git a/packages/ios/examples/ios-mirroring-demo.js b/packages/ios/examples/ios-mirroring-demo.js index 1e8af4ef1..077afb309 100644 --- a/packages/ios/examples/ios-mirroring-demo.js +++ b/packages/ios/examples/ios-mirroring-demo.js @@ -56,7 +56,7 @@ async function demonstrateIOSMirroring() { console.log('📱 Step 1: Setting up iOS device mirroring...'); const device = new iOSDevice({ - serverPort: 1412 + serverPort: 1412, }); try { diff --git a/packages/ios/src/agent/index.ts b/packages/ios/src/agent/index.ts index 2ebf5b9c4..0ae54cd99 100644 --- a/packages/ios/src/agent/index.ts +++ b/packages/ios/src/agent/index.ts @@ -21,7 +21,6 @@ export class iOSAgent extends PageAgent { } return this.connectionPromise; } - } export async function agentFromPyAutoGUI(opts?: iOSAgentOpt & iOSDeviceOpt) { diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index f3b4eb5fc..a2e43dac7 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -6,10 +6,7 @@ import { getTmpFile, sleep } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; import { resizeImgBuffer } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; -import { - type AndroidDeviceInputOpt, - type AndroidDevicePage, -} from '@midscene/web'; +import type { AndroidDeviceInputOpt, AndroidDevicePage } from '@midscene/web'; import { commonWebActionsForWebPage } from '@midscene/web/utils'; import { type ScreenInfo, getScreenSize } from '../utils'; @@ -151,9 +148,7 @@ export class iOSDevice implements AndroidDevicePage { call: async (context, param) => { const { element } = context; if (!element) { - throw new Error( - 'IOSLongPress requires an element to be located', - ); + throw new Error('IOSLongPress requires an element to be located'); } const [x, y] = element.center; await this.longPress(x, y, param?.duration); @@ -194,7 +189,6 @@ export class iOSDevice implements AndroidDevicePage { return allActions; } - public async connect(): Promise { if (this.destroyed) { throw new Error('iOSDevice has been destroyed and cannot be used'); @@ -212,26 +206,30 @@ export class iOSDevice implements AndroidDevicePage { debugPage(`Python server is running: ${JSON.stringify(healthData)}`); } catch (error: any) { debugPage(`Python server connection failed: ${error.message}`); - + // Try to start server automatically debugPage('Attempting to start Python server automatically...'); - + try { await this.startPyAutoGUIServer(); debugPage('Python server started successfully'); - + // Verify server is now running const response = await fetch(`${this.serverUrl}/health`); if (!response.ok) { - throw new Error(`Server still not responding after startup: ${response.status}`); + throw new Error( + `Server still not responding after startup: ${response.status}`, + ); } - + const healthData = await response.json(); - debugPage(`Python server is now running: ${JSON.stringify(healthData)}`); + debugPage( + `Python server is now running: ${JSON.stringify(healthData)}`, + ); } catch (startError: any) { throw new Error( `Failed to auto-start Python server: ${startError.message}. ` + - `Please manually start the server by running: node packages/ios/bin/server.js ${this.serverPort}` + `Please manually start the server by running: node packages/ios/bin/server.js ${this.serverPort}`, ); } } @@ -262,52 +260,58 @@ export class iOSDevice implements AndroidDevicePage { // Configure iOS mirroring if provided await this.initializeMirrorConfiguration(); - } private async startPyAutoGUIServer(): Promise { try { const { spawn } = await import('node:child_process'); const serverScriptPath = path.resolve(__dirname, '../../bin/server.js'); - - debugPage(`Starting PyAutoGUI server using: node ${serverScriptPath} ${this.serverPort}`); - + + debugPage( + `Starting PyAutoGUI server using: node ${serverScriptPath} ${this.serverPort}`, + ); + // Start server process in background (similar to server.js background mode) - this.serverProcess = spawn('node', [serverScriptPath, this.serverPort.toString()], { - detached: true, - stdio: 'pipe', // Capture output - env: { - ...process.env, + this.serverProcess = spawn( + 'node', + [serverScriptPath, this.serverPort.toString()], + { + detached: true, + stdio: 'pipe', // Capture output + env: { + ...process.env, + }, }, - }); - + ); + // Handle server process events this.serverProcess.on('error', (error: any) => { debugPage(`Server process error: ${error.message}`); }); - + this.serverProcess.on('exit', (code: number, signal: string) => { debugPage(`Server process exited with code ${code}, signal ${signal}`); }); - + // Capture and log server output if (this.serverProcess.stdout) { this.serverProcess.stdout.on('data', (data: Buffer) => { debugPage(`Server stdout: ${data.toString().trim()}`); }); } - + if (this.serverProcess.stderr) { this.serverProcess.stderr.on('data', (data: Buffer) => { debugPage(`Server stderr: ${data.toString().trim()}`); }); } - - debugPage(`Started PyAutoGUI server process with PID: ${this.serverProcess.pid}`); - + + debugPage( + `Started PyAutoGUI server process with PID: ${this.serverProcess.pid}`, + ); + // Wait for server to start up (similar to server.js timeout) await sleep(3000); - } catch (error: any) { throw new Error(`Failed to start PyAutoGUI server: ${error.message}`); } @@ -327,16 +331,18 @@ export class iOSDevice implements AndroidDevicePage { this.options.mirrorConfig = mirrorConfig; debugPage( - `Auto-detected iOS mirror config: ${mirrorConfig.mirrorWidth}x${mirrorConfig.mirrorHeight} at (${mirrorConfig.mirrorX}, ${mirrorConfig.mirrorY})` + `Auto-detected iOS mirror config: ${mirrorConfig.mirrorWidth}x${mirrorConfig.mirrorHeight} at (${mirrorConfig.mirrorX}, ${mirrorConfig.mirrorY})`, ); - + // Configure the detected mirror settings await this.configureIOSMirror(mirrorConfig); } else { debugPage('No iPhone Mirroring app found or auto-detection failed'); } } catch (error: any) { - debugPage(`Failed to auto-detect iPhone Mirroring app: ${error.message}`); + debugPage( + `Failed to auto-detect iPhone Mirroring app: ${error.message}`, + ); } } @@ -421,7 +427,9 @@ export class iOSDevice implements AndroidDevicePage { end tell `; - const { stdout, stderr } = await execAsync(`osascript -e '${applescript}'`); + const { stdout, stderr } = await execAsync( + `osascript -e '${applescript}'`, + ); if (stderr) { debugPage(`AppleScript error: ${stderr}`); @@ -431,7 +439,9 @@ export class iOSDevice implements AndroidDevicePage { const result = JSON.parse(stdout.trim()); if (!result.found) { - debugPage(`iPhone Mirroring app not found: ${result.error || 'Unknown error'}`); + debugPage( + `iPhone Mirroring app not found: ${result.error || 'Unknown error'}`, + ); return null; } @@ -440,11 +450,14 @@ export class iOSDevice implements AndroidDevicePage { const windowWidth = result.width; const windowHeight = result.height; - debugPage(`Detected iPhone Mirroring window: ${windowWidth}x${windowHeight} at (${windowX}, ${windowY})`); + debugPage( + `Detected iPhone Mirroring window: ${windowWidth}x${windowHeight} at (${windowX}, ${windowY})`, + ); // Calculate device content area with smart detection based on window size - let titleBarHeight = 28; - let contentPaddingH, contentPaddingV; + const titleBarHeight = 28; + let contentPaddingH; + let contentPaddingV; if (windowWidth < 500 && windowHeight < 1000) { // Small window - minimal padding @@ -462,7 +475,8 @@ export class iOSDevice implements AndroidDevicePage { // Calculate the actual iOS device screen area within the window const contentX = windowX + Math.floor(contentPaddingH / 2); - const contentY = windowY + titleBarHeight + Math.floor(contentPaddingV / 2); + const contentY = + windowY + titleBarHeight + Math.floor(contentPaddingV / 2); const contentWidth = windowWidth - contentPaddingH; const contentHeight = windowHeight - titleBarHeight - contentPaddingV; @@ -475,7 +489,9 @@ export class iOSDevice implements AndroidDevicePage { const minimalContentHeight = windowHeight - titleBarHeight - 20; if (minimalContentWidth < 200 || minimalContentHeight < 400) { - debugPage(`Detected window seems too small for iPhone content: ${windowWidth}x${windowHeight}`); + debugPage( + `Detected window seems too small for iPhone content: ${windowWidth}x${windowHeight}`, + ); return null; } @@ -487,7 +503,9 @@ export class iOSDevice implements AndroidDevicePage { }; } - debugPage(`Calculated content area: ${contentWidth}x${contentHeight} at (${contentX}, ${contentY})`); + debugPage( + `Calculated content area: ${contentWidth}x${contentHeight} at (${contentX}, ${contentY})`, + ); return { mirrorX: contentX, @@ -496,7 +514,9 @@ export class iOSDevice implements AndroidDevicePage { mirrorHeight: contentHeight, }; } catch (error: any) { - debugPage(`Exception during iPhone Mirroring app detection: ${error.message}`); + debugPage( + `Exception during iPhone Mirroring app detection: ${error.message}`, + ); return null; } } @@ -613,7 +633,7 @@ export class iOSDevice implements AndroidDevicePage { { width, height, - } + }, ); // Clean up temporary file @@ -649,7 +669,7 @@ export class iOSDevice implements AndroidDevicePage { { width, height, - } + }, ); debugPage('screenshotBase64 end (via screencapture)'); @@ -1202,7 +1222,7 @@ export class iOSDevice implements AndroidDevicePage { async destroy(): Promise { debugPage('destroy iOS device'); this.destroyed = true; - + // Clean up server process if we started it if (this.serverProcess) { try { diff --git a/packages/ios/src/utils/index.ts b/packages/ios/src/utils/index.ts index 52e8ab780..93eccc55d 100644 --- a/packages/ios/src/utils/index.ts +++ b/packages/ios/src/utils/index.ts @@ -81,7 +81,7 @@ export async function getScreenSize(): Promise { export async function startPyAutoGUIServer(port = 1412): Promise { const { spawn } = await import('node:child_process'); const path = await import('node:path'); - + // Use __dirname in a way that works for both ESM and CommonJS let currentDir: string; if (typeof __dirname !== 'undefined') { @@ -90,7 +90,7 @@ export async function startPyAutoGUIServer(port = 1412): Promise { const { fileURLToPath } = await import('node:url'); currentDir = path.dirname(fileURLToPath(import.meta.url)); } - + const serverPath = path.join(currentDir, '../../idb/auto_server.py'); const server = spawn('python3', [serverPath], { From ae9853b5d8be99b619fc294ccaa0e23bf98d124e Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Sat, 16 Aug 2025 00:40:03 +0800 Subject: [PATCH 16/17] build(ios-playground): configure explicit input entries --- packages/ios-playground/modern.config.ts | 6 ++++++ tsconfig.json | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tsconfig.json diff --git a/packages/ios-playground/modern.config.ts b/packages/ios-playground/modern.config.ts index 059fb95dc..e22c2867a 100644 --- a/packages/ios-playground/modern.config.ts +++ b/packages/ios-playground/modern.config.ts @@ -3,6 +3,12 @@ import { defineConfig, moduleTools } from '@modern-js/module-tools'; export default defineConfig({ plugins: [moduleTools()], buildConfig: { + // Provide explicit input entries so Modern can run JS compilation/dts tasks. + // The package's runtime entry points live under `bin/` (server.js and cli). + input: { + server: './bin/server.js', + cli: './bin/ios-playground', + }, buildType: 'bundle', format: 'cjs', target: 'es2019', diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..ac17eb985 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ESNext", "DOM"], + "jsx": "preserve", + "esModuleInterop": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "baseUrl": "." + }, + "exclude": ["node_modules", "dist", "out"] +} From fc1c8a8a07a8ee24c53eeaff6b7747911f8a37c5 Mon Sep 17 00:00:00 2001 From: Huanyu Luo Date: Sat, 16 Aug 2025 21:52:53 +0800 Subject: [PATCH 17/17] feat(ios): improve automation server stability and logging --- packages/ios-playground/bin/server.js | 15 +++++++++++---- packages/ios/bin/server.js | 4 ++-- packages/ios/idb/auto_server.py | 3 ++- packages/ios/src/agent/index.ts | 4 +--- packages/ios/src/page/index.ts | 26 +++++++++++++++++++++----- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/packages/ios-playground/bin/server.js b/packages/ios-playground/bin/server.js index 301be59fd..0bf6a4cf6 100644 --- a/packages/ios-playground/bin/server.js +++ b/packages/ios-playground/bin/server.js @@ -85,15 +85,20 @@ const startAutoServer = async () => { // Handle auto server output autoServerProcess.stdout.on('data', (data) => { const output = data.toString().trim(); - if (output) { - console.log(`[PyAutoGUI] ${output}`); - } + if (!output) return; + console.log(`[PyAutoGUI] ${output}`); }); autoServerProcess.stderr.on('data', (data) => { const output = data.toString().trim(); - if (output) { + if (!output) return; + // Only surface real errors + const isRealError = + /Traceback|Exception|Error|Trace|Failed|CRITICAL/i.test(output); + if (isRealError) { console.error(`[PyAutoGUI Error] ${output}`); + } else { + console.error(`[PyAutoGUI] ${output}`); } }); @@ -122,10 +127,12 @@ const startAutoServer = async () => { console.error( `❌ Failed to start PyAutoGUI server on port ${AUTO_SERVER_PORT}`, ); + lastStartFailed = true; return false; } } catch (error) { console.error('Error starting auto server:', error); + lastStartFailed = true; return false; } }; diff --git a/packages/ios/bin/server.js b/packages/ios/bin/server.js index 71e150832..ec89cc4b1 100755 --- a/packages/ios/bin/server.js +++ b/packages/ios/bin/server.js @@ -11,12 +11,12 @@ const port = process.argv[2] || '1412'; console.log(`Starting PyAutoGUI server on port ${port}...`); -// kill process on port 1412 first +// kill process on port first; if nothing is listening, silently ignore try { execSync(`lsof -ti:${port} | xargs kill -9`, { stdio: 'ignore' }); console.log(`Killed existing process on port ${port}`); } catch (error) { - console.error(`Failed to kill process on port ${port}:`, error); + console.warn(`No existing process to kill on port ${port}`); } const server = spawn('python3', [serverPath, port], { diff --git a/packages/ios/idb/auto_server.py b/packages/ios/idb/auto_server.py index 89f1578e8..aafaa5e78 100644 --- a/packages/ios/idb/auto_server.py +++ b/packages/ios/idb/auto_server.py @@ -609,4 +609,5 @@ def run_actions(): print(f"Starting PyAutoGUI server on port {port}") print(f"Screen size: {pyautogui.size()}") print("Health check available at: http://localhost:{}/health".format(port)) - app.run(host="0.0.0.0", port=port, debug=False) \ No newline at end of file + # use_threaded True and disable reloader to avoid duplicate startup logs + app.run(host="0.0.0.0", port=port, debug=False, use_reloader=False, threaded=True) \ No newline at end of file diff --git a/packages/ios/src/agent/index.ts b/packages/ios/src/agent/index.ts index 0ae54cd99..69c798899 100644 --- a/packages/ios/src/agent/index.ts +++ b/packages/ios/src/agent/index.ts @@ -1,8 +1,6 @@ -import { vlLocateMode } from '@midscene/shared/env'; import { PageAgent, type PageAgentOpt } from '@midscene/web/agent'; import { iOSDevice, type iOSDeviceOpt } from '../page'; -import { debugPage } from '../page'; -import { getScreenSize, startPyAutoGUIServer } from '../utils'; +import { startPyAutoGUIServer } from '../utils'; type iOSAgentOpt = PageAgentOpt; diff --git a/packages/ios/src/page/index.ts b/packages/ios/src/page/index.ts index a2e43dac7..316578faf 100644 --- a/packages/ios/src/page/index.ts +++ b/packages/ios/src/page/index.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import type { Point, Size } from '@midscene/core'; -import type { DeviceAction, ExecutorContext, PageType } from '@midscene/core'; +import type { DeviceAction, PageType } from '@midscene/core'; import { getTmpFile, sleep } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; import { resizeImgBuffer } from '@midscene/shared/img'; @@ -272,11 +272,12 @@ export class iOSDevice implements AndroidDevicePage { ); // Start server process in background (similar to server.js background mode) + // Start server process (non-detached so parent can reliably terminate it) this.serverProcess = spawn( 'node', [serverScriptPath, this.serverPort.toString()], { - detached: true, + detached: false, stdio: 'pipe', // Capture output env: { ...process.env, @@ -289,9 +290,24 @@ export class iOSDevice implements AndroidDevicePage { debugPage(`Server process error: ${error.message}`); }); - this.serverProcess.on('exit', (code: number, signal: string) => { - debugPage(`Server process exited with code ${code}, signal ${signal}`); - }); + // Listen for both exit and close for robust termination handling + this.serverProcess.on( + 'exit', + (code: number | null, signal: string | null) => { + debugPage( + `Server process exit event: code=${code}, signal=${signal}`, + ); + }, + ); + + this.serverProcess.on( + 'close', + (code: number | null, signal: string | null) => { + debugPage(`Server process closed: code=${code}, signal=${signal}`); + // Ensure reference is cleared when process actually stops + this.serverProcess = undefined; + }, + ); // Capture and log server output if (this.serverProcess.stdout) {