feat(ios): add iOS automation support via screen mirroring

apps/ios-playground/README.md

@@ -0,0 +1,66 @@
+# Midscene iOS Playground
+
+A playground for testing Midscene iOS automation features with automatic device mirroring setup.
+
+See https://midscenejs.com/ for details.
+
+## Features
+
+### ✨ Auto-Detection of iPhone Mirroring
+
+The playground can automatically detect and configure the iPhone Mirroring app window:
+
+1. **Automatic Setup**: When you connect, the playground automatically tries to detect your iPhone Mirroring window
+2. **Smart Configuration**: It calculates the optimal screen mapping based on window size and device type  
+3. **Manual Override**: If auto-detection doesn't work, you can manually configure the mirror settings
+
+### 🎯 How Auto-Detection Works
+
+1. **Window Detection**: Uses AppleScript to find the iPhone Mirroring app window
+2. **Content Area Calculation**: Automatically calculates the device screen area within the window (excluding title bars and padding)
+3. **Device Matching**: Matches the aspect ratio to common iOS devices for optimal coordinate mapping
+4. **Instant Configuration**: Sets up the coordinate transformation automatically
+
+## Usage
+
+### Prerequisites
+
+1. **macOS with iPhone Mirroring**: Ensure iPhone Mirroring is available and working
+2. **iOS Device**: Connected and mirroring to your Mac
+3. **Python Server**: The PyAutoGUI server running on port 1412
+
+### Quick Start
+
+1. **Start the server**:
+   ```bash
+   cd packages/ios/idb
+   python auto_server.py
+   ```
+
+2. **Launch the playground**:
+   ```bash
+   npm run dev
+   ```
+
+3. **Open iPhone Mirroring app** on your Mac
+
+4. **Auto-configure**: Click "Auto Detect" to automatically set up the mirroring coordinates
+
+### UI Controls
+
+- **📷 Screenshot**: Take a screenshot of the configured iOS device area
+- **🔍 Auto Detect**: Automatically detect and configure iPhone Mirroring window  
+- **⚙️ Manual Config**: Manually set mirror window coordinates
+
+## Troubleshooting
+
+### Auto-Detection Issues
+
+1. **"iPhone Mirroring app not found"**: Make sure iPhone Mirroring app is open and visible
+2. **"Window seems too small"**: Try resizing the iPhone Mirroring window to be larger
+3. **Coordinates seem wrong**: Use manual configuration to fine-tune the coordinates
+
+### Server Connection Issues
+
+1. **Server not responding**: Check if server is running on port 1412
+2. **Permission issues**: Ensure macOS accessibility permissions are granted to Terminal/Python

apps/ios-playground/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "ios-playground",
+  "private": true,
+  "version": "0.12.4",
+  "type": "module",
+  "scripts": {
+    "build": "rsbuild build",
+    "dev": "rsbuild dev --open",
+    "preview": "rsbuild preview"
+  },
+  "dependencies": {
+    "@ant-design/icons": "^5.3.1",
+    "@midscene/ios": "workspace:*",
+    "@midscene/core": "workspace:*",
+    "@midscene/shared": "workspace:*",
+    "@midscene/visualizer": "workspace:*",
+    "@midscene/web": "workspace:*",
+    "antd": "^5.21.6",
+    "dayjs": "^1.11.11",
+    "react": "18.3.1",
+    "react-dom": "18.3.1"
+  },
+  "devDependencies": {
+    "@rsbuild/core": "^1.3.22",
+    "@rsbuild/plugin-less": "^1.2.4",
+    "@rsbuild/plugin-node-polyfill": "1.3.0",
+    "@rsbuild/plugin-react": "^1.3.1",
+    "@rsbuild/plugin-svgr": "^1.1.1",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.1",
+    "archiver": "^6.0.0",
+    "less": "^4.2.0",
+    "typescript": "^5.8.3"
+  }
+}

apps/ios-playground/rsbuild.config.ts

@@ -0,0 +1,88 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { defineConfig } from '@rsbuild/core';
+import { pluginLess } from '@rsbuild/plugin-less';
+import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill';
+import { pluginReact } from '@rsbuild/plugin-react';
+import { pluginSvgr } from '@rsbuild/plugin-svgr';
+import { pluginTypeCheck } from '@rsbuild/plugin-type-check';
+
+const copyAndroidPlaygroundStatic = () => ({
+  name: 'copy-android-playground-static',
+  setup(api) {
+    api.onAfterBuild(async () => {
+      const srcDir = path.join(__dirname, 'dist');
+      const destDir = path.join(
+        __dirname,
+        '..',
+        '..',
+        'packages',
+        'android-playground',
+        'static',
+      );
+      const faviconSrc = path.join(__dirname, 'src', 'favicon.ico');
+      const faviconDest = path.join(destDir, 'favicon.ico');
+
+      await fs.promises.mkdir(destDir, { recursive: true });
+      // Copy directory contents recursively
+      await fs.promises.cp(srcDir, destDir, { recursive: true });
+      // Copy favicon
+      await fs.promises.copyFile(faviconSrc, faviconDest);
+
+      console.log(`Copied build artifacts to ${destDir}`);
+      console.log(`Copied favicon to ${faviconDest}`);
+    });
+  },
+});
+
+export default defineConfig({
+  environments: {
+    web: {
+      source: {
+        entry: {
+          index: './src/index.tsx',
+        },
+      },
+      output: {
+        target: 'web',
+        sourceMap: true,
+      },
+      html: {
+        title: 'Midscene iOS Playground',
+      },
+    },
+  },
+  dev: {
+    writeToDisk: true,
+  },
+  server: {
+    proxy: {
+      '/api/pyautogui': {
+        target: 'http://localhost:1412',
+        changeOrigin: true,
+        pathRewrite: {
+          '^/api/pyautogui': ''
+        }
+      }
+    }
+  },
+  resolve: {
+    alias: {
+      async_hooks: path.join(__dirname, './src/scripts/blank_polyfill.ts'),
+      'node:async_hooks': path.join(
+        __dirname,
+        './src/scripts/blank_polyfill.ts',
+      ),
+      react: path.resolve(__dirname, 'node_modules/react'),
+      'react-dom': path.resolve(__dirname, 'node_modules/react-dom'),
+    },
+  },
+  plugins: [
+    pluginReact(),
+    pluginNodePolyfill(),
+    pluginLess(),
+    pluginSvgr(),
+    copyAndroidPlaygroundStatic(),
+    pluginTypeCheck(),
+  ],
+});

apps/ios-playground/src/App.less

@@ -0,0 +1,183 @@
+body {
+  margin: 0;
+  padding: 0;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans',
+    Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji';
+  font-size: 14px;
+}
+
+.app-container {
+  width: 100%;
+  height: 100vh;
+  display: flex;
+  flex-direction: column;
+  background-color: #f5f5f5;
+}
+
+.app-content {
+  height: 100vh;
+  overflow: hidden;
+}
+
+.app-grid-layout {
+  height: 100%;
+  display: flex;
+
+  .ant-row {
+    flex: 1;
+    width: 100%;
+    height: 100%;
+    display: flex;
+    flex-wrap: nowrap;
+    width: 100%;
+  }
+}
+
+.app-panel {
+  height: 100%;
+  background-color: #fff;
+  border-radius: 0;
+  box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+  transition: box-shadow 0.3s;
+  overflow: hidden;
+
+  &:hover {
+    box-shadow: 0 2px 8px rgba(0, 0, 0, 0.09);
+  }
+
+  &.left-panel {
+    width: 480px;
+    flex: none;
+    overflow: hidden;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  &.right-panel {
+    border-radius: 0;
+    flex: 1;
+    overflow: hidden;
+    box-shadow: -4px 0px 20px 0px #0000000A;
+  }
+}
+
+.panel-content {
+  padding: 12px 24px 24px 24px;
+  height: 100%;
+  overflow: auto;
+  display: flex;
+  flex-direction: column;
+  border-left: 1px solid rgba(0, 0, 0, 0.08);
+
+  &.left-panel-content {
+    display: flex;
+    flex-direction: column;
+    height: 100%;
+    overflow: hidden;
+  }
+
+  &.right-panel-content {
+    border-radius: 0;
+  }
+
+  h2 {
+    color: #000;
+    font-size: 18px;
+    margin-top: 16px;
+    margin-bottom: 12px;
+  }
+
+  canvas {
+    max-width: 100%;
+    margin-top: 16px;
+    border: 1px solid #f0f0f0;
+    border-radius: 4px;
+  }
+}
+
+.command-form {
+  display: flex;
+  flex-direction: column;
+  height: 100%;
+  overflow: hidden;
+
+  .form-content {
+    display: flex;
+    flex-direction: column;
+    height: 100%;
+    gap: 24px;
+  }
+
+  .command-input-wrapper {
+    margin-top: 8px;
+  }
+}
+
+.result-container {
+  flex: 1;
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
+  min-height: 0;
+  position: relative;
+  height: 100%;
+}
+
+@media (max-width: 768px) {
+  .app-container {
+    height: auto;
+    min-height: 100vh;
+  }
+
+  .app-grid-layout .ant-row {
+    flex-wrap: wrap !important;
+  }
+
+  .app-panel {
+    margin-bottom: 16px;
+    height: auto;
+    min-height: 200px;
+    width: 100% !important;
+    flex: 0 0 100% !important;
+
+    &:first-child {
+      border-radius: 20px;
+
+      .panel-content {
+        border-radius: 20px;
+      }
+    }
+  }
+
+  .panel-content {
+    padding: 12px;
+
+    h2 {
+      font-size: 16px;
+      margin-bottom: 12px;
+      padding-bottom: 6px;
+    }
+
+    textarea {
+      min-height: 100px;
+    }
+  }
+}
+
+@media (min-width: 769px) and (max-width: 992px) {
+  .app-panel {
+    margin-bottom: 16px;
+    min-height: 300px;
+  }
+}
+
+.resize-handle {
+  width: 2px;
+  background-color: #f0f0f0;
+  transition: background-color 0.2s;
+
+  &:hover {
+    background-color: #1677ff;
+  }
+}