feat: support iOS ui viewer

Author: codematrixer

pyproject.toml

@@ -14,7 +14,8 @@ fastapi = "^0.68.0"
 aiofiles = "^23.1.0"
 uiautomator2 = "^3.0.0"
 facebook-wda = "^1.0.5"
-hmdriver2 = "^1.2.8"
+tidevice = "^0.12.10"
+hmdriver2 = "^1.2.9"
 
 [tool.poetry.extras]
 

uiviewer/_device.py

@@ -2,12 +2,16 @@
 
 import abc
 import tempfile
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Tuple
+from functools import cached_property  # python3.8+
 
 from PIL import Image
+import tidevice
 import adbutils
+import wda
 import uiautomator2 as u2
 from hmdriver2 import hdc
+from hmdriver2.driver import Driver
 from fastapi import HTTPException
 
 from uiviewer._utils import file_to_base64, image_to_base64
@@ -20,10 +24,13 @@ def list_serials(platform: str) -> List[str]:
     if platform == Platform.ANDROID:
         raws = adbutils.AdbClient().device_list()
         devices = [item.serial for item in raws]
-    if platform == Platform.IOS:
-        devices = []
-    if platform == Platform.HARMONY:
+    elif platform == Platform.IOS:
+        raw = tidevice.Usbmux().device_list()
+        devices = [d.udid for d in raw]
+    elif platform == Platform.HARMONY:
         devices = hdc.list_devices()
+    else:
+        raise HTTPException(status_code=200, detail="Unsupported platform")
 
     return devices
 
@@ -41,7 +48,11 @@ def dump_hierarchy(self) -> Dict:
 class HarmonyDevice(DeviceMeta):
     def __init__(self, serial: str):
         self.serial = serial
-        self.client = hdc.HdcWrapper(serial)
+        self.client = Driver(serial)
+
+    @cached_property
+    def display_size(self) -> Tuple:
+        return self.client.display_size
 
     def take_screenshot(self) -> str:
         with tempfile.NamedTemporaryFile(delete=True, suffix=".png") as f:
@@ -50,15 +61,27 @@ def take_screenshot(self) -> str:
             return file_to_base64(path)
 
     def dump_hierarchy(self) -> BaseHierarchy:
-        raw = self.client.dump_hierarchy()
-        return harmony_hierarchy.convert_harmony_hierarchy(raw)
+        packageName, pageName = self.client.current_app()
+        raw: Dict = self.client.dump_hierarchy()
+        hierarchy: Dict = harmony_hierarchy.convert_harmony_hierarchy(raw)
+        return BaseHierarchy(
+            jsonHierarchy=hierarchy,
+            activityName=pageName,
+            packageName=packageName,
+            windowSize=self.display_size,
+            scale=1
+        )
 
 
 class AndroidDevice(DeviceMeta):
     def __init__(self, serial: str):
         self.serial = serial
         self.d: u2.Device = u2.connect(serial)
 
+    @cached_property
+    def window_size(self) -> Tuple:
+        return self.d.window_size()
+
     def take_screenshot(self) -> str:
         img: Image.Image = self.d.screenshot()
         return image_to_base64(img)
@@ -69,20 +92,51 @@ def dump_hierarchy(self) -> BaseHierarchy:
         page_json = android_hierarchy.get_android_hierarchy(page_xml)
         return BaseHierarchy(
             jsonHierarchy=page_json,
-            activity=current['activity'],
+            activityName=current['activity'],
             packageName=current['package'],
-            windowSize=self.d.window_size(),
+            windowSize=self.window_size,
             scale=1
         )
 
 
-def get_device(platform: str, serial: str) -> Union[HarmonyDevice, AndroidDevice]:
+class IosDevice(DeviceMeta):
+    def __init__(self, udid: str, wda_url: str) -> None:
+        self.udid = udid
+        self.client = wda.Client(wda_url)
+
+    @cached_property
+    def scale(self) -> int:
+        return self.client.scale
+
+    @cached_property
+    def window_size(self) -> Tuple:
+        return self.client.window_size()
+
+    def take_screenshot(self) -> str:
+        img: Image.Image = self.client.screenshot()
+        return image_to_base64(img)
+
+    def dump_hierarchy(self) -> BaseHierarchy:
+        data: Dict = self.client.source(format="json")
+        hierarchy: Dict = ios_hierarchy.convert_ios_hierarchy(data, self.scale)
+        return BaseHierarchy(
+            jsonHierarchy=hierarchy,
+            activityName=None,
+            packageName=self.client.bundle_id,
+            windowSize=self.window_size,
+            scale=self.scale
+        )
+
+
+def get_device(platform: str, serial: str, wda_url: str = None) -> Union[HarmonyDevice, AndroidDevice]:
     if serial not in list_serials(platform):
         raise HTTPException(status_code=200, detail="Device not found")
     if platform == Platform.HARMONY:
         return HarmonyDevice(serial)
     elif platform == Platform.ANDROID:
         return AndroidDevice(serial)
+    elif platform == Platform.IOS:
+        return IosDevice(serial, wda_url)
     else:
         raise HTTPException(status_code=200, detail="Unsupported platform")
 
@@ -91,7 +145,7 @@ def get_device(platform: str, serial: str) -> Union[HarmonyDevice, AndroidDevice
 cached_devices = {}
 
 
-def init_device(platform: str, serial: str):
-    device = get_device(platform, serial)
+def init_device(platform: str, serial: str, wda_url: str = None):
+    device = get_device(platform, serial, wda_url)
     cached_devices[(platform, serial)] = device
     return platform, serial

uiviewer/app.py

@@ -3,15 +3,21 @@
 import os
 import webbrowser
 import uvicorn
+from typing import Union,Optional
 
-from fastapi import FastAPI
-from fastapi import Request
-from fastapi import HTTPException
+from fastapi import FastAPI, Request, Query, HTTPException
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import JSONResponse, RedirectResponse
 
+from uiviewer._device import (
+    list_serials,
+    init_device,
+    cached_devices,
+    AndroidDevice,
+    IosDevice,
+    HarmonyDevice
+)
 from uiviewer._models import ApiResponse
-from uiviewer._device import list_serials, init_device, cached_devices
 
 
 app = FastAPI()
@@ -60,21 +66,21 @@ async def get_serials(platform: str):
 
 
 @app.post("/{platform}/{serial}/init", response_model=ApiResponse)
-async def init(platform: str, serial: str):
-    device = init_device(platform, serial)
+async def init(platform: str, serial: str, wdaUrl: Optional[str] = Query(None)):
+    device = init_device(platform, serial, wdaUrl)
     return ApiResponse.doSuccess(device)
 
 
 @app.get("/{platform}/{serial}/screenshot", response_model=ApiResponse)
 async def screenshot(platform: str, serial: str):
-    device = cached_devices.get((platform, serial))
+    device: Union[AndroidDevice, IosDevice, HarmonyDevice] = cached_devices.get((platform, serial))
     data = device.take_screenshot()
     return ApiResponse.doSuccess(data)
 
 
 @app.get("/{platform}/{serial}/hierarchy", response_model=ApiResponse)
 async def dump_hierarchy(platform: str, serial: str):
-    device = cached_devices.get((platform, serial))
+    device: Union[AndroidDevice, IosDevice, HarmonyDevice] = cached_devices.get((platform, serial))
     data = device.dump_hierarchy()
     return ApiResponse.doSuccess(data)
 

uiviewer/parser/harmony_hierarchy.py

@@ -1,20 +1,11 @@
 # -*- coding: utf-8 -*-
 
 import uuid
-from uiviewer._models import BaseHierarchy
+from typing import Dict
 
 
-def convert_harmony_hierarchy(data: dict) -> BaseHierarchy:
-    ret = {
-        "jsonHierarchy": {
-            "_id": "3e7bbe34-c6ed-4a06-abef-f44c3b852467",
-            "children": []
-        },
-        "abilityName": "",
-        "bundleName": "",
-        "windowSize": [0, 0],
-        "scale": 1
-    }
+def convert_harmony_hierarchy(data: Dict) -> Dict:
+    ret = {"_id": str(uuid.uuid4()), "children": []}
 
     def __recursive_convert(node_a):
         node_b = {
@@ -60,32 +51,13 @@ def __recursive_convert(node_a):
         node_b["rect"]["width"] = int(bounds[1].split(",")[0]) - int(bounds[0].split(",")[0])
         node_b["rect"]["height"] = int(bounds[1].split(",")[1]) - int(bounds[0].split(",")[1])
 
-        # TODO Remove unnecessary attributes
-
         children = node_a.get("children", [])
         if children:
             node_b["children"] = [__recursive_convert(child) for child in children]
 
         return node_b
 
     # Recursively convert children of a to match b's structure
-    ret["jsonHierarchy"]["children"] = [__recursive_convert(child) for child in data.get("children", [])]
-
-    # Set windowSize based on the bounds of the first attributes in a
-    first_bounds = data["attributes"].get("bounds", "")
-    if first_bounds:
-        first_bounds = first_bounds.strip("[]").split("][")
-        ret["windowSize"] = [
-            int(first_bounds[1].split(",")[0]) - int(first_bounds[0].split(",")[0]),
-            int(first_bounds[1].split(",")[1]) - int(first_bounds[0].split(",")[1])
-        ]
-
-    # Set abilityName based on the abilityName of the first attributes in the first children of a
-    first_child_attributes = data.get("children", [{}])[0].get("attributes", {})
-    ret["activityName"] = first_child_attributes.get("abilityName", "")
-
-    # Set bundleName based on the bundleName of the first attributes in the first children of a
-    first_child_attributes = data.get("children", [{}])[0].get("attributes", {})
-    ret["packageName"] = first_child_attributes.get("bundleName", "")
+    ret["children"] = [__recursive_convert(child) for child in data.get("children", [])]
 
-    return BaseHierarchy(**ret)
+    return ret

uiviewer/parser/ios_hierarchy.py

@@ -1,12 +1,10 @@
 # -*- coding: utf-8 -*-
 
 import uuid
+from typing import Dict
 
 
-def get_ios_hierarchy(client, scale, md=None):
-    if md:
-        client.appium_settings({"snapshotMaxDepth": md})
-    sourcejson = client.source(format='json')
+def convert_ios_hierarchy(data: Dict, scale: int) -> Dict:
 
     def __travel(node):
         node['_id'] = str(uuid.uuid4())
@@ -18,4 +16,4 @@ def __travel(node):
             __travel(child)
         return node
 
-    return __travel(sourcejson)
+    return __travel(data)

uiviewer/static/css/style.css

@@ -12,9 +12,13 @@ body, html {
     border-color: #3966C3  !important;
     color: white !important;
 }
+
 .el-select .el-input__inner:focus {
     border-color: #3966C3 !important;
 }
+.el-select .el-input__inner::placeholder {
+    color: #a5a2a2;
+}
 .el-select-dropdown,
 .el-select .el-input__inner,
 .el-select-dropdown .el-select-dropdown__item {
@@ -23,6 +27,17 @@ body, html {
   border-color: #212933 !important;
 }
 
+.custom-input .el-input__inner {
+    border-color: #2D3744;
+}
+.custom-input .el-input__inner::placeholder {
+    color: #a5a2a2;
+}
+.custom-input .el-input__inner:focus,
+.custom-input .el-input__inner:hover {
+    border-color: #3966C3;
+}
+
 
 #app {
     display: flex;
@@ -140,19 +155,7 @@ body, html {
 }
 .divider-hover, .divider-dragging {
     background-color: #3679E3;
-}
-  
-.no-border-input ::placeholder {
-    color: #BBBBBB;
-    opacity: 1;
-}
-
-.no-border-input {
-    margin-left: 10px;
-}
-.no-border-input .el-input__inner:focus,
-.no-border-input .el-input__inner:hover {
-    border-color: #3966C3;
+    width: 3px;
 }
 
 .custom-link {

uiviewer/static/index.html

@@ -37,6 +37,14 @@
               </el-option>
             </el-select>
 
+            <el-input
+              class="custom-input"
+              v-if="platform === 'ios'"
+              v-model="wdaUrl"
+              style="width: 250px; margin-right: 20px;"
+              placeholder="Please input WDA url">
+            </el-input>
+
             <el-button 
               :disabled="isConnecting"
               style="margin-right: 10px; width: 100px;"
@@ -89,7 +97,7 @@
             <div class="center" :style="{ width: centerWidth + 'px' }">
                 <p class="region-title">Selected Element Info</p>
                 <el-table
-                  :data="selectedNodeDetailsArray"
+                  :data="selectedNodeDetails"
                   style="width: 100%"
                   class="custom-table">
                   <el-table-column prop="key" label="Key" width="30%"></el-table-column>
@@ -116,18 +124,19 @@
                 <p class="region-title">UI hierarchy
                 </p>
                 <el-input
-                    class="no-border-input"
-                    placeholder="Search for..."
-                    v-model="nodeFilterText">
+                  class="custom-input"
+                  placeholder="Search for ..."
+                  style="margin-left: 10px;"
+                  v-model="nodeFilterText">
                 </el-input>
                 <el-tree
+                    class="custom-tree"
                     ref="treeRef"
                     :data="treeData"
                     :props="defaultProps"
                     @node-click="handleTreeNodeClick"
                     node-key="_id"
                     default-expand-all
-                    show-line
                     :expand-on-click-node="false"
                     :filter-node-method="filterNode">
                 </el-tree>