feat: use photon & sharp for resizing img (#974)

* chore(web-integration): log time consume

* feat(shared): add getPhoton function to load photon module and support image processing methods

* feat(shared): add sharp as external dependency and integrate image processing with sharp in resizeImg function

* feat(shared): update dependencies to @silvia-odwyer/photon and @silvia-odwyer/photon-node, and adjust image processing logic for Node.js and browser environments

* fix(shared): update import statement for photon package to support webpackIgnore for browser environments

* fix(shared): update sharp import type and adjust photon loading logic for better environment detection

* fix(dependencies): downgrade sharp version from 0.34.1 to 0.33.5 in package.json and pnpm-lock.yaml

* fix(shared): simplify photon import statement by removing webpackIgnore comment

* fix(config): add SSR external configuration for @silvia-odwyer/photon in vitest.config.ts files

* fix(shared): replace isNodeEnv checks with ifInNode for better environment detection

Author: quanru

apps/chrome-extension/static/manifest.json

@@ -14,6 +14,19 @@
   "background": {
     "service_worker": "./scripts/worker.js"
   },
+  "content_security_policy": {
+    "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self';"
+  },
+  "web_accessible_resources": [
+    {
+      "resources": [
+        "*.wasm"
+      ],
+      "matches": [
+        "<all_urls>"
+      ]
+    }
+  ],
   "host_permissions": [
     "<all_urls>"
   ],

apps/report/rsbuild.config.ts

@@ -137,6 +137,7 @@ export default defineConfig({
           },
         },
       },
+      externals: ['sharp'],
     },
   },
   output: {

packages/android/src/page/index.ts

@@ -74,7 +74,6 @@ export class AndroidDevice implements AndroidDevicePage {
     this.connectingAdb = (async () => {
       let error: Error | null = null;
       debugPage(`Initializing ADB with device ID: ${this.deviceId}`);
-
       try {
         const androidAdbPath =
           this.options?.androidAdbPath || getAIConfig(MIDSCENE_ADB_PATH);
@@ -139,10 +138,11 @@ ${Object.keys(size)
         return async (...args: any[]) => {
           try {
             debugPage(`adb ${String(prop)} ${args.join(' ')}`);
-            return (originalMethod as (...args: any[]) => any).apply(
-              target,
-              args,
-            );
+            const result = await (
+              originalMethod as (...args: any[]) => any
+            ).apply(target, args);
+            debugPage(`adb ${String(prop)} ${args.join(' ')} end`);
+            return result;
           } catch (error: any) {
             const methodName = String(prop);
             const deviceId = this.deviceId;
@@ -168,6 +168,7 @@ ${Object.keys(size)
     this.uri = uri;
 
     try {
+      debugPage(`Launching app: ${uri}`);
       if (
         uri.startsWith('http://') ||
         uri.startsWith('https://') ||
@@ -359,7 +360,9 @@ ${Object.keys(size)
     const androidScreenshotPath = `/data/local/tmp/midscene_screenshot_${randomUUID()}.png`;
 
     try {
+      debugPage('Taking screenshot via adb.takeScreenshot');
       screenshotBuffer = await adb.takeScreenshot(null);
+      debugPage('adb.takeScreenshot completed');
 
       // make sure screenshotBuffer is not null
       if (!screenshotBuffer) {
@@ -379,25 +382,34 @@ ${Object.keys(size)
       const screenshotPath = getTmpFile('png')!;
 
       try {
+        debugPage('Fallback: taking screenshot via shell screencap');
         try {
           // Take a screenshot and save it locally
           await adb.shell(`screencap -p ${androidScreenshotPath}`);
+          debugPage('adb.shell screencap completed');
         } catch (error) {
+          debugPage('screencap failed, using forceScreenshot');
           await this.forceScreenshot(androidScreenshotPath);
+          debugPage('forceScreenshot completed');
         }
 
+        debugPage('Pulling screenshot file from device');
         await adb.pull(androidScreenshotPath, screenshotPath);
+        debugPage('adb.pull completed');
         screenshotBuffer = await fs.promises.readFile(screenshotPath);
       } finally {
         await adb.shell(`rm -f ${androidScreenshotPath}`);
       }
     }
 
+    debugPage('Resizing screenshot image');
     const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
       width,
       height,
     });
+    debugPage('Image resize completed');
 
+    debugPage('Converting to base64');
     const result = `data:image/jpeg;base64,${resizedScreenshotBuffer.toString('base64')}`;
     debugPage('screenshotBase64 end');
     return result;

packages/android/vitest.config.ts

@@ -38,4 +38,7 @@ export default defineConfig({
   define: {
     __VERSION__: `'${version}'`,
   },
+  ssr: {
+    external: ['@silvia-odwyer/photon'],
+  },
 });

packages/cli/vitest.config.ts

@@ -19,4 +19,7 @@ export default defineConfig({
       '@': path.resolve(__dirname, 'src'),
     },
   },
+  ssr: {
+    external: ['@silvia-odwyer/photon'],
+  },
 });

packages/core/vitest.config.ts

@@ -28,4 +28,7 @@ export default defineConfig({
       '@': path.resolve(__dirname, 'src'),
     },
   },
+  ssr: {
+    external: ['@silvia-odwyer/photon'],
+  },
 });

packages/evaluation/package.json

@@ -31,7 +31,7 @@
     "cli-progress": "3.12.0",
     "dotenv": "16.4.5",
     "playwright": "1.44.1",
-    "sharp": "0.34.1",
+    "sharp": "0.33.5",
     "typescript": "^5.8.3",
     "vitest": "3.0.5"
   },

packages/shared/modern.config.ts

@@ -26,6 +26,7 @@ export default defineConfig({
       env: './src/env.ts',
       types: './src/types/index.ts',
     },
+    externals: ['sharp'],
     target: 'es2020',
     dts: {
       respectExternal: true,

packages/shared/package.json

@@ -56,9 +56,12 @@
     "test:u": "vitest --run -u"
   },
   "dependencies": {
+    "@silvia-odwyer/photon": "0.3.3",
+    "@silvia-odwyer/photon-node": "0.3.3",
     "debug": "4.4.0",
     "jimp": "0.22.12",
-    "js-sha256": "0.11.0"
+    "js-sha256": "0.11.0",
+    "sharp": "0.33.5"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",

packages/shared/src/common.ts

@@ -2,24 +2,21 @@ import { existsSync, mkdirSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { MIDSCENE_RUN_DIR, getAIConfig } from './env';
+import { ifInNode } from './utils';
 
 export const defaultRunDirName = 'midscene_run';
 // Define locally for now to avoid import issues
-export const isNodeEnv =
-  typeof process !== 'undefined' &&
-  process.versions != null &&
-  process.versions.node != null;
 
 export const getMidsceneRunDir = () => {
-  if (!isNodeEnv) {
+  if (!ifInNode) {
     return '';
   }
 
   return getAIConfig(MIDSCENE_RUN_DIR) || defaultRunDirName;
 };
 
 export const getMidsceneRunBaseDir = () => {
-  if (!isNodeEnv) {
+  if (!ifInNode) {
     return '';
   }
 
@@ -49,7 +46,7 @@ export const getMidsceneRunBaseDir = () => {
 export const getMidsceneRunSubDir = (
   subdir: 'dump' | 'cache' | 'report' | 'tmp' | 'log' | 'output',
 ): string => {
-  if (!isNodeEnv) {
+  if (!ifInNode) {
     return '';
   }