chore(shared): add localImg2Base64 method (#979)

* chore(core): remove unused elementSnapshot

* chore(shared): remove unused code

* chore(shared): reafactor localImg2Base64

* chore(shared): log content-type when url is not a image

Author: EAGzzyCSL

packages/core/src/image/index.ts

@@ -1,9 +1,9 @@
 export {
   imageInfo,
   imageInfoOfBase64,
-  base64Encoded,
+  localImg2Base64,
+  httpImg2Base64,
   resizeImg,
-  transformImgPathToBase64,
   saveBase64Image,
   zoomForGPT4o,
 } from '@midscene/shared/img';

packages/core/tests/ai/connectivity.test.ts

@@ -2,7 +2,7 @@ import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { AIActionType } from '@/ai-model/common';
 import { call, callToGetJSONObject } from '@/ai-model/service-caller';
-import { base64Encoded } from '@/image';
+import { localImg2Base64 } from '@/image';
 import dotenv from 'dotenv';
 import { getFixture } from 'tests/utils';
 import { beforeAll, describe, expect, it, vi } from 'vitest';
@@ -121,7 +121,7 @@ vi.setConfig({
               {
                 type: 'image_url',
                 image_url: {
-                  url: base64Encoded(imagePath),
+                  url: localImg2Base64(imagePath),
                   detail: 'high',
                 },
               },

packages/core/tests/ai/streaming.test.ts

@@ -2,7 +2,7 @@ import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { AIActionType } from '@/ai-model/common';
 import { call } from '@/ai-model/service-caller';
-import { base64Encoded } from '@/image';
+import { localImg2Base64 } from '@/image';
 import type { AIUsageInfo, CodeGenerationChunk } from '@/types';
 import dotenv from 'dotenv';
 import { getFixture } from 'tests/utils';
@@ -104,7 +104,7 @@ describe(
               {
                 type: 'image_url',
                 image_url: {
-                  url: base64Encoded(imagePath),
+                  url: localImg2Base64(imagePath),
                   detail: 'high',
                 },
               },

packages/core/tests/evaluation.ts

@@ -2,7 +2,7 @@ import { existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { describeUserPage } from '@/index';
 import { vlLocateMode } from '@midscene/shared/env';
-import { base64Encoded, imageInfoOfBase64 } from '@midscene/shared/img';
+import { imageInfoOfBase64, localImg2Base64 } from '@midscene/shared/img';
 
 export async function buildContext(targetDir: string): Promise<{
   context: {
@@ -23,7 +23,7 @@ export async function buildContext(targetDir: string): Promise<{
     targetDir,
     existsSync(path.join(targetDir, 'input.png')) ? 'input.png' : 'input.jpeg',
   );
-  const originalScreenshotBase64 = base64Encoded(originalInputImgP);
+  const originalScreenshotBase64 = localImg2Base64(originalInputImgP);
 
   const resizeOutputImgP = path.join(targetDir, 'output_without_text.png');
   const snapshotJsonPath = path.join(targetDir, 'element-snapshot.json');
@@ -58,13 +58,12 @@ export async function buildContext(targetDir: string): Promise<{
   }
 
   const snapshotJson = readFileSync(snapshotJsonPath, { encoding: 'utf-8' });
-  const elementSnapshot = JSON.parse(snapshotJson);
   const elementTree = JSON.parse(
     readFileSync(elementTreeJsonPath, { encoding: 'utf-8' }),
   );
   const screenshotBase64 = vlLocateMode()
     ? originalScreenshotBase64
-    : base64Encoded(resizeOutputImgP);
+    : localImg2Base64(resizeOutputImgP);
 
   const size = await imageInfoOfBase64(screenshotBase64);
   const baseContext = {

packages/core/tests/utils.ts

@@ -1,11 +1,7 @@
-import { readFileSync, writeFileSync } from 'node:fs';
-import path, { join } from 'node:path';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
 import type { callAiFn } from '@/ai-model/common';
-import {
-  base64Encoded,
-  imageInfoOfBase64,
-  transformImgPathToBase64,
-} from '@/image';
+import { localImg2Base64 } from '@/image';
 import Insight from '@/insight';
 import type { AIElementLocatorResponse, BaseElement, UIContext } from '@/types';
 import { NodeType } from '@midscene/shared/constants';
@@ -31,7 +27,7 @@ export function sleep(ms: number) {
 export function fakeInsight(content: string) {
   const screenshot = getFixture('baidu.png');
   const basicContext = {
-    screenshotBase64: base64Encoded(screenshot),
+    screenshotBase64: localImg2Base64(screenshot),
     size: { width: 1920, height: 1080 },
     content: [
       {
@@ -86,30 +82,3 @@ export function fakeInsight(content: string) {
 
   return insight;
 }
-
-export function generateUIContext(testDataPath: string) {
-  return async () => {
-    const screenshotBase64 = await transformImgPathToBase64(
-      path.join(testDataPath, 'input.png'),
-    );
-    const size = await imageInfoOfBase64(screenshotBase64);
-
-    const captureElementSnapshot = readFileSync(
-      path.join(testDataPath, 'element-snapshot.json'),
-      'utf-8',
-    );
-
-    // align element
-    const elementsInfo = JSON.parse(captureElementSnapshot) as BaseElement[];
-
-    const baseContext = {
-      size: { width: size.width, height: size.height },
-      content: elementsInfo,
-      screenshotBase64: `data:image/png;base64,${screenshotBase64}`,
-    };
-
-    return {
-      ...baseContext,
-    };
-  };
-}

packages/evaluation/tests/util.ts

@@ -4,9 +4,9 @@ import type { PlanningAIResponse, Rect } from '@midscene/core';
 import { NodeType } from '@midscene/shared/constants';
 import { vlLocateMode } from '@midscene/shared/env';
 import {
-  base64Encoded,
   compositeElementInfoImg,
   imageInfoOfBase64,
+  localImg2Base64,
 } from '@midscene/shared/img';
 import { parseContextFromWebPage } from '@midscene/web';
 
@@ -197,7 +197,7 @@ export function getCases(
 }
 
 export async function buildContextByImage(imagePath: string) {
-  const screenshotBase64 = base64Encoded(imagePath);
+  const screenshotBase64 = localImg2Base64(imagePath);
   const size = await imageInfoOfBase64(screenshotBase64);
 
   const fakePage = {
@@ -221,7 +221,7 @@ export async function buildContextByImage(imagePath: string) {
 export async function buildContext(pageName: string) {
   const targetDir = path.join(__dirname, '../page-data/', pageName);
   const screenshotBase64Path = path.join(targetDir, 'input.png');
-  const screenshotBase64 = base64Encoded(screenshotBase64Path);
+  const screenshotBase64 = localImg2Base64(screenshotBase64Path);
   const size = await imageInfoOfBase64(screenshotBase64);
 
   const fakePage = {

packages/shared/src/img/index.ts

@@ -2,21 +2,20 @@ export {
   imageInfo,
   imageInfoOfBase64,
   bufferFromBase64,
-  base64Encoded,
   isValidPNGImageBuffer,
 } from './info';
 export {
-  trimImage,
   resizeImg,
   resizeImgBase64,
-  transformImgPathToBase64,
   zoomForGPT4o,
   saveBase64Image,
   paddingToMatchBlock,
   paddingToMatchBlockByBase64,
   cropByRect,
   jimpFromBase64,
   jimpToBase64,
+  localImg2Base64,
+  httpImg2Base64,
 } from './transform';
 export { processImageElementInfo, compositeElementInfoImg } from './box-select';
 export { drawBoxOnImage, savePositionImg } from './draw-box';

packages/shared/src/img/info.ts

@@ -1,6 +1,5 @@
 import assert from 'node:assert';
 import { Buffer } from 'node:buffer';
-import { readFileSync } from 'node:fs';
 import type Jimp from 'jimp';
 import getJimp from './get-jimp';
 
@@ -64,31 +63,6 @@ export async function bufferFromBase64(imageBase64: string): Promise<Buffer> {
   return Buffer.from(base64Data, 'base64');
 }
 
-/**
- * Encodes an image file to a base64 encoded string
- *
- * @param image The path of the image file
- * @param withHeader Determine whether to return data including the file header information, the default is true
- *
- * @returns The base64 encoded string of the image file, which may or may not include header information depending on the withHeader parameter
- *
- * @throws When the image type is not supported, an error will be thrown
- */
-export function base64Encoded(image: string, withHeader = true) {
-  // get base64 encoded image
-  const imageBuffer = readFileSync(image);
-  if (!withHeader) {
-    return imageBuffer.toString('base64');
-  }
-  if (image.endsWith('png')) {
-    return `data:image/png;base64,${imageBuffer.toString('base64')}`;
-  }
-  if (image.endsWith('jpg') || image.endsWith('jpeg')) {
-    return `data:image/jpeg;base64,${imageBuffer.toString('base64')}`;
-  }
-  throw new Error('unsupported image type');
-}
-
 /**
  * Check if the Buffer is a valid PNG image
  * @param buffer The Buffer to check

packages/shared/src/img/transform.ts

@@ -1,11 +1,13 @@
 import assert from 'node:assert';
 import { Buffer } from 'node:buffer';
-
+import { readFileSync } from 'node:fs';
 import getDebug from 'debug';
 import type Jimp from 'jimp';
 import type { Rect } from 'src/types';
 import getJimp from './get-jimp';
 const debugImg = getDebug('img');
+import path from 'node:path';
+
 /**
 /**
  * Saves a Base64-encoded image to a file
@@ -34,22 +36,6 @@ export async function saveBase64Image(options: {
   debugImg(`saveBase64Image done: ${options.outputPath}`);
 }
 
-/**
- * Transforms an image path into a base64-encoded string
- * @param inputPath - The path of the image file to be encoded
- * @returns A Promise that resolves to a base64-encoded string representing the image file
- */
-export async function transformImgPathToBase64(inputPath: string) {
-  // Use Jimp to process images and generate base64 data
-  debugImg(`transformImgPathToBase64 start: ${inputPath}`);
-  const Jimp = await getJimp();
-  const image = await Jimp.read(inputPath);
-  const buffer = await image.getBufferAsync(Jimp.MIME_JPEG);
-  const res = buffer.toString('base64');
-  debugImg(`transformImgPathToBase64 done: ${inputPath}`);
-  return res;
-}
-
 /**
  * Resizes an image from a base64-encoded string
  *
@@ -168,50 +154,6 @@ export function zoomForGPT4o(originalWidth: number, originalHeight: number) {
   };
 }
 
-/**
- * Trims an image and returns the trimming information, including the offset from the left and top edges, and the trimmed width and height
- *
- * @param image - The image to be trimmed. This can be a file path or a Buffer object containing the image data
- * @returns A Promise that resolves to an object containing the trimming information. If the image does not need to be trimmed, this object will be null
- */
-export async function trimImage(image: string | Buffer): Promise<{
-  trimOffsetLeft: number; // attention: trimOffsetLeft is a negative number
-  trimOffsetTop: number; // so as trimOffsetTop
-  width: number;
-  height: number;
-} | null> {
-  const Jimp = await getJimp();
-  const jimpImage = await Jimp.read(
-    Buffer.isBuffer(image) ? image : Buffer.from(image),
-  );
-  const { width, height } = jimpImage.bitmap;
-
-  if (width <= 3 || height <= 3) {
-    return null;
-  }
-
-  const trimmedImage = jimpImage.autocrop();
-  const { width: trimmedWidth, height: trimmedHeight } = trimmedImage.bitmap;
-
-  const trimOffsetLeft = (width - trimmedWidth) / 2;
-  const trimOffsetTop = (height - trimmedHeight) / 2;
-
-  if (trimOffsetLeft === 0 && trimOffsetTop === 0) {
-    return null;
-  }
-
-  return {
-    trimOffsetLeft: -trimOffsetLeft,
-    trimOffsetTop: -trimOffsetTop,
-    width: trimmedWidth,
-    height: trimmedHeight,
-  };
-}
-
-export function prependBase64Header(base64: string, mimeType = 'image/png') {
-  return `data:${mimeType};base64,${base64}`;
-}
-
 export async function jimpFromBase64(base64: string): Promise<Jimp> {
   const Jimp = await getJimp();
   const imageBuffer = await bufferFromBase64(base64);
@@ -269,3 +211,41 @@ export async function jimpToBase64(image: Jimp): Promise<string> {
   const Jimp = await getJimp();
   return image.getBase64Async(Jimp.MIME_JPEG);
 }
+
+export const httpImg2Base64 = async (url: string): Promise<string> => {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch image: ${url}`);
+  }
+  const contentType = response.headers.get('content-type');
+  if (!contentType) {
+    throw new Error(`Failed to fetch image: ${url}`);
+  }
+  assert(
+    contentType.startsWith('image/'),
+    `The url ${url} is not a image, because of content-type in header is ${contentType}.`,
+  );
+  const buffer = Buffer.from(await response.arrayBuffer());
+  return `data:${contentType};base64,${buffer.toString('base64')}`;
+};
+
+/**
+ * Convert image file to base64 string
+ * Because this method is synchronous, the npm package `sharp` cannot be used to detect the file type.
+ * TODO: convert to webp to reduce base64 size.
+ */
+export const localImg2Base64 = (
+  imgPath: string,
+  withoutHeader = false,
+): string => {
+  const body = readFileSync(imgPath).toString('base64');
+  if (withoutHeader) {
+    return body;
+  }
+
+  // Detect image type by extname.
+  const type = path.extname(imgPath).slice(1);
+  const finalType = type === 'svg' ? 'svg+xml' : type || 'jpg';
+
+  return `data:image/${finalType};base64,${body}`;
+};