feat: fetch and analyze screenshots of automate and app-automate

Author: tech-sushant

src/index.ts

@@ -13,6 +13,7 @@ import addAccessibilityTools from "./tools/accessibility.js";
 import addTestManagementTools from "./tools/testmanagement.js";
 import addAppAutomationTools from "./tools/appautomate.js";
 import addFailureLogsTools from "./tools/getFailureLogs.js";
+import addAppAutomateTools from "./tools/automate.js";
 import { trackMCP } from "./lib/instrumentation.js";
 
 function registerTools(server: McpServer) {
@@ -24,6 +25,7 @@ function registerTools(server: McpServer) {
   addTestManagementTools(server);
   addAppAutomationTools(server);
   addFailureLogsTools(server);
+  addAppAutomateTools(server);
 }
 
 // Create an MCP server

src/lib/constants.ts

@@ -0,0 +1,20 @@
+export const SessionType = {
+  Automate: "automate",
+  AppAutomate: "app-automate",
+} as const;
+
+export const AutomateLogType = {
+  NetworkLogs: "networkLogs",
+  SessionLogs: "sessionLogs",
+  ConsoleLogs: "consoleLogs",
+} as const;
+
+export const AppAutomateLogType = {
+  DeviceLogs: "deviceLogs",
+  AppiumLogs: "appiumLogs",
+  CrashLogs: "crashLogs",
+} as const;
+
+export type SessionType = typeof SessionType[keyof typeof SessionType]; 
+export type AutomateLogType = typeof AutomateLogType[keyof typeof AutomateLogType];
+export type AppAutomateLogType = typeof AppAutomateLogType[keyof typeof AppAutomateLogType];

src/lib/utils.ts

@@ -27,13 +27,13 @@ export interface HarEntry {
   time?: number;
 }
 
-/**
- * Compresses a base64 image intelligently to keep it under 1 MB if needed.
- */
+const ONE_MB = 1048576;
+
+//Compresses a base64 image intelligently to keep it under 1 MB if needed.
 export async function maybeCompressBase64(base64: string): Promise<string> {
   const buffer = Buffer.from(base64, "base64");
 
-  if (buffer.length <= 1048576) {
+  if (buffer.length <= ONE_MB) {
     return base64;
   }
 

src/tools/automate-utils/fetch-screenshots.ts

@@ -0,0 +1,80 @@
+import config from "../../config.js";
+import { assertOkResponse, maybeCompressBase64 } from "../../lib/utils.js";
+import { SessionType } from "../../lib/constants.js";
+
+//Extracts screenshot URLs from BrowserStack session logs
+async function extractScreenshotUrls(sessionId: string, sessionType: SessionType): Promise<string[]> {
+  
+  const credentials = `${config.browserstackUsername}:${config.browserstackAccessKey}`;
+  const auth = Buffer.from(credentials).toString("base64");
+
+  const baseUrl = `https://api.browserstack.com/${sessionType === SessionType.Automate ? 'automate' : 'app-automate'}`;
+
+  const url = `${baseUrl}/sessions/${sessionId}/logs`;
+  const response = await fetch(url, {
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Basic ${auth}`,
+    },
+  });
+
+  await assertOkResponse(response, "Session");
+
+  const text = await response.text();
+    
+  const urls: string[] = [];
+  const SCREENSHOT_PATTERN = /REQUEST.*GET.*\/screenshot/;
+  const RESPONSE_VALUE_PATTERN = /"value"\s*:\s*"([^"]+)"/;
+
+  // Split logs into lines and process them
+  const lines = text.split("\n");
+
+  for (let i = 0; i < lines.length - 1; i++) {
+    const currentLine = lines[i];
+    const nextLine = lines[i + 1];
+
+    if (SCREENSHOT_PATTERN.test(currentLine)) {
+      const match = nextLine.match(RESPONSE_VALUE_PATTERN);
+      if (match && match[1]) {
+        urls.push(match[1]);
+      }
+    }
+  }
+
+  return urls;
+}
+
+//Converts screenshot URLs to base64 encoded images
+async function convertUrlsToBase64(
+  urls: string[],
+): Promise<Array<{ url: string; base64: string }>> {
+  const screenshots = await Promise.all(
+    urls.map(async (url) => {
+      const response = await fetch(url);
+      const arrayBuffer = await response.arrayBuffer();
+      const base64 = Buffer.from(arrayBuffer).toString("base64");
+
+      // Compress the base64 image if needed
+      const compressedBase64 = await maybeCompressBase64(base64);
+
+      return {
+        url,
+        base64: compressedBase64,
+      };
+    }),
+  );
+
+  return screenshots;
+}
+
+//Fetches and converts screenshot URLs to base64 encoded images
+export async function fetchAutomationScreenshots(sessionId: string, sessionType: SessionType = SessionType.Automate) {
+  const urls = await extractScreenshotUrls(sessionId, sessionType);
+  if (urls.length === 0) {
+    return [];
+  }
+
+  // Take only the last 5 URLs
+  const lastFiveUrls = urls.slice(-5);
+  return await convertUrlsToBase64(lastFiveUrls);
+}

src/tools/automate.ts

@@ -0,0 +1,84 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { fetchAutomationScreenshots } from "./automate-utils/fetch-screenshots.js";
+import { SessionType } from "../lib/constants.js";
+import { trackMCP } from "../lib/instrumentation.js";
+import logger from "../logger.js";
+
+// Tool function that fetches and processes screenshots from BrowserStack Automate session
+export async function fetchAutomationScreenshotsTool(args: {
+  sessionId: string;
+  sessionType: SessionType;
+}): Promise<CallToolResult> {
+  try {
+    const screenshots = await fetchAutomationScreenshots(args.sessionId, args.sessionType);
+
+    if (screenshots.length === 0) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: "No screenshots found in the session or some unexpected error occurred",
+          },
+        ],
+        isError: true,
+      };
+    }
+
+    const results = screenshots.map((screenshot, index) => ({
+      type: "image" as const,
+      text: `Screenshot ${index + 1}`,
+      data: screenshot.base64,
+      mimeType: "image/png",
+      metadata: { url: screenshot.url },
+    }));
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: `Retrieved ${screenshots.length} screenshot(s) from the end of the session.`,
+        },
+        ...results,
+      ],
+    };
+  } catch (error) {
+    logger.error("Error during fetching screenshots", error);
+    throw error;
+  }
+}
+
+//Registers the fetchAutomationScreenshots tool with the MCP server
+export default function addAutomationTools(server: McpServer) {
+  server.tool(
+    "fetchAutomationScreenshots",
+    "Fetch and process screenshots from a BrowserStack Automate session",
+    {
+      sessionId: z
+        .string()
+        .describe("The BrowserStack session ID to fetch screenshots from"),
+      sessionType: z
+        .enum([SessionType.Automate, SessionType.AppAutomate])
+        .describe("Type of BrowserStack session")
+    },
+    async (args) => {
+      try {
+        trackMCP("fetchAutomationScreenshots", server.server.getClientVersion()!);
+        return await fetchAutomationScreenshotsTool(args);
+      } catch (error) {
+        trackMCP("fetchAutomationScreenshots", server.server.getClientVersion()!,error);
+        const errorMessage =
+          error instanceof Error ? error.message : "Unknown error";
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error during fetching automate screenshots: ${errorMessage}`,
+            },
+          ],
+        };
+      }
+    },
+  );
+}

src/tools/getFailureLogs.ts

@@ -15,31 +15,10 @@ import {
   retrieveCrashLogs,
 } from "./failurelogs-utils/app-automate.js";
 import { trackMCP } from "../lib/instrumentation.js";
+import { AppAutomateLogType, AutomateLogType, SessionType } from "../lib/constants.js";
 
-const AutomateLogType = {
-  NetworkLogs: "networkLogs",
-  SessionLogs: "sessionLogs",
-  ConsoleLogs: "consoleLogs",
-} as const;
-
-const AppAutomateLogType = {
-  DeviceLogs: "deviceLogs",
-  AppiumLogs: "appiumLogs",
-  CrashLogs: "crashLogs",
-} as const;
-
-const SessionType = {
-  Automate: "automate",
-  AppAutomate: "app-automate",
-} as const;
-
-// Type aliases
-type AutomateLogTypeValues =
-  (typeof AutomateLogType)[keyof typeof AutomateLogType];
-type AppAutomateLogTypeValues =
-  (typeof AppAutomateLogType)[keyof typeof AppAutomateLogType];
-type LogType = AutomateLogTypeValues | AppAutomateLogTypeValues;
-type SessionTypeValues = (typeof SessionType)[keyof typeof SessionType];
+type LogType = AutomateLogType | AppAutomateLogType;
+type SessionTypeValues = SessionType;
 
 // Main log fetcher function
 export async function getFailureLogs(args: {
@@ -63,10 +42,10 @@ export async function getFailureLogs(args: {
   // Validate log types and collect errors
   validLogTypes = args.logTypes.filter((logType) => {
     const isAutomate = Object.values(AutomateLogType).includes(
-      logType as AutomateLogTypeValues,
+      logType as AutomateLogType,
     );
     const isAppAutomate = Object.values(AppAutomateLogType).includes(
-      logType as AppAutomateLogTypeValues,
+      logType as AppAutomateLogType,
     );
 
     if (!isAutomate && !isAppAutomate) {