feat(core): add continuous screenshot capturing feature and enhance context handling

apps/report/src/components/detail-panel/index.tsx

@@ -49,6 +49,36 @@ const DetailPanel = (): JSX.Element => {
   );
   const imageWidth = useExecutionDump((store) => store.insightWidth);
   const imageHeight = useExecutionDump((store) => store.insightHeight);
+  type RecorderScreenshotEntry = {
+    img: string;
+    ts: number;
+    order: number;
+    timing?: string;
+  };
+  const screenshotEntries: RecorderScreenshotEntry[] =
+    activeTask?.recorder?.flatMap((item) => {
+      const entries: RecorderScreenshotEntry[] = [];
+      const seen = new Set<string>();
+      const addImage = (img: string | undefined, order: number) => {
+        if (!img || seen.has(img)) {
+          return;
+        }
+        seen.add(img);
+        entries.push({
+          img,
+          ts: item.ts,
+          order,
+          timing: item.timing,
+        });
+      };
+
+      addImage(item.screenshot, 0);
+      item.screenshots?.forEach((img, idx) => {
+        addImage(img, idx + 1);
+      });
+
+      return entries;
+    }) || [];
 
   // Check if page context is frozen
   const isPageContextFrozen = Boolean(
@@ -118,20 +148,23 @@ const DetailPanel = (): JSX.Element => {
       content = <div>invalid view</div>;
     }
   } else if (viewType === VIEW_TYPE_SCREENSHOT) {
-    if (activeTask.recorder?.length) {
+    if (screenshotEntries.length) {
       content = (
         <div className="screenshot-item-wrapper scrollable">
-          {activeTask.recorder
-            .filter((item) => item.screenshot)
-            .map((item, index) => {
-              const fullTime = timeStr(item.ts);
-              const str = item.timing
-                ? `${fullTime} / ${item.timing}`
-                : fullTime;
-              return (
-                <ScreenshotItem key={index} time={str} img={item.screenshot!} />
-              );
-            })}
+          {screenshotEntries.map((entry, index) => {
+            const baseTime = timeStr(entry.ts);
+            const label = entry.timing
+              ? `${baseTime} / ${entry.timing}`
+              : baseTime;
+            const suffix = entry.order > 0 ? ` (#${entry.order + 1})` : '';
+            return (
+              <ScreenshotItem
+                key={`${entry.ts}-${entry.order}-${index}`}
+                time={`${label}${suffix}`}
+                img={entry.img}
+              />
+            );
+          })}
         </div>
       );
     } else {

apps/report/src/components/global-hover-preview/index.tsx

@@ -16,13 +16,27 @@ const GlobalHoverPreview = () => {
 
   const images = hoverTask?.recorder
     ?.filter((item) => {
-      let valid = Boolean(item.screenshot);
+      let valid = Boolean(item.screenshot || item.screenshots?.length);
       if (hoverTimestamp) {
         valid = valid && item.ts >= hoverTimestamp;
       }
       return valid;
     })
-    .map((item) => item.screenshot);
+    .flatMap((item) => {
+      const list: string[] = [];
+      const seen = new Set<string>();
+      const addImage = (img?: string) => {
+        if (!img || seen.has(img)) {
+          return;
+        }
+        seen.add(img);
+        list.push(img);
+      };
+
+      addImage(item.screenshot);
+      item.screenshots?.forEach((img) => addImage(img));
+      return list;
+    });
 
   const { x, y } = hoverPreviewConfig || {};
   let left = 0;

apps/report/src/components/timeline/index.tsx

@@ -3,14 +3,17 @@ import * as PIXI from 'pixi.js';
 import { useEffect, useMemo, useRef } from 'react';
 
 import './index.less';
-import type { ExecutionRecorderItem, ExecutionTask } from '@midscene/core';
+import type { ExecutionTask } from '@midscene/core';
 import { getTextureFromCache, loadTexture } from '../pixi-loader';
 import { useAllCurrentTasks, useExecutionDump } from '../store';
 
 interface TimelineItem {
   id: string;
   img: string;
   timeOffset: number;
+  order?: number;
+  overlapIndex?: number;
+  overlapCount?: number;
   x?: number;
   y?: number;
   width?: number;
@@ -254,24 +257,37 @@ const TimelineWidget = (props: {
                 (screenshotHeight / originalHeight) * originalWidth,
               );
 
-              const screenshotX = leftForTimeOffset(screenshot.timeOffset);
-              allScreenshots[index].x = screenshotX;
+              const baseX = leftForTimeOffset(screenshot.timeOffset);
+              let overlapX = baseX;
+
+              if ((screenshot.overlapCount ?? 1) > 1) {
+                const overlapCount = screenshot.overlapCount ?? 1;
+                const overlapIndex = screenshot.overlapIndex ?? 0;
+                const centeredIndex = overlapIndex - (overlapCount - 1) / 2;
+                const overlapGap = Math.min(
+                  Math.max(Math.floor(screenshotWidth * 0.25), 10 * sizeRatio),
+                  Math.max(screenshotWidth, 30 * sizeRatio),
+                );
+                overlapX = baseX + centeredIndex * overlapGap;
+              }
+
+              allScreenshots[index].x = overlapX;
               allScreenshots[index].y = screenshotTop;
               allScreenshots[index].width = screenshotWidth;
               allScreenshots[index].height = screenshotMaxHeight;
 
               const border = new PIXI.Graphics();
               border.lineStyle(sizeRatio, shotBorderColor, 1);
               border.drawRect(
-                screenshotX,
+                overlapX,
                 screenshotTop,
                 screenshotWidth,
                 screenshotMaxHeight,
               );
               border.endFill();
               container.addChild(border);
 
-              screenshotSprite.x = screenshotX;
+              screenshotSprite.x = overlapX;
               screenshotSprite.y = screenshotTop;
               screenshotSprite.width = screenshotWidth;
               screenshotSprite.height = screenshotMaxHeight;
@@ -392,7 +408,9 @@ const TimelineWidget = (props: {
           indicatorContainer.addChild(indicator);
 
           // time string
-          const text = pixiTextForNumber(timeOffsetForLeft(x));
+          const timeToDisplay =
+            closestScreenshot?.timeOffset ?? timeOffsetForLeft(x);
+          const text = pixiTextForNumber(timeToDisplay);
           text.x = x + 5;
           text.y = timeTextTop;
           const textBg = new PIXI.Graphics();
@@ -459,41 +477,79 @@ const Timeline = () => {
   let startingTime = -1;
   let idCount = 1;
   const idTaskMap: Record<string, ExecutionTask> = {};
-  const allScreenshots: TimelineItem[] = allTasks
-    .reduce<(ExecutionRecorderItem & { id: string })[]>((acc, current) => {
-      const recorders = current.recorder || [];
-      recorders.forEach((item) => {
-        if (startingTime === -1 || startingTime > item.ts) {
-          startingTime = item.ts;
-        }
-      });
-      if (
-        current.timing?.start &&
-        (startingTime === -1 || startingTime > current.timing.start)
-      ) {
-        startingTime = current.timing.start;
+  type RecorderEntry = {
+    id: string;
+    img: string;
+    ts: number;
+    order: number;
+    overlapIndex: number;
+    overlapCount: number;
+  };
+
+  const recorderEntries = allTasks.reduce<RecorderEntry[]>((acc, current) => {
+    const recorders = current.recorder || [];
+    recorders.forEach((item) => {
+      if (startingTime === -1 || startingTime > item.ts) {
+        startingTime = item.ts;
       }
-      const recorderItemWithId = recorders.map((item) => {
+    });
+    if (
+      current.timing?.start &&
+      (startingTime === -1 || startingTime > current.timing.start)
+    ) {
+      startingTime = current.timing.start;
+    }
+
+    recorders.forEach((item) => {
+      const imageCandidates = [
+        item.screenshot,
+        ...(item.screenshots ?? []),
+      ].filter((img): img is string => Boolean(img));
+      if (!imageCandidates.length) {
+        return;
+      }
+      const uniqueImages = Array.from(new Set(imageCandidates));
+      const overlapCount = uniqueImages.length;
+      uniqueImages.forEach((img, idx) => {
         const idStr = `id_${idCount++}`;
         idTaskMap[idStr] = current;
-        return {
-          ...item,
+        acc.push({
           id: idStr,
-        };
+          img,
+          ts: item.ts,
+          order: idx,
+          overlapIndex: idx,
+          overlapCount,
+        });
       });
-      return acc.concat(recorderItemWithId || []);
-    }, [])
-    .filter((item) => {
-      return item.screenshot;
-    })
-    .map((recorderItem) => {
-      return {
-        id: recorderItem.id,
-        img: recorderItem.screenshot!,
-        timeOffset: recorderItem.ts - startingTime,
-      };
-    })
-    .sort((a, b) => a.timeOffset - b.timeOffset);
+    });
+
+    return acc;
+  }, []);
+
+  if (startingTime === -1 && recorderEntries.length) {
+    startingTime = recorderEntries[0]!.ts;
+  }
+
+  if (startingTime === -1) {
+    startingTime = 0;
+  }
+
+  const allScreenshots: TimelineItem[] = recorderEntries
+    .map((entry) => ({
+      id: entry.id,
+      img: entry.img,
+      timeOffset: entry.ts - startingTime,
+      order: entry.order,
+      overlapIndex: entry.overlapIndex,
+      overlapCount: entry.overlapCount,
+    }))
+    .sort((a, b) => {
+      if (a.timeOffset === b.timeOffset) {
+        return (a.order ?? 0) - (b.order ?? 0);
+      }
+      return a.timeOffset - b.timeOffset;
+    });
 
   const itemOnTap = (item: TimelineItem) => {
     const task = idTaskMap[item.id];

packages/cli/src/create-yaml-player.ts

@@ -149,6 +149,7 @@ export async function createYamlPlayer(
         const agent = new AgentOverChromeBridge({
           closeNewTabsAfterDisconnect: webTarget.closeNewTabsAfterDisconnect,
           cacheId: fileName,
+          continuousScreenshot: webTarget.continuousScreenshot,
         });
 
         if (webTarget.bridgeMode === 'newTabWithUrl') {

packages/cli/tests/midscene_scripts/online/video-player.yaml

@@ -0,0 +1,20 @@
+web:
+  url: https://www.bilibili.com
+  headless: false
+  continuousScreenshot: 
+    enabled: true
+    intervalMs: 1000
+    maxCount: 10
+
+
+
+tasks:
+  - name: 随机播放视频
+    flow:
+      - ai: 随机选择一个视频播放
+      - sleep: 3000
+
+  - name: 检查结果
+    flow:
+      - aiAssert: 视频播放区域有控制面板浮层
+        screenshotListIncluded: true