diff --git a/apps/report/package.json b/apps/report/package.json index cafc4dfa64..abc3946f64 100644 --- a/apps/report/package.json +++ b/apps/report/package.json @@ -23,8 +23,6 @@ "@rsbuild/plugin-type-check": "^1.3.2", "@types/chrome": "0.0.279", "antd": "^5.21.6", - "pixi-filters": "6.0.5", - "pixi.js": "8.1.1", "react": "18.3.1", "react-dom": "18.3.1", "react-resizable-panels": "2.0.22", diff --git a/apps/report/src/App.less b/apps/report/src/App.less index efe35a4e26..25d2d7e69d 100644 --- a/apps/report/src/App.less +++ b/apps/report/src/App.less @@ -81,12 +81,27 @@ footer.mt-8 { padding: 0 8px; } +.main-layout { + display: flex; + flex-direction: row; + flex-grow: 1; + height: 100%; + overflow: hidden; +} + .page-side { height: calc(100% - 8px); background: #f2f4f7; - padding-right: 8px; margin-bottom: 8px; - overflow-x: auto; + flex-shrink: 0; + overflow-x: hidden; + overflow-y: auto; +} + +.resize-handle { + width: 8px; + flex-shrink: 0; + cursor: col-resize; } [data-theme='dark'] .page-side { @@ -181,6 +196,8 @@ footer.mt-8 { border-radius: 16px; background: #fff; margin-bottom: 8px; + min-width: 0; + overflow: hidden; .main-right-header { height: 50px; diff --git a/apps/report/src/App.tsx b/apps/report/src/App.tsx index 0fc0719f76..8ddab1f99c 100644 --- a/apps/report/src/App.tsx +++ b/apps/report/src/App.tsx @@ -31,6 +31,8 @@ import type { } from './types'; let globalRenderCount = 1; +const SIDEBAR_WIDTH_KEY = 'midscene-sidebar-width'; +const DEFAULT_SIDEBAR_WIDTH = 280; function Visualizer(props: VisualizerProps): JSX.Element { const { dumps } = props; @@ -53,7 +55,10 @@ function Visualizer(props: VisualizerProps): JSX.Element { const modelBriefs = useExecutionDump((store) => store.modelBriefs); const reset = useExecutionDump((store) => store.reset); const [mainLayoutChangeFlag, setMainLayoutChangeFlag] = useState(0); - const mainLayoutChangedRef = useRef(false); + const [sidebarWidth, setSidebarWidth] = useState(() => { + const saved = localStorage.getItem(SIDEBAR_WIDTH_KEY); + return saved ? Number(saved) : DEFAULT_SIDEBAR_WIDTH; + }); const dump = useExecutionDump((store) => store.dump); const [timelineCollapsed, setTimelineCollapsed] = useState(false); const { @@ -153,62 +158,65 @@ function Visualizer(props: VisualizerProps): JSX.Element { ); mainContent = ( - { - if (!mainLayoutChangedRef.current) { - setMainLayoutChangeFlag((prev) => prev + 1); - } - }} - > - -
- -
- - +
+ +
+
{ - if (mainLayoutChangedRef.current && !isChanging) { + onMouseDown={(e) => { + e.preventDefault(); + const startX = e.clientX; + const startWidth = sidebarWidth; + let latestWidth = startWidth; + const onMouseMove = (ev: MouseEvent) => { + latestWidth = Math.max( + 200, + Math.min(500, startWidth + ev.clientX - startX), + ); + setSidebarWidth(latestWidth); + }; + const onMouseUp = () => { + document.removeEventListener('mousemove', onMouseMove); + document.removeEventListener('mouseup', onMouseUp); + localStorage.setItem(SIDEBAR_WIDTH_KEY, String(latestWidth)); setMainLayoutChangeFlag((prev) => prev + 1); - } - mainLayoutChangedRef.current = isChanging; + }; + document.addEventListener('mousemove', onMouseMove); + document.addEventListener('mouseup', onMouseUp); }} /> - -
-
setTimelineCollapsed(!timelineCollapsed)} - style={{ cursor: 'pointer', userSelect: 'none' }} +
+
setTimelineCollapsed(!timelineCollapsed)} + style={{ cursor: 'pointer', userSelect: 'none' }} + > + - - ▼ - - Record -
- {!timelineCollapsed && } -
{content}
+ ▼ + + Record
- - + {!timelineCollapsed && } +
{content}
+
+
); } diff --git a/apps/report/src/components/pixi-loader/index.tsx b/apps/report/src/components/pixi-loader/index.tsx deleted file mode 100644 index 5310120470..0000000000 --- a/apps/report/src/components/pixi-loader/index.tsx +++ /dev/null @@ -1,24 +0,0 @@ -import 'pixi.js/unsafe-eval'; -import * as PIXI from 'pixi.js'; - -const globalTextureMap = new Map(); - -export const loadTexture = async (img: string) => { - if (globalTextureMap.has(img)) return; - return PIXI.Assets.load(img).then((texture) => { - globalTextureMap.set(img, texture); - }); -}; - -export const getTextureFromCache = (name: string) => { - return globalTextureMap.get(name); -}; - -export const getTexture = async (name: string) => { - if (globalTextureMap.has(name)) { - return globalTextureMap.get(name); - } - - await loadTexture(name); - return globalTextureMap.get(name); -}; diff --git a/apps/report/src/components/sidebar/index.less b/apps/report/src/components/sidebar/index.less index ea65939a5d..3ec533cc61 100644 --- a/apps/report/src/components/sidebar/index.less +++ b/apps/report/src/components/sidebar/index.less @@ -37,9 +37,13 @@ cursor: pointer; display: flex; align-items: center; + justify-content: center; + padding: 6px; + border-radius: 8px; + transition: all 0.2s; &:hover { - background: #f5f5f5; + background: #e6e8eb; } } } @@ -456,7 +460,7 @@ .page-nav-left { .page-nav-toolbar .icon-button { &:hover { - background: rgba(255, 255, 255, 0.08); + background: #2c2e31; } svg { diff --git a/apps/report/src/components/store/index.tsx b/apps/report/src/components/store/index.tsx index c1ffe3cc65..3e86e1942f 100644 --- a/apps/report/src/components/store/index.tsx +++ b/apps/report/src/components/store/index.tsx @@ -59,6 +59,7 @@ export interface DumpStoreType { modelBriefs: string[]; insightWidth: number | null; insightHeight: number | null; + deviceType: string | undefined; activeExecution: ExecutionDump | null; activeExecutionAnimation: AnimationScript[] | null; activeTask: ExecutionTask | null; @@ -94,6 +95,7 @@ export const useExecutionDump = create((set, get) => { modelBriefs: [], insightWidth: null, insightHeight: null, + deviceType: undefined, activeTask: null, activeExecution: null, activeExecutionAnimation: null, @@ -159,6 +161,7 @@ export const useExecutionDump = create((set, get) => { height, modelBriefs, sdkVersion, + deviceType, } = allScriptsInfo; set({ @@ -167,6 +170,7 @@ export const useExecutionDump = create((set, get) => { insightHeight: height, modelBriefs, sdkVersion, + deviceType, }); const replayAvailable = allScripts.length > 0; diff --git a/apps/report/src/components/timeline/index.tsx b/apps/report/src/components/timeline/index.tsx index 94fbbe950d..0a02e78588 100644 --- a/apps/report/src/components/timeline/index.tsx +++ b/apps/report/src/components/timeline/index.tsx @@ -1,11 +1,8 @@ -import * as PIXI from 'pixi.js'; -/* eslint-disable max-lines */ -import { useEffect, useMemo, useRef } from 'react'; +import { useEffect, useRef } from 'react'; import './index.less'; import type { ExecutionRecorderItem, ExecutionTask } from '@midscene/core'; import { useTheme } from '@midscene/visualizer'; -import { getTextureFromCache, loadTexture } from '../pixi-loader'; import { useAllCurrentTasks, useExecutionDump } from '../store'; interface TimelineItem { @@ -29,18 +26,17 @@ interface HighlightMask { endMs: number; } -// Function to clone a sprite -function cloneSprite(sprite: PIXI.Sprite) { - const clonedSprite = new PIXI.Sprite(sprite.texture); - - // Copy properties - clonedSprite.position.copyFrom(sprite.position); - clonedSprite.scale.copyFrom(sprite.scale); - clonedSprite.rotation = sprite.rotation; - clonedSprite.alpha = sprite.alpha; - clonedSprite.visible = sprite.visible; +function hexToCSS(hex: number): string { + return `#${hex.toString(16).padStart(6, '0')}`; +} - return clonedSprite; +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = src; + }); } const TimelineWidget = (props: { @@ -51,29 +47,24 @@ const TimelineWidget = (props: { highlightMask?: HighlightMask; hoverMask?: HighlightMask; }): JSX.Element => { - const domRef = useRef(null); // Should be HTMLDivElement not HTMLInputElement - const appRef = useRef(null); + const domRef = useRef(null); + const canvasRef = useRef(null); + const stateRef = useRef<{ + imgCache: Map; + hoverX: number | null; + highlightMask?: HighlightMask; + hoverMask?: HighlightMask; + }>({ + imgCache: new Map(), + hoverX: null, + highlightMask: props.highlightMask, + hoverMask: props.hoverMask, + }); - // Detect dark mode const { isDarkMode } = useTheme(); - const gridsContainer = useMemo(() => new PIXI.Container(), []); - const screenshotsContainer = useMemo(() => new PIXI.Container(), []); - const highlightMaskContainer = useMemo(() => new PIXI.Container(), []); - const containerUpdaterRef = useRef( - // eslint-disable-next-line @typescript-eslint/no-empty-function - ( - _s: number | undefined, - _e: number | undefined, - _hs: number | undefined, - _he: number | undefined, - ) => {}, - ); - const indicatorContainer = useMemo(() => new PIXI.Container(), []); - const allScreenshots = props.screenshots || []; let maxTime = 500; - if (allScreenshots.length >= 2) { maxTime = Math.max( allScreenshots[allScreenshots.length - 1].timeOffset, @@ -82,9 +73,8 @@ const TimelineWidget = (props: { } const sizeRatio = 2; - const BASE_HEIGHT = 110; // Match @base-height in index.less + const BASE_HEIGHT = 110; - // Color configuration based on theme const titleBg = isDarkMode ? 0x1f1f1f : 0xffffff; const sideBg = isDarkMode ? 0x1f1f1f : 0xffffff; const gridTextColor = isDarkMode ? 0xd9d9d9 : 0x000000; @@ -98,30 +88,25 @@ const TimelineWidget = (props: { const timeTitleBottom = timeTextTop * 2 + timeContentFontSize; const hoverMaskAlpha = 0.3; - const closestScreenshotItemOnXY = (x: number, _y: number) => { - // find out the screenshot that is closest to the mouse on the left - let closestScreenshot: TimelineItem | undefined; // already sorted + const closestScreenshotItemOnXY = (x: number) => { + let closestScreenshot: TimelineItem | undefined; let closestIndex = -1; for (let i = 0; i < allScreenshots.length; i++) { - const shot = allScreenshots[i]; - if (shot.x! <= x) { + if (allScreenshots[i].x! <= x) { closestScreenshot = allScreenshots[i]; closestIndex = i; } else { break; } } - return { - closestScreenshot, - closestIndex, - }; + return { closestScreenshot, closestIndex }; }; - useMemo(() => { - const { startMs, endMs } = props.highlightMask || {}; - const { startMs: hoverStartMs, endMs: hoverEndMs } = props.hoverMask || {}; - const fn = containerUpdaterRef.current; - fn(startMs, endMs, hoverStartMs, hoverEndMs); + // Update masks and trigger redraw + useEffect(() => { + stateRef.current.highlightMask = props.highlightMask; + stateRef.current.hoverMask = props.hoverMask; + redraw(); }, [ props.highlightMask?.startMs, props.highlightMask?.endMs, @@ -129,338 +114,261 @@ const TimelineWidget = (props: { props.hoverMask?.endMs, ]); - useEffect(() => { - let freeFn = () => {}; - let isMounted = true; + // Shared redraw ref so event handlers can call it + const redrawRef = useRef<() => void>(() => {}); + const redraw = () => redrawRef.current(); - (async () => { - if (!domRef.current) { - return; + useEffect(() => { + if (!domRef.current) return; + + const { clientWidth } = domRef.current; + const canvasWidth = clientWidth * sizeRatio; + const canvasHeight = BASE_HEIGHT * sizeRatio; + + // Grid calculations + let singleGridWidth = 100 * sizeRatio; + let gridCount = Math.floor(canvasWidth / singleGridWidth); + const stepCandidate = [ + 50, 100, 200, 300, 500, 1000, 2000, 3000, 5000, 6000, 8000, 9000, 10000, + 20000, 30000, 40000, 60000, 90000, 12000, 300000, + ]; + let timeStep = stepCandidate[0]; + for (let i = stepCandidate.length - 1; i >= 0; i--) { + if (gridCount * stepCandidate[i] >= maxTime) { + timeStep = stepCandidate[i]; } + } + const gridRatio = maxTime / (gridCount * timeStep); + if (gridRatio <= 0.8) { + singleGridWidth = Math.floor(singleGridWidth * (1 / gridRatio) * 0.9); + gridCount = Math.floor(canvasWidth / singleGridWidth); + } - // Create new PIXI application - const app = new PIXI.Application(); - - // width of domRef - const { clientWidth } = domRef.current; - const canvasWidth = clientWidth * sizeRatio; - const canvasHeight = BASE_HEIGHT * sizeRatio; - - let singleGridWidth = 100 * sizeRatio; - let gridCount = Math.floor(canvasWidth / singleGridWidth); - const stepCandidate = [ - 50, 100, 200, 300, 500, 1000, 2000, 3000, 5000, 6000, 8000, 9000, 10000, - 20000, 30000, 40000, 60000, 90000, 12000, 300000, - ]; - let timeStep = stepCandidate[0]; - for (let i = stepCandidate.length - 1; i >= 0; i--) { - if (gridCount * stepCandidate[i] >= maxTime) { - timeStep = stepCandidate[i]; - } - } - const gridRatio = maxTime / (gridCount * timeStep); - if (gridRatio <= 0.8) { - singleGridWidth = Math.floor(singleGridWidth * (1 / gridRatio) * 0.9); - gridCount = Math.floor(canvasWidth / singleGridWidth); - } + const leftForTimeOffset = (t: number) => + Math.floor((singleGridWidth * t) / timeStep); + const timeOffsetForLeft = (l: number) => + Math.floor((l * timeStep) / singleGridWidth); + + // Create canvas + const canvas = document.createElement('canvas'); + canvas.width = canvasWidth; + canvas.height = canvasHeight; + canvasRef.current = canvas; + domRef.current.replaceChildren(canvas); + const ctx = canvas.getContext('2d')!; + + const screenshotTop = timeTitleBottom + commonPadding * 1.5; + const screenshotMaxHeight = + canvasHeight - screenshotTop - commonPadding * 1.5; + + const formatTime = (num: number) => { + const s = num / 1000; + return s % 1 === 0 ? `${s}s` : `${s.toFixed(1)}s`; + }; - const leftForTimeOffset = (timeOffset: number) => { - return Math.floor((singleGridWidth * timeOffset) / timeStep); - }; - const timeOffsetForLeft = (left: number) => { - return Math.floor((left * timeStep) / singleGridWidth); - }; + // Load all screenshot images + const { imgCache } = stateRef.current; + let isMounted = true; - await app.init({ - width: canvasWidth, - height: canvasHeight, - backgroundColor: sideBg, - }); + const loadAllImages = async () => { + await Promise.all( + allScreenshots.map(async (shot, index) => { + if (imgCache.has(shot.img)) { + // Compute layout from cached image + const img = imgCache.get(shot.img)!; + const w = Math.floor( + (screenshotMaxHeight / img.naturalHeight) * img.naturalWidth, + ); + allScreenshots[index].x = leftForTimeOffset(shot.timeOffset); + allScreenshots[index].y = screenshotTop; + allScreenshots[index].width = w; + allScreenshots[index].height = screenshotMaxHeight; + return; + } + try { + const img = await loadImage(shot.img); + if (!isMounted) return; + imgCache.set(shot.img, img); + const w = Math.floor( + (screenshotMaxHeight / img.naturalHeight) * img.naturalWidth, + ); + allScreenshots[index].x = leftForTimeOffset(shot.timeOffset); + allScreenshots[index].y = screenshotTop; + allScreenshots[index].width = w; + allScreenshots[index].height = screenshotMaxHeight; + } catch { + /* skip broken images */ + } + }), + ); + if (isMounted) redraw(); + }; - if (!isMounted) { - app.destroy(); - return; - } + // ── Draw function ── + const drawAll = () => { + ctx.clearRect(0, 0, canvasWidth, canvasHeight); - appRef.current = app; + // Background + ctx.fillStyle = hexToCSS(sideBg); + ctx.fillRect(0, 0, canvasWidth, canvasHeight); - freeFn = () => { - app.destroy(); - appRef.current = null; - }; - if (!domRef.current) { - app.destroy(); - return; - } - domRef.current.replaceChildren(app.canvas); - - const pixiTextForNumber = (num: number) => { - const seconds = num / 1000; - const textContent = - seconds % 1 === 0 ? `${seconds}s` : `${seconds.toFixed(1)}s`; - const text = new PIXI.Text(`${textContent}`, { - fontSize: timeContentFontSize, - fill: gridTextColor, - }); - return text; - }; + // Title bar background + ctx.fillStyle = hexToCSS(titleBg); + ctx.fillRect(0, 0, canvasWidth, timeTitleBottom); - // drawing vertical grids, texts, title bg - gridsContainer.removeChildren(); - const titleBgSection = new PIXI.Graphics(); - titleBgSection.beginFill(titleBg); - titleBgSection.drawRect(0, 0, canvasWidth, timeTitleBottom); - titleBgSection.endFill(); - gridsContainer.addChild(titleBgSection); - const titleBottomBorder = new PIXI.Graphics(); - titleBottomBorder.beginFill(gridLineColor); - titleBottomBorder.drawRect(0, timeTitleBottom, canvasWidth, sizeRatio); - titleBottomBorder.endFill(); - gridsContainer.addChild(titleBottomBorder); - - const gridHeight = canvasHeight; - for (let i = 1; i <= gridCount; i++) { - const gridLine = new PIXI.Graphics(); - const gridLineLeft = leftForTimeOffset(i * timeStep); - gridLine.beginFill(gridLineColor); - gridLine.drawRect(gridLineLeft, 0, sizeRatio, gridHeight); - gridLine.endFill(); - gridsContainer.addChild(gridLine); - - // mark text at the left of each line - const text = pixiTextForNumber(i * timeStep); // `${i * timeStep}ms`; - // measure text width - const textLeft = gridLineLeft - text.width - commonPadding; - - text.x = textLeft; - text.y = timeTextTop; - - gridsContainer.addChild(text); - } - app.stage.addChild(gridsContainer); + // Title bottom border + ctx.fillStyle = hexToCSS(gridLineColor); + ctx.fillRect(0, timeTitleBottom, canvasWidth, sizeRatio); - if (!allScreenshots.length) { - console.warn('No screenshots found'); - return; + // Grid lines + time labels + ctx.font = `${timeContentFontSize}px sans-serif`; + for (let i = 1; i <= gridCount; i++) { + const x = leftForTimeOffset(i * timeStep); + ctx.fillStyle = hexToCSS(gridLineColor); + ctx.fillRect(x, 0, sizeRatio, canvasHeight); + + const label = formatTime(i * timeStep); + const tw = ctx.measureText(label).width; + ctx.fillStyle = hexToCSS(gridTextColor); + ctx.fillText( + label, + x - tw - commonPadding, + timeTextTop + timeContentFontSize, + ); } - const shotContainers: PIXI.Container[] = []; - - // draw all screenshots - screenshotsContainer.removeChildren(); - const screenshotTop = timeTitleBottom + commonPadding * 1.5; - const screenshotMaxHeight = - canvasHeight - screenshotTop - commonPadding * 1.5; - allScreenshots.forEach((screenshot, index) => { - const container = new PIXI.Container(); - shotContainers.push(container); - app.stage.addChild(container); - Promise.resolve( - (async () => { - await loadTexture(screenshot.img); - const texture = getTextureFromCache(screenshot.img); - if (!texture) { - return; - } - - // clone the sprite - const screenshotSprite = PIXI.Sprite.from(texture); - - // get width / height of img - const originalWidth = screenshotSprite.width; - const originalHeight = screenshotSprite.height; - - const screenshotHeight = screenshotMaxHeight; - const screenshotWidth = Math.floor( - (screenshotHeight / originalHeight) * originalWidth, - ); - - const screenshotX = leftForTimeOffset(screenshot.timeOffset); - allScreenshots[index].x = screenshotX; - allScreenshots[index].y = screenshotTop; - allScreenshots[index].width = screenshotWidth; - allScreenshots[index].height = screenshotMaxHeight; - - const border = new PIXI.Graphics(); - border.lineStyle(sizeRatio, shotBorderColor, 1); - border.drawRect( - screenshotX, - screenshotTop, - screenshotWidth, - screenshotMaxHeight, - ); - border.endFill(); - container.addChild(border); - - screenshotSprite.x = screenshotX; - screenshotSprite.y = screenshotTop; - screenshotSprite.width = screenshotWidth; - screenshotSprite.height = screenshotMaxHeight; - container.addChild(screenshotSprite); - })(), + // Screenshots + for (const shot of allScreenshots) { + const img = imgCache.get(shot.img); + if (!img || shot.x == null || shot.width == null) continue; + ctx.drawImage( + img, + shot.x, + screenshotTop, + shot.width, + screenshotMaxHeight, ); - }); + ctx.strokeStyle = hexToCSS(shotBorderColor); + ctx.lineWidth = sizeRatio; + ctx.strokeRect(shot.x, screenshotTop, shot.width, screenshotMaxHeight); + } - const highlightMaskUpdater = ( + // Highlight masks + const drawMask = ( start: number | undefined, end: number | undefined, - hoverStart: number | undefined, - hoverEnd: number | undefined, + alpha: number, ) => { - highlightMaskContainer.removeChildren(); - - const mask = ( - start: number | undefined, - end: number | undefined, - alpha: number, - ) => { - if ( - typeof start === 'undefined' || - typeof end === 'undefined' || - end === 0 - ) { - return; - } - const leftBorder = new PIXI.Graphics(); - leftBorder.beginFill(gridHighlightColor, 1); - leftBorder.drawRect( - leftForTimeOffset(start), - 0, - sizeRatio, - canvasHeight, - ); - leftBorder.endFill(); - highlightMaskContainer.addChild(leftBorder); - - const rightBorder = new PIXI.Graphics(); - rightBorder.beginFill(gridHighlightColor, 1); - rightBorder.drawRect( - leftForTimeOffset(end), - 0, - sizeRatio, - canvasHeight, - ); - rightBorder.endFill(); - highlightMaskContainer.addChild(rightBorder); - - const mask = new PIXI.Graphics(); - mask.beginFill(gridHighlightColor, alpha); - mask.drawRect( - leftForTimeOffset(start), - 0, - leftForTimeOffset(end) - leftForTimeOffset(start), - canvasHeight, - ); - mask.endFill(); - highlightMaskContainer.addChild(mask); - }; - - mask(start, end, highlightMaskAlpha); - mask(hoverStart, hoverEnd, hoverMaskAlpha); + if (start == null || end == null || end === 0) return; + const x1 = leftForTimeOffset(start); + const x2 = leftForTimeOffset(end); + ctx.globalAlpha = alpha; + ctx.fillStyle = hexToCSS(gridHighlightColor); + ctx.fillRect(x1, 0, x2 - x1, canvasHeight); + ctx.globalAlpha = 1; + ctx.fillRect(x1, 0, sizeRatio, canvasHeight); + ctx.fillRect(x2, 0, sizeRatio, canvasHeight); }; - highlightMaskUpdater( - props.highlightMask?.startMs, - props.highlightMask?.endMs, - 0, - 0, + + const { highlightMask, hoverMask } = stateRef.current; + drawMask( + highlightMask?.startMs, + highlightMask?.endMs, + highlightMaskAlpha, ); - containerUpdaterRef.current = highlightMaskUpdater; - - // keep tracking the position of the mouse moving above the canvas - app.stage.interactive = true; - const onPointerMove = (event: PointerEvent) => { - const x = event.offsetX * sizeRatio; - const y = event.offsetY * sizeRatio; - indicatorContainer.removeChildren(); - - // find out the screenshot that is closest to the mouse on the left - const { closestScreenshot, closestIndex } = closestScreenshotItemOnXY( - x, - y, - ); - if (closestIndex < 0) { - props.onUnhighlight?.(); - return; - } - const closestContainer = shotContainers[closestIndex]; - - // highlight the items in closestContainer - closestContainer.children.forEach((child) => { - if (child instanceof PIXI.Sprite) { - // border - const newSpirit = new PIXI.Graphics(); - newSpirit.lineStyle(2, gridHighlightColor, 1); - newSpirit.drawRect( - x, // follow mouse - closestScreenshot?.y!, - closestScreenshot?.width!, - closestScreenshot?.height!, - ); - newSpirit.endFill(); - indicatorContainer.addChild(newSpirit); + drawMask(hoverMask?.startMs, hoverMask?.endMs, hoverMaskAlpha); + + // Hover indicator + const hoverX = stateRef.current.hoverX; + if (hoverX != null) { + const { closestScreenshot } = closestScreenshotItemOnXY(hoverX); - const screenshotSpirit = cloneSprite(child); - screenshotSpirit.x = x; - indicatorContainer.addChild(screenshotSpirit); + // Cursor line + ctx.fillStyle = hexToCSS(gridHighlightColor); + ctx.fillRect(hoverX - 1, 0, 3, canvasHeight); + + // Hover screenshot clone + if (closestScreenshot) { + const img = imgCache.get(closestScreenshot.img); + if (img && closestScreenshot.width && closestScreenshot.height) { + ctx.drawImage( + img, + hoverX, + closestScreenshot.y!, + closestScreenshot.width, + closestScreenshot.height, + ); + ctx.strokeStyle = hexToCSS(gridHighlightColor); + ctx.lineWidth = 2; + ctx.strokeRect( + hoverX, + closestScreenshot.y!, + closestScreenshot.width, + closestScreenshot.height, + ); } - }); + } - // cursor line - const indicator = new PIXI.Graphics(); - indicator.beginFill(gridHighlightColor, 1); - indicator.drawRect(x - 1, 0, 3, canvasHeight); - indicator.endFill(); - indicatorContainer.addChild(indicator); - - // time string - const text = pixiTextForNumber(timeOffsetForLeft(x)); - text.x = x + 5; - text.y = timeTextTop; - const textBg = new PIXI.Graphics(); - textBg.beginFill(titleBg, 1); - textBg.drawRect(text.x, text.y, text.width + 10, text.height); - textBg.endFill(); - - indicatorContainer.addChild(textBg); - indicatorContainer.addChild(text); + // Time label at cursor + const label = formatTime(timeOffsetForLeft(hoverX)); + const tw = ctx.measureText(label).width; + ctx.fillStyle = hexToCSS(titleBg); + ctx.fillRect(hoverX + 5, timeTextTop, tw + 10, timeContentFontSize + 4); + ctx.fillStyle = hexToCSS(gridTextColor); + ctx.fillText(label, hoverX + 5, timeTextTop + timeContentFontSize); + } + }; + + redrawRef.current = drawAll; + + // Event handlers + const onPointerMove = (e: PointerEvent) => { + const x = e.offsetX * sizeRatio; + const y = e.offsetY * sizeRatio; + stateRef.current.hoverX = x; + drawAll(); + const { closestScreenshot } = closestScreenshotItemOnXY(x); + if (closestScreenshot) { props.onHighlight?.({ mouseX: x / sizeRatio, mouseY: y / sizeRatio, - item: closestScreenshot!, + item: closestScreenshot, }); - }; - // app.stage.on('pointermove', onPointerMove); - // on pointer move out - const onPointerOut = () => { - indicatorContainer.removeChildren(); + } else { props.onUnhighlight?.(); - }; + } + }; - const onPointerTap = (event: PointerEvent) => { - const x = event.offsetX * sizeRatio; - const y = event.offsetY * sizeRatio; - const { closestScreenshot } = closestScreenshotItemOnXY(x, y); - if (closestScreenshot) { - props.onTap?.(closestScreenshot); - } - }; + const onPointerOut = () => { + stateRef.current.hoverX = null; + drawAll(); + props.onUnhighlight?.(); + }; + + const onPointerDown = (e: PointerEvent) => { + const x = e.offsetX * sizeRatio; + const { closestScreenshot } = closestScreenshotItemOnXY(x); + if (closestScreenshot) { + props.onTap?.(closestScreenshot); + } + }; - app.stage.addChild(screenshotsContainer); - app.stage.addChild(highlightMaskContainer); - app.stage.addChild(indicatorContainer); + canvas.addEventListener('pointermove', onPointerMove); + canvas.addEventListener('pointerout', onPointerOut); + canvas.addEventListener('pointerdown', onPointerDown); - const canvas = app.view; - canvas.addEventListener('pointermove', onPointerMove); - canvas.addEventListener('pointerout', onPointerOut); - canvas.addEventListener('pointerdown', onPointerTap); - })(); + // Initial draw + load images + drawAll(); + loadAllImages(); return () => { isMounted = false; - freeFn(); + canvas.removeEventListener('pointermove', onPointerMove); + canvas.removeEventListener('pointerout', onPointerOut); + canvas.removeEventListener('pointerdown', onPointerDown); + redrawRef.current = () => {}; }; - // eslint-disable-next-line react-hooks/exhaustive-deps }, [ isDarkMode, titleBg, @@ -485,13 +393,11 @@ const Timeline = () => { (store) => store.setHoverPreviewConfig, ); - // should be first task time ? let startingTime = -1; let idCount = 1; const idTaskMap: Record = {}; const allScreenshots: TimelineItem[] = allTasks .reduce<(ExecutionRecorderItem & { id: string })[]>((acc, current) => { - // Extract uiContext screenshot FIRST (before recorder processing) const uiContextRecorderItem: (ExecutionRecorderItem & { id: string })[] = []; const screenshotFromContext = current.uiContext?.screenshot; @@ -507,7 +413,6 @@ const Timeline = () => { }); } - // Process recorder items (existing logic) const recorders = current.recorder || []; recorders.forEach((item) => { if (startingTime === -1 || startingTime > item.ts) { @@ -523,33 +428,22 @@ const Timeline = () => { const recorderItemWithId = recorders.map((item) => { const idStr = `id_${idCount++}`; idTaskMap[idStr] = current; - return { - ...item, - id: idStr, - }; + return { ...item, id: idStr }; }); - // Concatenate uiContext items BEFORE recorder items return acc.concat(uiContextRecorderItem, recorderItemWithId || []); }, []) - .filter((item) => { - return item.screenshot; - }) - .map((recorderItem) => { - const screenshotBase64 = recorderItem.screenshot?.base64 || ''; - return { - id: recorderItem.id, - img: screenshotBase64, - timeOffset: recorderItem.ts - startingTime, - }; - }) + .filter((item) => item.screenshot) + .map((recorderItem) => ({ + id: recorderItem.id, + img: recorderItem.screenshot?.base64 || '', + timeOffset: recorderItem.ts - startingTime, + })) .sort((a, b) => a.timeOffset - b.timeOffset); const itemOnTap = (item: TimelineItem) => { const task = idTaskMap[item.id]; - if (task) { - setActiveTask(task); - } + if (task) setActiveTask(task); }; const onHighlightItem = (param: HighlightParam) => { @@ -576,9 +470,7 @@ const Timeline = () => { const maskConfigForTask = ( task?: ExecutionTask | null, ): HighlightMask | undefined => { - if (!task) { - return undefined; - } + if (!task) return undefined; return task.timing?.start && task.timing?.end ? { startMs: task.timing.start - startingTime || 0, diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index f814f3c33a..b24570f977 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -409,6 +409,7 @@ export class Agent< groupDescription: this.opts.groupDescription, executions: [], modelBriefs: [], + deviceType: this.interface.interfaceType, }); this.executionDumpIndexByRunner = new WeakMap(); diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 2589768fc7..c514d04c0e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -705,6 +705,7 @@ export interface IGroupedActionDump { groupDescription?: string; modelBriefs: string[]; executions: IExecutionDump[]; + deviceType?: string; } /** @@ -716,6 +717,7 @@ export class GroupedActionDump implements IGroupedActionDump { groupDescription?: string; modelBriefs: string[]; executions: ExecutionDump[]; + deviceType?: string; constructor(data: IGroupedActionDump) { this.sdkVersion = data.sdkVersion; @@ -725,6 +727,7 @@ export class GroupedActionDump implements IGroupedActionDump { this.executions = data.executions.map((exec) => exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec), ); + this.deviceType = data.deviceType; } /** @@ -771,6 +774,7 @@ export class GroupedActionDump implements IGroupedActionDump { groupDescription: this.groupDescription, modelBriefs: this.modelBriefs, executions: this.executions.map((exec) => exec.toJSON()), + deviceType: this.deviceType, }; } diff --git a/packages/visualizer/package.json b/packages/visualizer/package.json index 080f34f21a..330b20d34a 100644 --- a/packages/visualizer/package.json +++ b/packages/visualizer/package.json @@ -25,7 +25,6 @@ "react-dom": ">=19.1.0" }, "devDependencies": { - "@pixi/unsafe-eval": "7.4.2", "@rsbuild/plugin-less": "^1.5.0", "@rsbuild/plugin-node-polyfill": "1.4.2", "@rsbuild/plugin-react": "^1.4.1", @@ -37,8 +36,6 @@ "@types/react-dom": "^18.3.1", "execa": "9.3.0", "http-server": "14.1.1", - "pixi-filters": "6.0.5", - "pixi.js": "8.1.1", "query-string": "9.1.1", "react": "18.3.1", "react-dom": "18.3.1", diff --git a/packages/visualizer/src/component/blackboard/index.less b/packages/visualizer/src/component/blackboard/index.less index 64dc1a660d..bb1a58c55b 100644 --- a/packages/visualizer/src/component/blackboard/index.less +++ b/packages/visualizer/src/component/blackboard/index.less @@ -19,24 +19,103 @@ .bottom-tip-item { max-width: 500px; - color: #AAA; + color: #aaa; text-overflow: ellipsis; word-wrap: break-word; } } -.blackboard-filter { - margin: 10px 0; +.blackboard-main-content { + position: relative; + overflow: hidden; } -.blackboard-main-content { - canvas { - width: 100%; - border: 1px solid @heavy-border-color; - box-sizing: border-box; +.blackboard-screenshot { + width: 100%; + display: block; + border: 1px solid @heavy-border-color; + box-sizing: border-box; +} + +.blackboard-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + pointer-events: none; +} + +// ── Rect overlays ── + +.blackboard-rect { + position: absolute; + box-sizing: border-box; + pointer-events: none; +} + +.blackboard-rect-label { + position: absolute; + bottom: 100%; + left: 0; + font-size: 14px; + font-weight: 600; + white-space: nowrap; + padding: 1px 4px; + line-height: 1.4; +} + +.blackboard-rect-search { + background: rgba(2, 131, 145, 0.4); + border: 1px solid #028391; + box-shadow: 4px 4px 2px rgba(51, 51, 51, 0.4); + + .blackboard-rect-label { + color: #028391; + } +} + +.blackboard-rect-highlight { + background: rgba(253, 89, 7, 0.4); + border: 1px solid #fd5907; + box-shadow: 4px 4px 2px rgba(51, 51, 51, 0.4); + animation: blackboard-pulse 1.2s ease-in-out infinite; + + .blackboard-rect-label { + color: #000; + } +} + +// ── Point overlays ── + +.blackboard-point { + position: absolute; + width: 20px; + height: 20px; + margin-left: -10px; + margin-top: -10px; + border-radius: 50%; + background: rgba(253, 89, 7, 0.4); + border: 1px solid #fd5907; + box-shadow: 0 0 8px rgba(253, 89, 7, 0.5); + pointer-events: none; + animation: blackboard-pulse 1.2s ease-in-out infinite; +} + +@keyframes blackboard-pulse { + 0%, + 100% { + opacity: 0.4; + box-shadow: 4px 4px 2px rgba(51, 51, 51, 0.4); + } + 50% { + opacity: 1; + box-shadow: 4px 4px 2px rgba(51, 51, 51, 0.4), + 0 0 16px rgba(253, 89, 7, 0.5); } } -// Dark mode styles + +// ── Dark mode ── + [data-theme='dark'] { .blackboard { .footer { @@ -48,7 +127,11 @@ } } - .blackboard-main-content canvas { + .blackboard-screenshot { border-color: rgba(255, 255, 255, 0.12); } + + .blackboard-rect-highlight .blackboard-rect-label { + color: #fff; + } } diff --git a/packages/visualizer/src/component/blackboard/index.tsx b/packages/visualizer/src/component/blackboard/index.tsx index e173ea6de3..4fe5b90cbe 100644 --- a/packages/visualizer/src/component/blackboard/index.tsx +++ b/packages/visualizer/src/component/blackboard/index.tsx @@ -1,79 +1,9 @@ 'use client'; -import 'pixi.js/unsafe-eval'; import type { BaseElement, Rect, UIContext } from '@midscene/core'; -import { Checkbox } from 'antd'; -import type { CheckboxProps } from 'antd'; -import * as PIXI from 'pixi.js'; -import { type ReactElement, useEffect, useMemo, useRef, useState } from 'react'; +import type { ReactElement } from 'react'; +import { useMemo, useRef } from 'react'; import { colorForName, highlightColorForType } from '../../utils/color'; import './index.less'; -import { treeToList } from '@midscene/shared/extractor'; -import { DropShadowFilter, GlowFilter } from 'pixi-filters'; -import { useGlobalPreference } from '../../store/store'; - -const itemFillAlpha = 0.4; -const highlightAlpha = 0.4; -const pointRadius = 10; - -export const pointMarkForItem = ( - point: [number, number], - type: 'highlightPoint', -) => { - const [x, y] = point; - const themeColor = highlightColorForType('element'); - - const graphics = new PIXI.Graphics(); - // draw a circle - graphics.beginFill(themeColor, itemFillAlpha); - graphics.drawCircle(x, y, pointRadius); - graphics.endFill(); - return graphics; -}; - -export const rectMarkForItem = ( - rect: Rect, - name: string, - type: 'element' | 'searchArea' | 'highlight', -) => { - const { left, top, width, height } = rect; - let themeColor: string; - if (type === 'element') { - themeColor = colorForName(name); - } else if (type === 'searchArea') { - themeColor = highlightColorForType('searchArea'); - } else { - themeColor = highlightColorForType('element'); - } - - const alpha = type === 'highlight' ? highlightAlpha : itemFillAlpha; - const graphics = new PIXI.Graphics(); - graphics.beginFill(themeColor, alpha); - graphics.lineStyle(1, themeColor, 1); - graphics.drawRect(left, top, width, height); - graphics.endFill(); - - const dropShadowFilter = new DropShadowFilter({ - blur: 2, - quality: 3, - alpha: 0.4, - offset: { x: 4, y: 4 }, - color: 0x333333, - }); - - graphics.filters = [dropShadowFilter]; - - const nameFontSize = 18; - if (!name) { - return [graphics]; - } - const texts = new PIXI.Text(name, { - fontSize: nameFontSize, - fill: 0x0, - }); - texts.x = left; - texts.y = Math.max(top - (nameFontSize + 4), 0); - return [graphics, texts]; -}; export const Blackboard = (props: { uiContext: UIContext | undefined | null; @@ -84,11 +14,9 @@ export const Blackboard = (props: { onCanvasClick?: (position: [number, number]) => void; }) => { const highlightElements: BaseElement[] = props.highlightElements || []; - const highlightIds = highlightElements.map((e) => e.id); const highlightRect = props.highlightRect; const highlightPoints = props.highlightPoints; - // Handle undefined/null uiContext if (!props.uiContext?.shotSize) { return (
@@ -101,9 +29,9 @@ export const Blackboard = (props: { const context = props.uiContext; const { shotSize, screenshot } = context; + const screenWidth = shotSize.width; + const screenHeight = shotSize.height; - // Extract base64 string from screenshot - // After restoreImageReferences(), screenshot is { base64: string } const screenshotBase64 = useMemo(() => { if (!screenshot) return ''; if (typeof screenshot === 'object' && 'base64' in screenshot) { @@ -113,310 +41,19 @@ export const Blackboard = (props: { return ''; }, [screenshot]); - const screenWidth = shotSize.width; - const screenHeight = shotSize.height; - - const domRef = useRef(null); // Should be HTMLDivElement not HTMLInputElement - const app = useMemo(() => new PIXI.Application(), []); - const [appInitialed, setAppInitialed] = useState(false); - - const highlightContainer = useMemo(() => new PIXI.Container(), []); - const elementMarkContainer = useMemo(() => new PIXI.Container(), []); - - const [hoverElement, setHoverElement] = useState(null); - - // key overlays - const pixiBgRef = useRef(undefined); - const animationFrameRef = useRef(null); - const highlightGraphicsRef = useRef([]); - const glowFiltersRef = useRef([]); - // const { - // backgroundVisible, - // setBackgroundVisible, - // elementsVisible, - // setElementsVisible, - // } = useGlobalPreference(); - const backgroundVisible = true; - const elementsVisible = true; - - useEffect(() => { - Promise.resolve( - (async () => { - if (!domRef.current || !screenWidth) { - return; - } - await app.init({ - width: screenWidth, - height: screenHeight, - background: 0xffffff, - }); - const canvasEl = domRef.current; - domRef.current.appendChild(app.canvas); // Ensure app.view is appended - const { clientWidth } = domRef.current.parentElement!; - const targetHeight = window.innerHeight * 0.6; - const viewportRatio = clientWidth / targetHeight; - if (screenWidth / screenHeight <= viewportRatio) { - const ratio = targetHeight / screenHeight; - canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`; - canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`; - } - - app.stage.addChild(highlightContainer); - app.stage.addChild(elementMarkContainer); - - setAppInitialed(true); - })(), - ); - - // Clean up the PIXI application when the component unmounts - return () => { - console.log('will destroy'); - // Stop animation - if (animationFrameRef.current !== null) { - cancelAnimationFrame(animationFrameRef.current); - animationFrameRef.current = null; - } - try { - app.destroy(true, { children: true, texture: true }); - } catch (e) { - console.warn('destroy failed', e); - } - }; - }, [app, screenWidth, screenHeight]); - - useEffect(() => { - if (!appInitialed) { - return; - } - - // Enable interaction on the stage and all its children - app.stage.eventMode = 'static'; - app.stage.hitArea = new PIXI.Rectangle(0, 0, screenWidth, screenHeight); - - const clickHandler = (event: PIXI.FederatedPointerEvent) => { - console.log('pixi click', event); - const { x, y } = event.data.global; - props.onCanvasClick?.([Math.round(x), Math.round(y)]); - }; - - app.stage.on('click', clickHandler); - - return () => { - app?.stage?.off('click'); - }; - }, [appInitialed, props.onCanvasClick, screenWidth, screenHeight]); - - // draw all texts on PIXI app - useEffect(() => { - if (!appInitialed) { - return; - } - - // draw the screenshot base64 - const img = new Image(); - img.onload = () => { - if (!app.stage) return; - const screenshotTexture = PIXI.Texture.from(img); - const backgroundSprite = new PIXI.Sprite(screenshotTexture); - backgroundSprite.x = 0; - backgroundSprite.y = 0; - backgroundSprite.width = screenWidth; - backgroundSprite.height = screenHeight; - - // Ensure the background doesn't block interactivity - backgroundSprite.eventMode = 'passive'; - - app.stage.addChildAt(backgroundSprite, 0); - pixiBgRef.current = backgroundSprite; - backgroundSprite.visible = backgroundVisible; - }; - img.onerror = (e) => { - console.error('load screenshot failed', e); - }; - - if (screenshotBase64) { - img.src = screenshotBase64; - } else { - console.error('screenshotBase64 is empty, cannot load image'); - } - }, [app.stage, appInitialed, screenWidth, screenHeight, screenshotBase64]); - - const { highlightElementRects } = useMemo(() => { - const highlightElementRects: Rect[] = []; - - highlightContainer.removeChildren(); - elementMarkContainer.removeChildren(); - - // Make containers interactive but not blocking events - highlightContainer.eventMode = 'passive'; - elementMarkContainer.eventMode = 'passive'; - - // Clear previous highlight graphics references - highlightGraphicsRef.current = []; - glowFiltersRef.current = []; - - if (highlightRect) { - const [graphics] = rectMarkForItem( - highlightRect, - 'Search Area', - 'searchArea', - ); - highlightContainer.addChild(graphics); - } - - if (highlightElements.length) { - highlightElements.forEach((element) => { - const { rect, content, id } = element; - const items = rectMarkForItem(rect, content, 'highlight'); - const graphics = items[0] as PIXI.Graphics; // First element is always Graphics - - // Add glow filter for prominent highlight effect - const glowFilter = new GlowFilter({ - distance: 30, - outerStrength: 3, - innerStrength: 0, - color: 0xfd5907, // Orange color - quality: 0.5, - }); - - // Add both drop shadow and glow filters - const existingFilters = graphics.filters; - if (Array.isArray(existingFilters)) { - graphics.filters = [...existingFilters, glowFilter]; - } else if (existingFilters) { - graphics.filters = [existingFilters, glowFilter]; - } else { - graphics.filters = [glowFilter]; - } - - items.forEach((item) => highlightContainer.addChild(item)); - // Store references for animation - highlightGraphicsRef.current.push(graphics); - glowFiltersRef.current.push(glowFilter); - }); - } - - if (highlightPoints?.length) { - highlightPoints.forEach((point) => { - const graphics = pointMarkForItem(point, 'highlightPoint'); - - // Add glow filter for points too - const glowFilter = new GlowFilter({ - distance: 25, - outerStrength: 2.5, - innerStrength: 0, - color: 0xfd5907, - quality: 0.5, - }); - - graphics.filters = [glowFilter]; - - highlightContainer.addChild(graphics); - // Store references for animation - highlightGraphicsRef.current.push(graphics); - glowFiltersRef.current.push(glowFilter); - }); - } - - // element rects - // const elements = []; - // elements.forEach((element) => { - // const { rect, content, id } = element; - // const ifHighlight = highlightIds.includes(id) || hoverElement?.id === id; - - // if (ifHighlight) { - // return; - // } + const containerRef = useRef(null); - // const [graphics] = rectMarkForItem(rect, content, 'element'); - // elementMarkContainer.addChild(graphics); - // }); + const handleClick = (e: React.MouseEvent) => { + if (!props.onCanvasClick || !containerRef.current) return; + const rect = containerRef.current.getBoundingClientRect(); + const scaleX = screenWidth / rect.width; + const scaleY = screenHeight / rect.height; + const x = Math.round((e.clientX - rect.left) * scaleX); + const y = Math.round((e.clientY - rect.top) * scaleY); + props.onCanvasClick([x, y]); + }; - elementMarkContainer.visible = elementsVisible; - return { - highlightElementRects, - }; - }, [ - app, - appInitialed, - highlightElements, - hoverElement, - highlightRect, - highlightPoints, - // bgVisible, - // elementsVisible, - ]); - - // Pulsing animation for highlight elements - useEffect(() => { - if (!appInitialed || highlightGraphicsRef.current.length === 0) { - console.log('Animation skipped:', { - appInitialed, - graphicsCount: highlightGraphicsRef.current.length, - }); - return; - } - - console.log( - 'Starting pulsing animation for', - highlightGraphicsRef.current.length, - 'graphics', - ); - const graphicsToAnimate = highlightGraphicsRef.current; - const glowFilters = glowFiltersRef.current; - const pulseDuration = 1200; // 1.2 seconds for smooth pulsing - const minAlpha = 0.4; - const maxAlpha = 1.0; - const minGlowStrength = 2.0; - const maxGlowStrength = 5.0; - const startTime = performance.now(); - - const animate = () => { - const elapsed = performance.now() - startTime; - const progress = (elapsed % pulseDuration) / pulseDuration; - - // Use sine wave for smooth pulsing effect - const sineValue = Math.sin(progress * Math.PI * 2); - const normalizedSine = (sineValue + 1) / 2; // 0 to 1 - - const alpha = minAlpha + normalizedSine * (maxAlpha - minAlpha); - const glowStrength = - minGlowStrength + normalizedSine * (maxGlowStrength - minGlowStrength); - - graphicsToAnimate.forEach((graphics, index) => { - graphics.alpha = alpha; - - // Animate glow strength - if (glowFilters[index]) { - glowFilters[index].outerStrength = glowStrength; - } - }); - - animationFrameRef.current = requestAnimationFrame(animate); - }; - - animate(); - - return () => { - console.log('Stopping pulsing animation'); - if (animationFrameRef.current !== null) { - cancelAnimationFrame(animationFrameRef.current); - animationFrameRef.current = null; - } - }; - }, [appInitialed, highlightElements, highlightPoints]); - - // const onSetBackgroundVisible: CheckboxProps['onChange'] = (e) => { - // setBackgroundVisible(e.target.checked); - // if (pixiBgRef.current) { - // pixiBgRef.current.visible = e.target.checked; - // } - // }; - - // const onSetElementsVisible: CheckboxProps['onChange'] = (e) => { - // setElementsVisible(e.target.checked); - // elementMarkContainer.visible = e.target.checked; - // }; + const highlightElementRects: Rect[] = highlightElements.map((e) => e.rect); let bottomTipA: ReactElement | null = null; if (highlightElementRects.length === 1) { @@ -441,33 +78,81 @@ export const Blackboard = (props: {
- {/*
-
- - Background - - - Elements - + {screenshotBase64 && ( + screenshot + )} + + {/* Overlay container — scaled to match image coordinates */} +
+ {/* Search area */} + {highlightRect && ( +
+ Search Area +
+ )} + + {/* Highlight elements */} + {highlightElements.map((el, idx) => ( +
+ {el.content && ( + {el.content} + )} +
+ ))} + + {/* Highlight points */} + {highlightPoints?.map((point, idx) => ( +
+ ))}
-
*/} +
+
{bottomTipA}
- - {/* {footer} */}
); }; diff --git a/packages/visualizer/src/component/player/index.less b/packages/visualizer/src/component/player/index.less index 895c38049d..9afa14b257 100644 --- a/packages/visualizer/src/component/player/index.less +++ b/packages/visualizer/src/component/player/index.less @@ -15,11 +15,10 @@ max-width: 100%; max-height: 100%; padding: 12px; - background: #f2f4f7; box-sizing: border-box; border: 1px solid #f2f4f7; border-radius: @radius; - line-height: 100%; + line-height: normal; margin: 0 auto; display: flex; flex-direction: column; @@ -27,186 +26,217 @@ min-height: 300px; position: relative; - // height adaptive mode: swap background colors + // height adaptive mode &[data-fit-mode='height'] { background: #fff; .canvas-container { - background-color: #f2f4f7; + background-color: #000; } } .canvas-container { - flex: 0 0 auto; // let the container adjust height according to content, not force to fill + flex: 1 1 auto; min-height: 200px; - width: 100%; // ensure to fill width + width: 100%; display: flex; justify-content: center; align-items: center; overflow: hidden; position: relative; - background-color: #fff; - border-top-right-radius: 4px; - border-top-left-radius: 4px; - - // default to width adaptive mode - aspect-ratio: var(--canvas-aspect-ratio, 16/9); // use dynamic ratio + background-color: #000; + border-radius: 4px; - canvas { - width: 100%; // fill container width - height: auto; // height adaptive + .player-wrapper { + position: relative; max-width: 100%; max-height: 100%; - box-sizing: border-box; - display: block; - margin: 0 auto; - object-fit: contain; - border: none; - } + width: 100%; - // height adaptive mode - &[data-fit-mode='height'] { - flex: 1 1 auto; // allow container to stretch to fill available space - aspect-ratio: unset; // cancel fixed aspect ratio - min-height: 0; // allow container to shrink - height: auto; // let height be controlled by flex - - canvas { - height: 100%; // fill container height - width: auto; // width adaptive - max-width: 100%; - max-height: 100%; + &[data-portrait] { + width: auto; + height: 100%; } } + } - // width adaptive mode (current default behavior) - &[data-fit-mode='width'] { - aspect-ratio: var(--canvas-aspect-ratio, 16/9); + .player-subtitle { + position: absolute; + bottom: 52px; + left: 50%; + transform: translateX(-50%); + display: inline-flex; + align-items: center; + gap: 8px; + height: 32px; + padding: 0 14px; + background: rgba(80, 80, 80, 0.75); + backdrop-filter: blur(8px); + border-radius: 8px; + z-index: 3; + max-width: calc(100% - 24px); + pointer-events: none; + + .player-subtitle-badge { + font-size: 13px; + font-weight: 700; + color: #fff; + background: rgba(163, 77, 255, 1); + height: 22px; + border-radius: 4px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0 6px; + flex-shrink: 0; + } - canvas { - width: 100%; - height: auto; - } + .player-subtitle-text { + font-size: 14px; + font-weight: 500; + color: #fff; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + min-width: 0; } } - .player-timeline-wrapper { - width: 100%; - height: @timeline-height; - flex: none; - margin-bottom: 2px; - position: relative; + .control-bar { + position: absolute; + bottom: 0; + left: 0; + right: 0; + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + background: linear-gradient(transparent, rgba(0, 0, 0, 0.6)); + transition: opacity 0.3s; + z-index: 2; + + &.hidden { + opacity: 0; + pointer-events: none; + } } - .player-timeline { - width: 100%; - height: @timeline-height; - background: @player-control-bg; - position: relative; + .time-display { + color: rgba(255, 255, 255, 0.8); + font-size: 12px; + font-variant-numeric: tabular-nums; + white-space: nowrap; flex-shrink: 0; + } + + .seek-bar-track { + flex: 1; + height: 5px; + background: rgba(255, 255, 255, 0.3); + border-radius: 2.5px; + position: relative; + cursor: pointer; + touch-action: none; - .player-timeline-progress { - transition-timing-function: linear; + .seek-bar-fill { position: absolute; top: 0; left: 0; - height: @timeline-height; - background: @primary-color; + height: 100%; + background: rgba(43, 131, 255, 1); + border-radius: 2.5px; + pointer-events: none; } - } - .player-tools-wrapper { - width: 100%; - height: @tools-height + @player-vertical-spacing * 2; - flex: none; - position: relative; - padding-top: @player-vertical-spacing; - padding-bottom: @player-vertical-spacing; - padding-left: 16px; - padding-right: 16px; - box-sizing: border-box; + .seek-bar-knob { + position: absolute; + top: 50%; + width: 8px; + height: 16px; + background: #fff; + border-radius: 10px; + transform: translate(-50%, -50%); + pointer-events: none; + } + + .chapter-marker { + position: absolute; + top: 0; + width: 2px; + height: 100%; + background: #fff; + transform: translateX(-50%); + pointer-events: auto; + cursor: pointer; + z-index: 1; + + &::before { + content: ''; + position: absolute; + top: -8px; + left: -8px; + right: -8px; + bottom: -8px; + } + } } - .player-tools { - width: 100%; - height: @tools-height; - max-width: 100%; - overflow: hidden; - color: #000; - font-size: 14px; - box-sizing: border-box; + .status-icon { + transition: 0.2s; + width: 24px; + height: 24px; + border-radius: 4px; display: flex; - flex-direction: row; - justify-content: space-between; + align-items: center; + justify-content: center; flex-shrink: 0; + color: #fff; + cursor: pointer; + opacity: 0.7; - .ant-spin { - color: #333; + svg { + color: #fff; + font-size: 14px; } - .player-control { - flex-grow: 1; - display: flex; - flex-direction: row; - align-items: center; - overflow: hidden; - } - - .status-icon { - transition: 0.2s; - width: 32px; - height: 32px; - border-radius: @radius; - display: flex; - align-items: center; - justify-content: center; - flex-shrink: 0; - margin-left: 10px; - - &:hover { - cursor: pointer; - background: #f0f0f0; - } + &:hover { + opacity: 1; + background: rgba(255, 255, 255, 0.15); } + } - .status-text { - flex-grow: 1; - flex-shrink: 1; - min-width: 0; - overflow: hidden; - position: relative; - height: 100%; - display: flex; - flex-direction: column; - justify-content: space-between; - width: 0; - } + .player-custom-controls { + display: flex; + flex-direction: row; + align-items: center; + gap: 8px; + margin-right: 8px; - .title { - font-weight: 600; + .ant-spin { + color: #fff; } + } +} - .title, - .subtitle { - text-overflow: ellipsis; - overflow: hidden; - white-space: nowrap; - width: 100%; - } +// Chapter marker tooltip +.chapter-tooltip { + .ant-tooltip-inner { + background: rgba(80, 80, 80, 0.85); + backdrop-filter: blur(8px); + border-radius: 16px; + padding: 6px 12px; + font-size: 12px; + max-width: 360px; + } - .player-tools-item { - height: 100%; - display: flex; - flex-direction: column; - justify-content: center; - } + .ant-tooltip-arrow::before { + background: rgba(80, 80, 80, 0.85); } } // Dark mode styles [data-theme='dark'] { .player-container { - background: #141414; border-color: #292929; &[data-fit-mode='height'] { @@ -214,30 +244,12 @@ border-color: #292929; .canvas-container { - background-color: #141414; + background-color: #000; } } .canvas-container { - background-color: #1f1f1f; - } - - .player-tools { - color: #f8fafd; - - .ant-spin { - color: #f8fafd; - } - - .status-icon { - svg { - color: #f8fafd; - } - - &:hover { - background: rgba(255, 255, 255, 0.08); - } - } + background-color: #000; } } } diff --git a/packages/visualizer/src/component/player/index.tsx b/packages/visualizer/src/component/player/index.tsx index 63a02fc350..f3358459e3 100644 --- a/packages/visualizer/src/component/player/index.tsx +++ b/packages/visualizer/src/component/player/index.tsx @@ -1,136 +1,27 @@ 'use client'; -import 'pixi.js/unsafe-eval'; -import * as PIXI from 'pixi.js'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import './index.less'; -import { mouseLoading, mousePointer } from '@/utils'; import { CaretRightOutlined, + CompressOutlined, DownloadOutlined, + ExpandOutlined, ExportOutlined, - LoadingOutlined, + FontSizeOutlined, + PauseOutlined, ThunderboltOutlined, } from '@ant-design/icons'; -import type { BaseElement, LocateResultElement, Rect } from '@midscene/core'; import { Dropdown, Spin, Switch, Tooltip, message } from 'antd'; import GlobalPerspectiveIcon from '../../icons/global-perspective.svg'; import PlayerSettingIcon from '../../icons/player-setting.svg'; import { type PlaybackSpeedType, useGlobalPreference } from '../../store/store'; -import { getTextureFromCache, loadTexture } from '../../utils/pixi-loader'; -import type { - AnimationScript, - CameraState, - TargetCameraState, -} from '../../utils/replay-scripts'; -import { rectMarkForItem } from '../blackboard'; - -const canvasPaddingLeft = 0; -const canvasPaddingTop = 0; - -const cubicBezier = ( - t: number, - p0: number, - p1: number, - p2: number, - p3: number, -): number => { - const t2 = 1 - t; - return ( - p0 * t2 * t2 * t2 + - 3 * p1 * t * t2 * t2 + - 3 * p2 * t * t * t2 + - p3 * t * t * t - ); -}; - -const cubicImage = (t: number): number => { - // return cubicBezier(t, 0, 0.69, 0.43, 1); - return linear(t); -}; - -const cubicInsightElement = (t: number): number => { - return cubicBezier(t, 0, 0.5, 0.5, 1); -}; - -const cubicMouse = (t: number): number => { - return linear(t); -}; - -const linear = (t: number): number => { - return t; -}; - -type FrameFn = (callback: (current: number) => void) => void; - -const ERROR_FRAME_CANCEL = 'frame cancel (this is an error on purpose)'; -const frameKit = (): { - frame: FrameFn; - cancel: () => void; - timeout: (callback: () => void, ms: number) => void; - sleep: (ms: number) => Promise; - isCancelled: () => boolean; -} => { - let cancelFlag = false; - const pendingTimeouts: number[] = []; - - return { - frame: (callback: (current: number) => void) => { - if (cancelFlag) { - throw new Error(ERROR_FRAME_CANCEL); - } - requestAnimationFrame(() => { - if (cancelFlag) { - // Don't throw in requestAnimationFrame callback - just return silently - return; - } - callback(performance.now()); - }); - }, - timeout: (callback: () => void, ms: number) => { - if (cancelFlag) { - throw new Error(ERROR_FRAME_CANCEL); - } - const timeoutId = window.setTimeout(() => { - if (cancelFlag) { - // Don't throw in setTimeout callback - just return silently - return; - } - callback(); - }, ms); - pendingTimeouts.push(timeoutId); - }, - sleep: (ms: number): Promise => { - if (cancelFlag) { - return Promise.reject(new Error(ERROR_FRAME_CANCEL)); - } - return new Promise((resolve, reject) => { - const timeoutId = window.setTimeout(() => { - if (cancelFlag) { - reject(new Error(ERROR_FRAME_CANCEL)); - } else { - resolve(); - } - }, ms); - pendingTimeouts.push(timeoutId); - }); - }, - isCancelled: () => cancelFlag, - cancel: () => { - cancelFlag = true; - // Clear all pending timeouts to stop animations immediately - for (const id of pendingTimeouts) { - clearTimeout(id); - } - pendingTimeouts.length = 0; - }, - }; -}; - -const singleElementFadeInDuration = 80; -const LAYER_ORDER_IMG = 0; -const LAYER_ORDER_INSIGHT = 1; -const LAYER_ORDER_POINTER = 2; -const LAYER_ORDER_SPINNING_POINTER = 3; +import type { AnimationScript } from '../../utils/replay-scripts'; +import { StepsTimeline } from './remotion/StepScene'; +import { deriveFrameState } from './remotion/derive-frame-state'; +import { exportBrandedVideo } from './remotion/export-branded-video'; +import { calculateFrameMap } from './remotion/frame-calculator'; +import type { FrameMap, ScriptFrame } from './remotion/frame-calculator'; +import { useFramePlayer } from './use-frame-player'; const downloadReport = (content: string): void => { const blob = new Blob([content], { type: 'text/html' }); @@ -139,80 +30,32 @@ const downloadReport = (content: string): void => { a.href = url; a.download = 'midscene_report.html'; a.click(); + setTimeout(() => URL.revokeObjectURL(url), 0); }; -class RecordingSession { - canvas: HTMLCanvasElement; - mediaRecorder: MediaRecorder | null = null; - chunks: BlobPart[]; - recording = false; - - constructor(canvas: HTMLCanvasElement) { - this.canvas = canvas; - this.chunks = []; - } - - start() { - const stream = this.canvas.captureStream(60); // 60fps - const mediaRecorder = new MediaRecorder(stream, { - mimeType: 'video/webm', - }); - - mediaRecorder.ondataavailable = (event) => { - if (event.data.size > 0) { - this.chunks.push(event.data); +function deriveTaskId( + scriptFrames: ScriptFrame[], + stepsFrame: number, +): string | null { + let taskId: string | null = null; + for (const sf of scriptFrames) { + if (sf.durationInFrames === 0) { + if (sf.startFrame <= stepsFrame) { + taskId = sf.taskId ?? taskId; } - }; - - // Add error handler for MediaRecorder - mediaRecorder.onerror = (event) => { - console.error('MediaRecorder error:', event); - message.error('Video recording failed. Please try again.'); - this.recording = false; - this.mediaRecorder = null; - }; - - this.mediaRecorder = mediaRecorder; - this.recording = true; - return this.mediaRecorder.start(); - } - - stop() { - if (!this.recording || !this.mediaRecorder) { - console.warn('not recording'); - return; + continue; } - - // Bind onstop handler BEFORE calling stop() to ensure it's attached in time - this.mediaRecorder.onstop = () => { - // Check if we have any data - if (this.chunks.length === 0) { - console.error('No video data captured'); - message.error('Video export failed: No data captured.'); - return; - } - - const blob = new Blob(this.chunks, { type: 'video/webm' }); - - // Check blob size - if (blob.size === 0) { - console.error('Video blob is empty'); - message.error('Video export failed: Empty file.'); - return; - } - - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = 'midscene_replay.webm'; - a.click(); - URL.revokeObjectURL(url); - }; - - this.mediaRecorder.stop(); - this.recording = false; - this.mediaRecorder = null; + if (stepsFrame < sf.startFrame) break; + taskId = sf.taskId ?? taskId; } + return taskId; +} + +function formatTime(frame: number, fps: number): string { + const totalSeconds = Math.floor(frame / fps); + const m = Math.floor(totalSeconds / 60); + const s = totalSeconds % 60; + return `${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`; } export function Player(props?: { @@ -221,17 +64,20 @@ export function Player(props?: { imageHeight?: number; reportFileContent?: string | null; key?: string | number; - fitMode?: 'width' | 'height'; // 'width': width adaptive, 'height': height adaptive, default to 'height' - autoZoom?: boolean; // enable auto zoom when playing, default to true - canDownloadReport?: boolean; // enable download report, default to true - onTaskChange?: (taskId: string | null) => void; // callback when task changes during playback + fitMode?: 'width' | 'height'; + autoZoom?: boolean; + canDownloadReport?: boolean; + onTaskChange?: (taskId: string | null) => void; }) { - const [titleText, setTitleText] = useState(''); - const [subTitleText, setSubTitleText] = useState(''); - const { autoZoom, setAutoZoom, playbackSpeed, setPlaybackSpeed } = - useGlobalPreference(); + const { + autoZoom, + setAutoZoom, + playbackSpeed, + setPlaybackSpeed, + subtitleEnabled, + setSubtitleEnabled, + } = useGlobalPreference(); - // Update state when prop changes useEffect(() => { if (props?.autoZoom !== undefined) { setAutoZoom(props.autoZoom); @@ -239,1021 +85,495 @@ export function Player(props?: { }, [props?.autoZoom, setAutoZoom]); const scripts = props?.replayScripts; - const imageWidth = props?.imageWidth || 1920; - const imageHeight = props?.imageHeight || 1080; - const fitMode = props?.fitMode || 'height'; // default to height adaptive - const currentImg = useRef(scripts?.[0]?.img || null); - - const divContainerRef = useRef(null); - const app = useMemo(() => new PIXI.Application(), []); + const frameMap = useMemo(() => { + if (!scripts || scripts.length === 0) return null; + return calculateFrameMap(scripts); + }, [scripts]); - const pointerSprite = useRef(null); - const spinningPointerSprite = useRef(null); + const wrapperRef = useRef(null); + const renderLayerRef = useRef(null); + const lastTaskIdRef = useRef(null); + const [containerSize, setContainerSize] = useState({ width: 0, height: 0 }); - const [replayMark, setReplayMark] = useState(0); - const triggerReplay = () => { - setReplayMark(Date.now()); - }; - - const windowContentContainer = useMemo(() => { - const container = new PIXI.Container(); - return container; - }, []); - const insightMarkContainer = useMemo(() => { - const container = new PIXI.Container(); - container.zIndex = LAYER_ORDER_INSIGHT; - return container; + // Observe render layer size to compute scale factor + useEffect(() => { + const el = renderLayerRef.current; + if (!el) return; + const ro = new ResizeObserver((entries) => { + for (const entry of entries) { + const { width, height } = entry.contentRect; + setContainerSize((prev) => + prev.width === width && prev.height === height + ? prev + : { width, height }, + ); + } + }); + ro.observe(el); + return () => ro.disconnect(); }, []); - const basicCameraState = { - left: 0, - top: 0, - width: imageWidth, - pointerLeft: Math.round(imageWidth / 2), - pointerTop: Math.round(imageHeight / 2), - }; - - // -1: not started, 0: running, 1: finished - const [animationProgress, setAnimationProgress] = useState(-1); - const cancelFlag = useRef(false); - const appInitialized = useRef(false); + const player = useFramePlayer({ + durationInFrames: Math.max(frameMap?.totalDurationInFrames ?? 1, 1), + fps: frameMap?.fps ?? 30, + autoPlay: true, + loop: false, + playbackRate: playbackSpeed, + }); + // Track frame for taskId callback useEffect(() => { - cancelFlag.current = false; - return () => { - cancelFlag.current = true; - }; + if (!frameMap || !props?.onTaskChange) return; + const taskId = deriveTaskId(frameMap.scriptFrames, player.currentFrame); + if (taskId !== lastTaskIdRef.current) { + lastTaskIdRef.current = taskId; + props.onTaskChange(taskId); + } + }, [frameMap, props?.onTaskChange, player.currentFrame]); + + // Derive subtitle from current frame + const subtitle = useMemo(() => { + if (!frameMap) return null; + const state = deriveFrameState( + frameMap.scriptFrames, + player.currentFrame, + frameMap.imageWidth, + frameMap.imageHeight, + frameMap.fps, + ); + if (!state.title && !state.subTitle) return null; + return { title: state.title, subTitle: state.subTitle }; + }, [frameMap, player.currentFrame]); + + // Controls auto-hide + const [controlsVisible, setControlsVisible] = useState(true); + const hideTimerRef = useRef | null>(null); + + const showControls = useCallback(() => { + setControlsVisible(true); + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + hideTimerRef.current = setTimeout(() => setControlsVisible(false), 3000); }, []); - const cameraState = useRef({ ...basicCameraState }); + const onMouseEnter = useCallback(() => { + setControlsVisible(true); + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + }, []); - const resizeCanvasIfNeeded = async ( - newWidth: number, - newHeight: number, - ): Promise => { - // Guard: check if app is initialized before accessing app.screen - if (!appInitialized.current || !app.screen) { - return; - } - if (app.screen.width !== newWidth || app.screen.height !== newHeight) { - // Update renderer size - app.renderer.resize(newWidth, newHeight); + const onMouseLeave = useCallback(() => { + if (hideTimerRef.current) clearTimeout(hideTimerRef.current); + hideTimerRef.current = setTimeout(() => setControlsVisible(false), 1000); + }, []); - // Update container aspect ratio - if (divContainerRef.current) { - const aspectRatio = newWidth / newHeight; - divContainerRef.current.style.setProperty( - '--canvas-aspect-ratio', - aspectRatio.toString(), + // Seek bar drag + const seekBarRef = useRef(null); + const handleSeekPointerDown = useCallback( + (e: React.PointerEvent) => { + if (!frameMap || !seekBarRef.current) return; + const bar = seekBarRef.current; + bar.setPointerCapture(e.pointerId); + + const seek = (clientX: number) => { + const rect = bar.getBoundingClientRect(); + const ratio = Math.max( + 0, + Math.min(1, (clientX - rect.left) / rect.width), ); - } - - // Update basic camera state for new dimensions - const newBasicCameraState = { - left: 0, - top: 0, - width: newWidth, - pointerLeft: Math.round(newWidth / 2), - pointerTop: Math.round(newHeight / 2), + player.seekTo(Math.round(ratio * (frameMap.totalDurationInFrames - 1))); }; - cameraState.current = newBasicCameraState; - } - }; - - const repaintImage = async ( - scriptWidth?: number, - scriptHeight?: number, - ): Promise => { - const imgToUpdate = currentImg.current; - if (!imgToUpdate) { - console.warn('no image to update'); - return; - } - - // Use script-specific dimensions if provided, otherwise default to original - const targetWidth = scriptWidth || imageWidth; - const targetHeight = scriptHeight || imageHeight; - - // Resize canvas if dimensions changed - await resizeCanvasIfNeeded(targetWidth, targetHeight); - - if (!getTextureFromCache(imgToUpdate)) { - console.warn('image not loaded', imgToUpdate); - await loadTexture(imgToUpdate!); - } - const texture = getTextureFromCache(imgToUpdate); - if (!texture) { - throw new Error('texture not found'); - } - const sprite = PIXI.Sprite.from(texture); - if (!sprite) { - throw new Error('sprite not found'); - } - - const mainImgLabel = 'main-img'; - const child = windowContentContainer.getChildByLabel(mainImgLabel); - if (child) { - windowContentContainer.removeChild(child); - } - sprite.label = mainImgLabel; - sprite.zIndex = LAYER_ORDER_IMG; - - // use current canvas size, keep image quality - sprite.width = targetWidth; - sprite.height = targetHeight; - windowContentContainer.addChild(sprite); - }; + seek(e.clientX); - const spinningPointer = (frame: FrameFn): (() => void) => { - if (!spinningPointerSprite.current) { - spinningPointerSprite.current = PIXI.Sprite.from(mouseLoading); - spinningPointerSprite.current.zIndex = LAYER_ORDER_SPINNING_POINTER; - spinningPointerSprite.current.anchor.set(0.5, 0.5); - spinningPointerSprite.current.scale.set(0.5); - spinningPointerSprite.current.label = 'spinning-pointer'; - } - - spinningPointerSprite.current.x = pointerSprite.current?.x || 0; - spinningPointerSprite.current.y = pointerSprite.current?.y || 0; - windowContentContainer.addChild(spinningPointerSprite.current); - - let startTime: number; - let isCancelled = false; - - const animate = (currentTime: number) => { - if (isCancelled) return; - if (!startTime) startTime = currentTime; - const elapsedTime = currentTime - startTime; - - // Non-linear timing function (ease-in-out) - const progress = (Math.sin(elapsedTime / 500 - Math.PI / 2) + 1) / 2; - - const rotation = progress * Math.PI * 2; - - if (spinningPointerSprite.current) { - spinningPointerSprite.current.rotation = rotation; - } - - frame(animate); - }; - - frame(animate); - - const stopFn = () => { - if (spinningPointerSprite.current) { - windowContentContainer.removeChild(spinningPointerSprite.current); - } - isCancelled = true; - }; - - return stopFn; - }; + const onMove = (ev: PointerEvent) => seek(ev.clientX); + const onUp = () => { + bar.removeEventListener('pointermove', onMove); + bar.removeEventListener('pointerup', onUp); + }; + bar.addEventListener('pointermove', onMove); + bar.addEventListener('pointerup', onUp); + }, + [frameMap, player], + ); - const updatePointer = async ( - img: string, - x?: number, - y?: number, - ): Promise => { - if (!getTextureFromCache(img)) { - console.warn('image not loaded', img); - await loadTexture(img); + // Fullscreen + const [isFullscreen, setIsFullscreen] = useState(false); + const toggleFullscreen = useCallback(() => { + const el = wrapperRef.current; + if (!el) return; + if (!document.fullscreenElement) { + el.requestFullscreen().then(() => setIsFullscreen(true)); + } else { + document.exitFullscreen().then(() => setIsFullscreen(false)); } - const texture = getTextureFromCache(img); - if (!texture) { - throw new Error('texture not found'); - } - const sprite = PIXI.Sprite.from(texture); + }, []); - let targetX = pointerSprite.current?.x; - let targetY = pointerSprite.current?.y; - if (typeof x === 'number') { - targetX = x; - } - if (typeof y === 'number') { - targetY = y; - } - if (typeof targetX === 'undefined' || typeof targetY === 'undefined') { - console.warn('invalid pointer position', x, y); - return; - } + useEffect(() => { + const handler = () => setIsFullscreen(!!document.fullscreenElement); + document.addEventListener('fullscreenchange', handler); + return () => document.removeEventListener('fullscreenchange', handler); + }, []); - if (pointerSprite.current) { - const pointer = windowContentContainer.getChildByLabel('pointer'); - if (pointer) { - windowContentContainer.removeChild(pointer); - } + // Export video + const [isExporting, setIsExporting] = useState(false); + const [exportProgress, setExportProgress] = useState(0); + + const handleExportVideo = useCallback(async () => { + if (!frameMap || isExporting) return; + setIsExporting(true); + setExportProgress(0); + try { + await exportBrandedVideo(frameMap, (pct) => + setExportProgress(Math.round(pct * 100)), + ); + message.success('Video exported'); + } catch (e) { + console.error('Export failed:', e); + message.error('Export failed'); + } finally { + setIsExporting(false); + setExportProgress(0); } + }, [frameMap, isExporting]); - pointerSprite.current = sprite; - pointerSprite.current.x = targetX; - pointerSprite.current.y = targetY; - pointerSprite.current.label = 'pointer'; - pointerSprite.current.zIndex = LAYER_ORDER_POINTER; - windowContentContainer.addChild(pointerSprite.current); - }; - - const updateCamera = (state: CameraState, currentWidth?: number): void => { - cameraState.current = state; - - // Use current canvas width if provided, otherwise fall back to original imageWidth - const effectiveWidth = currentWidth || app.screen?.width || imageWidth; - - // If auto zoom is enabled, apply zoom - const newScale = autoZoom ? Math.max(1, effectiveWidth / state.width) : 1; - windowContentContainer.scale.set(newScale); - - // If auto zoom is enabled, pan the camera - windowContentContainer.x = autoZoom - ? Math.round(canvasPaddingLeft - state.left * newScale) - : canvasPaddingLeft; - windowContentContainer.y = autoZoom - ? Math.round(canvasPaddingTop - state.top * newScale) - : canvasPaddingTop; - - const pointer = windowContentContainer.getChildByLabel('pointer'); - if (pointer) { - pointer.scale.set(1 / newScale); + // Compute chapter markers + const chapterMarkers = useMemo(() => { + if (!frameMap) return []; + const { scriptFrames, totalDurationInFrames } = frameMap; + if (totalDurationInFrames === 0) return []; + const markers: { percent: number; title: string; frame: number }[] = []; + for (const sf of scriptFrames) { if ( - typeof state.pointerLeft === 'number' && - typeof state.pointerTop === 'number' - ) { - pointer.x = state.pointerLeft; - pointer.y = state.pointerTop; + (sf.type !== 'img' && sf.type !== 'insight') || + sf.durationInFrames === 0 + ) + continue; + const globalFrame = sf.startFrame; + const percent = (globalFrame / totalDurationInFrames) * 100; + if (percent > 1 && percent < 99) { + const parts = [sf.title, sf.subTitle].filter(Boolean); + markers.push({ + percent, + title: + parts.length > 0 + ? parts.join(': ') + : `Chapter ${markers.length + 1}`, + frame: globalFrame, + }); } } - }; - - const cameraAnimation = async ( - targetState: TargetCameraState, - duration: number, - frame: FrameFn, - ): Promise => { - // Get current canvas dimensions - const currentCanvasWidth = app.screen?.width || imageWidth; - const currentCanvasHeight = app.screen?.height || imageHeight; - - // If auto zoom is disabled, skip camera animation (only animate pointer) - if (!autoZoom) { - const currentState = { ...cameraState.current }; - const startPointerLeft = currentState.pointerLeft; - const startPointerTop = currentState.pointerTop; - const startTime = performance.now(); - - const shouldMovePointer = - typeof targetState.pointerLeft === 'number' && - typeof targetState.pointerTop === 'number' && - (targetState.pointerLeft !== startPointerLeft || - targetState.pointerTop !== startPointerTop); - - if (!shouldMovePointer) return; - - await new Promise((resolve) => { - const animate = (currentTime: number) => { - const elapsedTime = currentTime - startTime; - const rawProgress = Math.min(elapsedTime / duration, 1); - const progress = cubicMouse(rawProgress); - - const nextState: CameraState = { - ...currentState, - pointerLeft: - startPointerLeft + - (targetState.pointerLeft! - startPointerLeft) * progress, - pointerTop: - startPointerTop + - (targetState.pointerTop! - startPointerTop) * progress, - }; - - updateCamera(nextState, currentCanvasWidth); - - if (elapsedTime < duration) { - frame(animate); - } else { - resolve(); - } - }; - frame(animate); - }); - return; - } - - const currentState = { ...cameraState.current }; - const startLeft = currentState.left; - const startTop = currentState.top; - const startPointerLeft = currentState.pointerLeft; - const startPointerTop = currentState.pointerTop; - const startScale = currentState.width / currentCanvasWidth; - - const startTime = performance.now(); - const shouldMovePointer = - typeof targetState.pointerLeft === 'number' && - typeof targetState.pointerTop === 'number' && - (targetState.pointerLeft !== startPointerLeft || - targetState.pointerTop !== startPointerTop); - - // move pointer first, then move camera - const pointerMoveDuration = shouldMovePointer ? duration * 0.375 : 0; - const cameraMoveStart = pointerMoveDuration; - const cameraMoveDuration = duration - pointerMoveDuration; - - await new Promise((resolve) => { - const animate = (currentTime: number) => { - const nextState: CameraState = { ...cameraState.current }; - const elapsedTime = currentTime - startTime; - - // Mouse movement animation - if (shouldMovePointer) { - if (elapsedTime <= pointerMoveDuration) { - const rawMouseProgress = Math.min( - elapsedTime / pointerMoveDuration, - 1, - ); - const mouseProgress = cubicMouse(rawMouseProgress); - nextState.pointerLeft = - startPointerLeft + - (targetState.pointerLeft! - startPointerLeft) * mouseProgress; - nextState.pointerTop = - startPointerTop + - (targetState.pointerTop! - startPointerTop) * mouseProgress; - } else { - nextState.pointerLeft = targetState.pointerLeft!; - nextState.pointerTop = targetState.pointerTop!; - } - } - - // Camera movement animation (starts 500ms after mouse movement begins) - if (elapsedTime > cameraMoveStart) { - const cameraElapsedTime = elapsedTime - cameraMoveStart; - const rawCameraProgress = Math.min( - cameraElapsedTime / cameraMoveDuration, - 1, - ); - const cameraProgress = cubicImage(rawCameraProgress); - - // get the target scale - const targetScale = targetState.width / currentCanvasWidth; - const progressScale = - startScale + (targetScale - startScale) * cameraProgress; - const progressWidth = currentCanvasWidth * progressScale; - const progressHeight = currentCanvasHeight * progressScale; - nextState.width = progressWidth; - - const progressLeft = - startLeft + (targetState.left - startLeft) * cameraProgress; - const progressTop = - startTop + (targetState.top - startTop) * cameraProgress; - - const horizontalExceed = - progressLeft + progressWidth - currentCanvasWidth; - const verticalExceed = - progressTop + progressHeight - currentCanvasHeight; - - nextState.left = - horizontalExceed > 0 - ? progressLeft + horizontalExceed - : progressLeft; - nextState.top = - verticalExceed > 0 ? progressTop + verticalExceed : progressTop; - } - - updateCamera(nextState, currentCanvasWidth); - - if (elapsedTime < duration) { - frame(animate); - } else { - resolve(); - } - }; - - frame(animate); - }); - }; + return markers; + }, [frameMap]); - const fadeInGraphics = ( - graphics: PIXI.Container | PIXI.Graphics | PIXI.Text, - duration: number, - frame: FrameFn, - targetAlpha = 1, - ): Promise => { - return new Promise((resolve) => { - const startTime = performance.now(); - const animate = (currentTime: number) => { - const elapsedTime = currentTime - startTime; - const progress = Math.min(elapsedTime / duration, 1); - graphics.alpha = - targetAlpha === 0 ? 1 - linear(progress) : linear(progress); - if (elapsedTime < duration) { - frame(animate); - } else { - resolve(); - } - }; - - frame(animate); - }); - }; - - const fadeOutItem = async ( - graphics: PIXI.Container | PIXI.Graphics | PIXI.Text, - duration: number, - frame: FrameFn, - ): Promise => { - return fadeInGraphics(graphics, duration, frame, 0); - }; - - const insightElementsAnimation = async ( - elements: BaseElement[], - highlightElements: (BaseElement | LocateResultElement)[], - searchArea: Rect | undefined, - duration: number, - frame: FrameFn, - ): Promise => { - insightMarkContainer.removeChildren(); - - const elementsToAdd = [...elements]; - const totalLength = elementsToAdd.length; - let childrenCount = 0; - - await new Promise((resolve) => { - const startTime = performance.now(); - const animate = (currentTime: number) => { - const elapsedTime = currentTime - startTime; - const progress = cubicInsightElement( - Math.min(elapsedTime / duration, 1), - ); - - const elementsToAddNow = Math.floor(progress * totalLength); - - while (childrenCount < elementsToAddNow) { - const randomIndex = Math.floor(Math.random() * elementsToAdd.length); - const element = elementsToAdd.splice(randomIndex, 1)[0]; - if (element) { - const [insightMarkGraphic] = rectMarkForItem( - element.rect, - element.content, - 'element', - ); - insightMarkGraphic.alpha = 0; - insightMarkContainer.addChild(insightMarkGraphic); - childrenCount++; - fadeInGraphics( - insightMarkGraphic, - singleElementFadeInDuration, - frame, - ); - } - } - - if (elapsedTime < duration) { - frame(animate); - } else { - // Add all remaining items when time ends - while (elementsToAdd.length > 0) { - const randomIndex = Math.floor( - Math.random() * elementsToAdd.length, - ); - const element = elementsToAdd.splice(randomIndex, 1)[0]; - const [insightMarkGraphic] = rectMarkForItem( - element.rect, - element.content, - 'element', - ); - insightMarkGraphic.alpha = 1; // Set alpha to 1 immediately for remaining items - insightMarkContainer.addChild(insightMarkGraphic); - } - - if (searchArea) { - const [searchAreaGraphic] = rectMarkForItem( - searchArea, - 'Search Area', - 'searchArea', - ); - searchAreaGraphic.alpha = 1; - insightMarkContainer.addChild(searchAreaGraphic); - } - - highlightElements.map((element) => { - const [insightMarkGraphic] = rectMarkForItem( - element.rect, - (element as BaseElement).content || '', - 'highlight', - ); - insightMarkGraphic.alpha = 1; - insightMarkContainer.addChild(insightMarkGraphic); - }); - - resolve(); - } - }; - - frame(animate); - }); - }; - - const init = async (): Promise => { - if (!divContainerRef.current || !scripts) return; - - // use original image size for initialization - // this can keep the original image quality, then scale the canvas with CSS - await app.init({ - width: imageWidth, - height: imageHeight, - background: 0xf4f4f4, - autoDensity: true, - antialias: true, - }); - - // Mark app as initialized after app.init() completes - appInitialized.current = true; - - if (!divContainerRef.current) return; - divContainerRef.current.appendChild(app.canvas); - - windowContentContainer.x = 0; - windowContentContainer.y = 0; - app.stage.addChild(windowContentContainer); - - insightMarkContainer.x = 0; - insightMarkContainer.y = 0; - windowContentContainer.addChild(insightMarkContainer); - }; - - const [isRecording, setIsRecording] = useState(false); - const recorderSessionRef = useRef(null); - const cancelAnimationRef = useRef<(() => void) | null>(null); - // Playback session ID to prevent stale callbacks from cancelled playbacks - const playbackSessionIdRef = useRef(0); - - const handleExport = async () => { - if (recorderSessionRef.current) { - console.warn('recorderSession exists'); - return; - } - - if (!app.canvas) { - console.warn('canvas is not initialized'); - return; - } - - // Cancel any ongoing animation before starting export - if (cancelAnimationRef.current) { - cancelAnimationRef.current(); - cancelAnimationRef.current = null; - // Wait for animation cleanup to complete - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - recorderSessionRef.current = new RecordingSession(app.canvas); - setIsRecording(true); - triggerReplay(); - }; - - const play = (): (() => void) => { - let cancelFn: () => void; - // Increment session ID to invalidate callbacks from previous playback - const currentSessionId = ++playbackSessionIdRef.current; - // Helper to safely call onTaskChange only if this session is still active - const safeOnTaskChange = (taskId: string | null) => { - if (playbackSessionIdRef.current === currentSessionId) { - props?.onTaskChange?.(taskId); - } - }; - Promise.resolve( - (async () => { - if (!app || !appInitialized.current) { - throw new Error('app is not initialized'); - } - if (!scripts) { - throw new Error('scripts is required'); - } - const { frame, cancel, timeout, sleep: baseSleep } = frameKit(); - - // Scale duration by playback speed (faster playback = shorter duration) - const scaleByPlaybackSpeed = (duration: number) => - duration / playbackSpeed; - // Wrap sleep to apply playback speed - const sleep = (ms: number) => baseSleep(scaleByPlaybackSpeed(ms)); - cancelFn = cancel; - cancelAnimationRef.current = cancel; - const allImages: string[] = scripts - .filter((item) => !!item.img) - .map((item) => item.img!); - - // Load and display the image - await Promise.all( - [...allImages, mouseLoading, mousePointer].map(loadTexture), - ); - - // pointer on top - insightMarkContainer.removeChildren(); - await updatePointer(mousePointer, imageWidth / 2, imageHeight / 2); - await repaintImage(); - await updateCamera({ ...basicCameraState }); - const totalDuration = scaleByPlaybackSpeed( - scripts.reduce((acc, item) => { - return ( - acc + - item.duration + - (item.camera && item.insightCameraDuration - ? item.insightCameraDuration - : 0) - ); - }, 0), - ); - // progress bar - const progressUpdateInterval = 200; - const startTime = performance.now(); - setAnimationProgress(0); - const updateProgress = () => { - const progress = Math.min( - (performance.now() - startTime) / totalDuration, - 1, - ); - setAnimationProgress(progress); - if (progress < 1) { - return timeout(updateProgress, progressUpdateInterval); - } - }; - frame(updateProgress); - if (recorderSessionRef.current) { - recorderSessionRef.current.start(); - } - - // Track current taskId for callback - let currentTaskId: string | null = null; - - // Immediately notify the first task's taskId before starting the loop - // This ensures the sidebar highlights the first step right when playback starts - const firstTaskId = scripts[0]?.taskId ?? null; - if (firstTaskId) { - currentTaskId = firstTaskId; - safeOnTaskChange(currentTaskId); - } - - // play animation - for (const index in scripts) { - const item = scripts[index]; - setTitleText(item.title || ''); - setSubTitleText(item.subTitle || ''); - - // Notify task change if taskId changed - const newTaskId = item.taskId ?? null; - if (newTaskId !== currentTaskId) { - currentTaskId = newTaskId; - safeOnTaskChange(currentTaskId); - } - - if (item.type === 'sleep') { - await sleep(item.duration); - } else if (item.type === 'insight') { - if (!item.img) { - throw new Error('img is required'); - } - currentImg.current = item.img; - await repaintImage(item.imageWidth, item.imageHeight); - const highlightElements = item.highlightElement - ? [item.highlightElement] - : []; - await insightElementsAnimation( - [], - highlightElements, - item.searchArea, - scaleByPlaybackSpeed(item.duration), - frame, - ); - if (item.camera) { - if (!item.insightCameraDuration) { - throw new Error('insightCameraDuration is required'); - } - await cameraAnimation( - item.camera, - scaleByPlaybackSpeed(item.insightCameraDuration), - frame, - ); - } - } else if (item.type === 'clear-insight') { - await fadeOutItem( - insightMarkContainer, - scaleByPlaybackSpeed(item.duration), - frame, - ); - insightMarkContainer.removeChildren(); - insightMarkContainer.alpha = 1; - } else if (item.type === 'img') { - if (item.img && item.img !== currentImg.current) { - currentImg.current = item.img!; - await repaintImage(item.imageWidth, item.imageHeight); - } - if (item.camera) { - await cameraAnimation( - item.camera, - scaleByPlaybackSpeed(item.duration), - frame, - ); - } else { - await sleep(item.duration); - } - } else if (item.type === 'pointer') { - if (!item.img) { - throw new Error('pointer img is required'); - } - await updatePointer(item.img); - } else if (item.type === 'spinning-pointer') { - const stop = spinningPointer(frame); - await sleep(item.duration); - stop(); - } - } - - // Clear taskId when playback ends - safeOnTaskChange(null); - - if (recorderSessionRef.current) { - // Add delay to capture final frames before stopping the recorder - await sleep(1200); - recorderSessionRef.current.stop(); - recorderSessionRef.current = null; - setIsRecording(false); - } - })().catch((e) => { - console.error('player error', e); - - // Ignore frame cancel errors (these are expected when animation is cancelled) - if (e?.message === ERROR_FRAME_CANCEL) { - console.log('Animation cancelled (expected behavior)'); - // Clear taskId when cancelled (only if this session is still active) - safeOnTaskChange(null); - return; - } - - // Reset recording state on error - const wasRecording = !!recorderSessionRef.current; - if (recorderSessionRef.current) { - try { - recorderSessionRef.current.stop(); - } catch (stopError) { - console.error('Error stopping recorder:', stopError); - } - recorderSessionRef.current = null; - } - setIsRecording(false); - // Clear taskId on error (only if this session is still active) - safeOnTaskChange(null); - - // Only show error message if we were actually recording - if (wasRecording) { - message.error('Failed to export video. Please try again.'); - } - }), - ); - // Cleanup function - return () => { - cancelFn?.(); - cancelAnimationRef.current = null; - }; - }; - - useEffect(() => { - Promise.resolve( - (async () => { - await init(); - - // dynamically set the aspect ratio and fit mode of the container - if (divContainerRef.current && imageWidth && imageHeight) { - const aspectRatio = imageWidth / imageHeight; - divContainerRef.current.style.setProperty( - '--canvas-aspect-ratio', - aspectRatio.toString(), - ); - - // set adaptive mode for canvas container - divContainerRef.current.setAttribute('data-fit-mode', fitMode); - - // set adaptive mode for player container (for background color switching) - const playerContainer = divContainerRef.current.closest( - '.player-container', - ) as HTMLElement; - if (playerContainer) { - playerContainer.setAttribute('data-fit-mode', fitMode); - } - } - - triggerReplay(); - })(), - ); - - return () => { - // Mark app as not initialized before destroying - appInitialized.current = false; - try { - app.destroy(true, { children: true, texture: true }); - } catch (e) { - console.warn('destroy failed', e); - } - }; - }, [imageWidth, imageHeight, fitMode]); + // If no scripts, show empty + if (!scripts || scripts.length === 0 || !frameMap) { + return
; + } - useEffect(() => { - if (replayMark) { - return play(); - } - }, [replayMark]); + const imgW = frameMap.imageWidth; + const imgH = frameMap.imageHeight; + const isPortraitImage = imgH > imgW; - const [mouseOverStatusIcon, setMouseOverStatusIcon] = useState(false); - const [mouseOverSettingsIcon, setMouseOverSettingsIcon] = useState(false); - const progressString = Math.round(animationProgress * 100); - const transitionStyle = animationProgress === 0 ? 'none' : '0.3s'; + const compositionWidth = imgW; + const compositionHeight = imgH; + const isPortraitCanvas = imgH > imgW; - // press space to replay - const canReplayNow = animationProgress === 1; - useEffect(() => { - if (canReplayNow) { - const listener = (event: KeyboardEvent) => { - if (event.key === ' ') { - triggerReplay(); - } - }; - window.addEventListener('keydown', listener); - return () => { - window.removeEventListener('keydown', listener); - }; - } - }, [canReplayNow]); - - let statusIconElement; - let statusOnClick: () => void = () => {}; - if (animationProgress < 1) { - statusIconElement = ( - } size="default" /> - ); - } else { - // Finished - show replay button - statusIconElement = ( - } size="default" /> - ); - statusOnClick = () => triggerReplay(); - } + const totalFrames = frameMap.totalDurationInFrames; + const seekPercent = + totalFrames > 1 ? (player.currentFrame / (totalFrames - 1)) * 100 : 0; return ( -
-
-
-
+
+
+
+ {/* Render layer — renders at native resolution, scaled to fit & centered */}
-
-
-
-
-
-
-
{titleText}
- -
{subTitleText}
- -
- {isRecording ? null : ( -
setMouseOverStatusIcon(true)} - onMouseLeave={() => setMouseOverStatusIcon(false)} - onClick={statusOnClick} - > - {statusIconElement} -
- )} - - {props?.reportFileContent && props?.canDownloadReport !== false ? ( - + onClick={player.toggle} + > + {(() => { + const scale = + containerSize.width > 0 && containerSize.height > 0 + ? Math.min( + containerSize.width / compositionWidth, + containerSize.height / compositionHeight, + ) + : 1; + return (
setMouseOverStatusIcon(true)} - onMouseLeave={() => setMouseOverStatusIcon(false)} - onClick={() => downloadReport(props.reportFileContent!)} + style={{ + width: compositionWidth * scale, + height: compositionHeight * scale, + flexShrink: 0, + position: 'relative', + overflow: 'hidden', + }} > - +
+ +
- - ) : null} - + ); + })()} +
+ + {/* Subtitle — rendered in display coordinates, outside scaled content */} + {subtitleEnabled && subtitle && ( +
+ {subtitle.title && ( + {subtitle.title} + )} + {subtitle.subTitle && ( + + {subtitle.subTitle} + + )} +
+ )} + + {/* Control bar */} +
e.stopPropagation()} + > +
+ {player.playing ? : } +
+ + + {formatTime(player.currentFrame, frameMap.fps)} /{' '} + {formatTime(totalFrames, frameMap.fps)} + + +
- {isRecording ? ( - - ) : ( - - )} -
- - ( -
+ className="seek-bar-fill" + style={{ width: `${seekPercent}%` }} + /> +
+ {chapterMarkers.map((marker) => ( +
{ + e.stopPropagation(); + player.seekTo(marker.frame); }} + /> + + ))} +
+ + {/* Custom controls */} +
+ {props?.reportFileContent && + props?.canDownloadReport !== false ? ( + +
downloadReport(props.reportFileContent!)} > + +
+ + ) : null} + + ( +
+ {/* Export video */}
- - - Focus on cursor + {isExporting ? ( + + ) : ( + + )} + + {isExporting + ? `Exporting ${exportProgress}%` + : 'Export video'}
- { - setAutoZoom(checked); - triggerReplay(); - }} - /> -
-
-
- - Playback speed -
- {([0.5, 1, 1.5, 2] as PlaybackSpeedType[]).map((speed) => ( + +
+ + {/* Focus on cursor toggle */}
{ - setPlaybackSpeed(speed); - triggerReplay(); + className="player-settings-item" + style={{ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + height: '32px', + padding: '0 8px', + borderRadius: '4px', }} + > +
+ + + Focus on cursor + +
+ setAutoZoom(checked)} + /> +
+ + {/* Subtitle toggle */} +
- {speed}x +
+ + + Subtitle + +
+ setSubtitleEnabled(checked)} + />
- ))} -
- )} - menu={{ items: [] }} - > -
setMouseOverSettingsIcon(true)} - onMouseLeave={() => setMouseOverSettingsIcon(false)} - style={{ - cursor: 'pointer', - display: 'flex', - alignItems: 'center', - justifyContent: 'center', - opacity: mouseOverSettingsIcon ? 1 : 0.7, - transition: 'opacity 0.2s', - }} + +
+ + {/* Playback speed */} +
+ + Playback speed +
+ {([0.5, 1, 1.5, 2] as PlaybackSpeedType[]).map((speed) => ( +
setPlaybackSpeed(speed)} + style={{ + height: '32px', + lineHeight: '32px', + padding: '0 8px 0 24px', + fontSize: '12px', + cursor: 'pointer', + borderRadius: '4px', + }} + className={`player-speed-option${playbackSpeed === speed ? ' active' : ''}`} + > + {speed}x +
+ ))} +
+ )} + menu={{ items: [] }} > - +
+ +
+ + +
+ {isFullscreen ? : }
- +
diff --git a/packages/visualizer/src/component/player/remotion/StepScene.tsx b/packages/visualizer/src/component/player/remotion/StepScene.tsx new file mode 100644 index 0000000000..064b234ff5 --- /dev/null +++ b/packages/visualizer/src/component/player/remotion/StepScene.tsx @@ -0,0 +1,295 @@ +import { useMemo } from 'react'; +import { mouseLoading } from '../../../utils'; +import { deriveFrameState } from './derive-frame-state'; +import type { FrameMap } from './frame-calculator'; + +const POINTER_PHASE = 0.375; +const CROSSFADE_FRAMES = 10; + +// ── Main Component ── + +export const StepsTimeline: React.FC<{ + frameMap: FrameMap; + autoZoom: boolean; + frame: number; + width: number; + height: number; + fps: number; +}> = ({ + frameMap, + autoZoom, + frame, + width: compWidth, + height: compHeight, + fps, +}) => { + const { + scriptFrames, + imageWidth: baseImgW, + imageHeight: baseImgH, + } = frameMap; + + const state = useMemo( + () => deriveFrameState(scriptFrames, frame, baseImgW, baseImgH, fps), + [scriptFrames, frame, baseImgW, baseImgH, fps], + ); + + if (!state.img) return null; + + const { + img, + imageWidth: imgW, + imageHeight: imgH, + prevImg, + camera, + prevCamera, + insights, + spinning: spinningPointer, + spinningElapsedMs, + currentPointerImg, + title, + subTitle, + frameInScript, + imageChanged, + pointerMoved, + rawProgress, + } = state; + + // ── Camera interpolation ── + const pT = pointerMoved + ? Math.min(rawProgress / POINTER_PHASE, 1) + : rawProgress; + const cT = pointerMoved + ? rawProgress <= POINTER_PHASE + ? 0 + : Math.min((rawProgress - POINTER_PHASE) / (1 - POINTER_PHASE), 1) + : rawProgress; + + const pointerLeft = + prevCamera.pointerLeft + (camera.pointerLeft - prevCamera.pointerLeft) * pT; + const pointerTop = + prevCamera.pointerTop + (camera.pointerTop - prevCamera.pointerTop) * pT; + + const cameraLeft = autoZoom + ? prevCamera.left + (camera.left - prevCamera.left) * cT + : 0; + const cameraTop = autoZoom + ? prevCamera.top + (camera.top - prevCamera.top) * cT + : 0; + const cameraWidth = autoZoom + ? prevCamera.width + (camera.width - prevCamera.width) * cT + : imgW; + + // ── Layout calculations ── + const isPortraitImage = imgH > imgW; + const browserW = isPortraitImage + ? Math.round(compHeight * (imgW / imgH)) + : compWidth; + const portraitLeft = Math.round((compWidth - browserW) / 2); + + const zoom = imgW / cameraWidth; + const tx = -cameraLeft * (browserW / imgW); + const ty = -cameraTop * (compHeight / imgH); + const transformStyle = `scale(${zoom}) translate(${tx}px, ${ty}px)`; + + const camH = cameraWidth * (imgH / imgW); + const ptrX = ((pointerLeft - cameraLeft) / cameraWidth) * browserW; + const ptrY = ((pointerTop - cameraTop) / camH) * compHeight; + const showCursor = + camera.pointerLeft !== Math.round(imgW / 2) || + camera.pointerTop !== Math.round(imgH / 2) || + prevCamera.pointerLeft !== Math.round(imgW / 2) || + prevCamera.pointerTop !== Math.round(imgH / 2); + + const crossfadeAlpha = imageChanged + ? Math.min(frameInScript / CROSSFADE_FRAMES, 1) + : 1; + + const spinRotation = spinningPointer + ? ((Math.sin(spinningElapsedMs / 500 - Math.PI / 2) + 1) / 2) * Math.PI * 2 + : 0; + + // ── Insight overlay rendering ── + const renderInsightOverlays = () => { + if (insights.length === 0) return null; + return insights.map((insight, idx) => { + const overlays: React.ReactNode[] = []; + + if (insight.highlightElement) { + const rect = insight.highlightElement.rect; + overlays.push( +
, + ); + } + + if (insight.searchArea) { + const rect = insight.searchArea; + overlays.push( +
, + ); + } + + return overlays; + }); + }; + + // ── Content area rendering ── + const renderContentArea = (w: number, h: number) => ( +
+ {imageChanged && prevImg && crossfadeAlpha < 1 && ( +
+ +
+ )} + +
+ +
+ {renderInsightOverlays()} +
+
+ + {spinningPointer && ( + + )} + + {showCursor && !spinningPointer && ( + + )} +
+ ); + + return ( +
+ {isPortraitImage ? ( +
+ {renderContentArea(browserW, compHeight)} +
+ ) : ( + renderContentArea(compWidth, compHeight) + )} +
+ ); +}; diff --git a/packages/visualizer/src/component/player/remotion/derive-frame-state.ts b/packages/visualizer/src/component/player/remotion/derive-frame-state.ts new file mode 100644 index 0000000000..8e2f6587ad --- /dev/null +++ b/packages/visualizer/src/component/player/remotion/derive-frame-state.ts @@ -0,0 +1,285 @@ +/** + * Shared frame state derivation from ScriptFrame timeline. + * Used by both Remotion preview (StepScene.tsx) and Canvas export (export-branded-video.ts). + */ + +import { mousePointer } from '../../../utils'; +import type { ScriptFrame } from './frame-calculator'; + +export interface CameraState { + left: number; + top: number; + width: number; + pointerLeft: number; + pointerTop: number; +} + +export interface InsightOverlay { + highlightElement?: ScriptFrame['highlightElement']; + searchArea?: ScriptFrame['searchArea']; + alpha: number; +} + +export interface FrameState { + img: string; + imageWidth: number; + imageHeight: number; + prevImg: string | null; + camera: CameraState; + prevCamera: CameraState; + insights: InsightOverlay[]; + spinning: boolean; + spinningElapsedMs: number; + currentPointerImg: string; + title: string; + subTitle: string; + taskId: string | undefined; + frameInScript: number; + scriptIndex: number; + imageChanged: boolean; + pointerMoved: boolean; + rawProgress: number; +} + +/** Mutable accumulator used during frame state derivation */ +interface Acc { + img: string; + imgW: number; + imgH: number; + camera: CameraState; + prevCamera: CameraState; + prevImg: string | null; + insights: InsightOverlay[]; + spinning: boolean; + spinningElapsedMs: number; + pointerImg: string; + title: string; + subTitle: string; + taskId: string | undefined; + frameInScript: number; + scriptIndex: number; + imageChanged: boolean; + pointerMoved: boolean; + rawProgress: number; +} + +// ── Per-type handlers ── + +function updateImage( + acc: Acc, + sf: ScriptFrame, + baseW: number, + baseH: number, +): void { + if (!sf.img) return; + if (acc.img && sf.img !== acc.img) { + acc.prevImg = acc.img; + acc.imageChanged = true; + } + acc.img = sf.img; + acc.imgW = sf.imageWidth || baseW; + acc.imgH = sf.imageHeight || baseH; +} + +function checkPointerMoved(prev: CameraState, cur: CameraState): boolean { + return ( + Math.abs(prev.pointerLeft - cur.pointerLeft) > 1 || + Math.abs(prev.pointerTop - cur.pointerTop) > 1 + ); +} + +function handleImg( + acc: Acc, + sf: ScriptFrame, + frame: number, + baseW: number, + baseH: number, +): void { + updateImage(acc, sf, baseW, baseH); + const sfEnd = sf.startFrame + sf.durationInFrames; + if (sf.cameraTarget) { + acc.prevCamera = { ...acc.camera }; + acc.camera = { ...sf.cameraTarget }; + acc.pointerMoved = checkPointerMoved(acc.prevCamera, acc.camera); + } else if (frame >= sfEnd) { + acc.pointerMoved = false; + acc.imageChanged = false; + } + acc.spinning = false; +} + +function handleInsight( + acc: Acc, + sf: ScriptFrame, + frame: number, + baseW: number, + baseH: number, +): void { + updateImage(acc, sf, baseW, baseH); + + const alreadyAdded = acc.insights.some( + (ai) => + ai.highlightElement === sf.highlightElement && + ai.searchArea === sf.searchArea, + ); + if (!alreadyAdded) { + acc.insights.push({ + highlightElement: sf.highlightElement, + searchArea: sf.searchArea, + alpha: 1, + }); + } + + if (sf.cameraTarget && sf.insightPhaseFrames !== undefined) { + const cameraStartFrame = sf.startFrame + sf.insightPhaseFrames; + if (frame >= cameraStartFrame) { + acc.prevCamera = { ...acc.camera }; + acc.camera = { ...sf.cameraTarget }; + const cameraFrameIn = frame - cameraStartFrame; + const cameraDur = sf.cameraPhaseFrames || 1; + acc.rawProgress = Math.min(cameraFrameIn / cameraDur, 1); + acc.pointerMoved = checkPointerMoved(acc.prevCamera, acc.camera); + } + } + acc.spinning = false; +} + +function handleClearInsight(acc: Acc, sf: ScriptFrame, frame: number): void { + const sfEnd = sf.startFrame + sf.durationInFrames; + const alpha = 1 - acc.rawProgress; + acc.insights = acc.insights.map((ai) => ({ ...ai, alpha })); + if (frame >= sfEnd) { + acc.insights = []; + } + acc.spinning = false; +} + +function handleSpinningPointer(acc: Acc, fps: number): void { + acc.spinning = true; + acc.spinningElapsedMs = (acc.frameInScript / fps) * 1000; +} + +// ── Main derivation ── + +export function deriveFrameState( + scriptFrames: ScriptFrame[], + frame: number, + baseW: number, + baseH: number, + fps: number, +): FrameState { + const defaultCamera: CameraState = { + left: 0, + top: 0, + width: baseW, + pointerLeft: Math.round(baseW / 2), + pointerTop: Math.round(baseH / 2), + }; + + const acc: Acc = { + img: '', + imgW: baseW, + imgH: baseH, + camera: { ...defaultCamera }, + prevCamera: { ...defaultCamera }, + prevImg: null, + insights: [], + spinning: false, + spinningElapsedMs: 0, + pointerImg: mousePointer, + title: '', + subTitle: '', + taskId: undefined, + frameInScript: 0, + scriptIndex: 0, + imageChanged: false, + pointerMoved: false, + rawProgress: 0, + }; + + for (let i = 0; i < scriptFrames.length; i++) { + const sf = scriptFrames[i]; + const sfEnd = sf.startFrame + sf.durationInFrames; + + if (sf.durationInFrames === 0) { + if (sf.startFrame <= frame) { + if (sf.type === 'pointer' && sf.pointerImg) { + acc.pointerImg = sf.pointerImg; + } + acc.title = sf.title || acc.title; + acc.subTitle = sf.subTitle || acc.subTitle; + acc.taskId = sf.taskId ?? acc.taskId; + acc.scriptIndex = i; + } + continue; + } + + if (frame < sf.startFrame) break; + + acc.title = sf.title || acc.title; + acc.subTitle = sf.subTitle || acc.subTitle; + acc.taskId = sf.taskId ?? acc.taskId; + acc.scriptIndex = i; + acc.frameInScript = frame - sf.startFrame; + acc.rawProgress = Math.min(acc.frameInScript / sf.durationInFrames, 1); + + switch (sf.type) { + case 'img': + handleImg(acc, sf, frame, baseW, baseH); + break; + case 'insight': + handleInsight(acc, sf, frame, baseW, baseH); + break; + case 'clear-insight': + handleClearInsight(acc, sf, frame); + break; + case 'spinning-pointer': + handleSpinningPointer(acc, fps); + break; + case 'sleep': + acc.spinning = false; + break; + } + + if (frame >= sfEnd) { + if (sf.type !== 'clear-insight') acc.imageChanged = false; + acc.pointerMoved = false; + acc.rawProgress = 1; + if (sf.cameraTarget) { + acc.prevCamera = { ...acc.camera }; + } + } + } + + if (!acc.img) { + const firstImgScript = scriptFrames.find( + (sf) => sf.type === 'img' && sf.img, + ); + if (firstImgScript) { + acc.img = firstImgScript.img!; + acc.imgW = firstImgScript.imageWidth || baseW; + acc.imgH = firstImgScript.imageHeight || baseH; + } + } + + return { + img: acc.img, + imageWidth: acc.imgW, + imageHeight: acc.imgH, + prevImg: acc.imageChanged ? acc.prevImg : null, + camera: acc.camera, + prevCamera: acc.prevCamera, + insights: acc.insights, + spinning: acc.spinning, + spinningElapsedMs: acc.spinningElapsedMs, + currentPointerImg: acc.pointerImg, + title: acc.title, + subTitle: acc.subTitle, + taskId: acc.taskId, + frameInScript: acc.frameInScript, + scriptIndex: acc.scriptIndex, + imageChanged: acc.imageChanged, + pointerMoved: acc.pointerMoved, + rawProgress: acc.rawProgress, + }; +} diff --git a/packages/visualizer/src/component/player/remotion/export-branded-video.ts b/packages/visualizer/src/component/player/remotion/export-branded-video.ts new file mode 100644 index 0000000000..87014c69e5 --- /dev/null +++ b/packages/visualizer/src/component/player/remotion/export-branded-video.ts @@ -0,0 +1,310 @@ +import { mouseLoading, mousePointer } from '../../../utils'; +import { deriveFrameState } from './derive-frame-state'; +import type { InsightOverlay } from './derive-frame-state'; +import type { FrameMap } from './frame-calculator'; + +const W = 960; +const H = 540; +const POINTER_PHASE = 0.375; +const CROSSFADE_FRAMES = 10; + +// ── helpers ── + +function clamp(v: number, lo: number, hi: number): number { + return Math.min(Math.max(v, lo), hi); +} + +function lerp(a: number, b: number, t: number): number { + return a + (b - a) * t; +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = () => resolve(img); + img.onerror = reject; + img.src = src; + }); +} + +// ── Insight overlay drawing ── + +function drawInsightOverlays( + ctx: CanvasRenderingContext2D, + insights: InsightOverlay[], + cameraTransform: { zoom: number; tx: number; ty: number }, + bx: number, + contentY: number, +) { + for (const insight of insights) { + if (insight.alpha <= 0) continue; + ctx.save(); + ctx.globalAlpha *= insight.alpha; + + if (insight.highlightElement) { + const r = insight.highlightElement.rect; + const rx = + bx + + (r.left * cameraTransform.zoom + + cameraTransform.tx * cameraTransform.zoom); + const ry = + contentY + + (r.top * cameraTransform.zoom + + cameraTransform.ty * cameraTransform.zoom); + const rw = r.width * cameraTransform.zoom; + const rh = r.height * cameraTransform.zoom; + + ctx.fillStyle = 'rgba(253, 89, 7, 0.4)'; + ctx.fillRect(rx, ry, rw, rh); + ctx.strokeStyle = '#fd5907'; + ctx.lineWidth = 1; + ctx.strokeRect(rx, ry, rw, rh); + ctx.shadowColor = 'rgba(51, 51, 51, 0.4)'; + ctx.shadowBlur = 2; + ctx.shadowOffsetX = 4; + ctx.shadowOffsetY = 4; + ctx.strokeRect(rx, ry, rw, rh); + ctx.shadowBlur = 0; + ctx.shadowOffsetX = 0; + ctx.shadowOffsetY = 0; + } + + if (insight.searchArea) { + const r = insight.searchArea; + const rx = + bx + + (r.left * cameraTransform.zoom + + cameraTransform.tx * cameraTransform.zoom); + const ry = + contentY + + (r.top * cameraTransform.zoom + + cameraTransform.ty * cameraTransform.zoom); + const rw = r.width * cameraTransform.zoom; + const rh = r.height * cameraTransform.zoom; + + ctx.fillStyle = 'rgba(2, 131, 145, 0.4)'; + ctx.fillRect(rx, ry, rw, rh); + ctx.strokeStyle = '#028391'; + ctx.lineWidth = 1; + ctx.strokeRect(rx, ry, rw, rh); + } + + ctx.restore(); + } +} + +// ── Spinning pointer Canvas drawing ── + +function drawSpinningPointer( + ctx: CanvasRenderingContext2D, + img: HTMLImageElement, + x: number, + y: number, + elapsedMs: number, +) { + const progress = (Math.sin(elapsedMs / 500 - Math.PI / 2) + 1) / 2; + const rotation = progress * Math.PI * 2; + ctx.save(); + ctx.translate(x, y); + ctx.rotate(rotation); + ctx.drawImage(img, -11, -14, 22, 28); + ctx.restore(); +} + +// ── Steps rendering ── + +function drawSteps( + ctx: CanvasRenderingContext2D, + stepsFrame: number, + frameMap: FrameMap, + imgCache: Map, + cursorImg: HTMLImageElement | null, + spinnerImg: HTMLImageElement | null, +) { + const { scriptFrames, imageWidth: baseW, imageHeight: baseH, fps } = frameMap; + const st = deriveFrameState(scriptFrames, stepsFrame, baseW, baseH, fps); + if (!st.img) return; + + const { + img, + prevImg, + imageWidth: imgW, + imageHeight: imgH, + camera, + prevCamera, + pointerMoved, + imageChanged, + rawProgress, + frameInScript: fInScript, + spinning, + spinningElapsedMs, + insights, + } = st; + + const pT = pointerMoved + ? Math.min(rawProgress / POINTER_PHASE, 1) + : rawProgress; + const cT = pointerMoved + ? rawProgress <= POINTER_PHASE + ? 0 + : Math.min((rawProgress - POINTER_PHASE) / (1 - POINTER_PHASE), 1) + : rawProgress; + + const camL = lerp(prevCamera.left, camera.left, cT); + const camT2 = lerp(prevCamera.top, camera.top, cT); + const camW = lerp(prevCamera.width, camera.width, cT); + const ptrX = lerp(prevCamera.pointerLeft, camera.pointerLeft, pT); + const ptrY = lerp(prevCamera.pointerTop, camera.pointerTop, pT); + + const zoom = imgW / camW; + const tx = -camL * (W / imgW); + const ty = -camT2 * (H / imgH); + + const crossAlpha = imageChanged + ? clamp(fInScript / CROSSFADE_FRAMES, 0, 1) + : 1; + + ctx.fillStyle = '#000'; + ctx.fillRect(0, 0, W, H); + + const drawImg = (src: string, alpha: number) => { + const imgEl = imgCache.get(src); + if (!imgEl || alpha <= 0) return; + ctx.save(); + ctx.globalAlpha = alpha; + ctx.beginPath(); + ctx.rect(0, 0, W, H); + ctx.clip(); + ctx.translate(tx * zoom, ty * zoom); + ctx.scale(zoom, zoom); + ctx.drawImage(imgEl, 0, 0, W, H); + ctx.restore(); + }; + + if (imageChanged && prevImg && crossAlpha < 1) { + drawImg(prevImg, 1 - crossAlpha); + } + drawImg(img, imageChanged ? crossAlpha : 1); + + if (insights.length > 0) { + drawInsightOverlays(ctx, insights, { zoom, tx, ty }, 0, 0); + } + + const camH = camW * (imgH / imgW); + const sX = ((ptrX - camL) / camW) * W; + const sY = ((ptrY - camT2) / camH) * H; + + const hasPtrData = + Math.abs(camera.pointerLeft - Math.round(baseW / 2)) > 1 || + Math.abs(camera.pointerTop - Math.round(baseH / 2)) > 1 || + Math.abs(prevCamera.pointerLeft - Math.round(baseW / 2)) > 1 || + Math.abs(prevCamera.pointerTop - Math.round(baseH / 2)) > 1; + + if (spinning && spinnerImg) { + drawSpinningPointer(ctx, spinnerImg, sX, sY, spinningElapsedMs); + } + + if (!spinning && hasPtrData && cursorImg) { + ctx.drawImage(cursorImg, sX - 3, sY - 2, 22, 28); + } +} + +// ── main export function ── + +export async function exportBrandedVideo( + frameMap: FrameMap, + onProgress?: (pct: number) => void, +): Promise { + const { totalDurationInFrames: total, fps } = frameMap; + + // 1. pre-load all images + const imgSrcs = new Set(); + for (const sf of frameMap.scriptFrames) { + if (sf.img) imgSrcs.add(sf.img); + } + const imgCache = new Map(); + await Promise.all( + [...imgSrcs].map(async (src) => { + try { + imgCache.set(src, await loadImage(src)); + } catch { + /* skip */ + } + }), + ); + + let cursorImg: HTMLImageElement | null = null; + let spinnerImg: HTMLImageElement | null = null; + try { + cursorImg = await loadImage(mousePointer); + } catch { + /* optional */ + } + try { + spinnerImg = await loadImage(mouseLoading); + } catch { + /* optional */ + } + + // 2. canvas + recorder + const canvas = document.createElement('canvas'); + canvas.width = W; + canvas.height = H; + const ctx = canvas.getContext('2d')!; + + const stream = canvas.captureStream(fps); + const recorder = new MediaRecorder(stream, { mimeType: 'video/webm' }); + const chunks: BlobPart[] = []; + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + + // 3. render loop + return new Promise((resolve, reject) => { + recorder.onerror = () => reject(new Error('MediaRecorder error')); + recorder.onstop = () => { + if (chunks.length === 0) { + reject(new Error('No video data')); + return; + } + const blob = new Blob(chunks, { type: 'video/webm' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'midscene_replay.webm'; + a.click(); + stream.getTracks().forEach((track) => track.stop()); + setTimeout(() => URL.revokeObjectURL(url), 1000); + resolve(); + }; + + recorder.start(); + const frameDuration = 1000 / fps; + const startTime = performance.now(); + let lastFrame = -1; + + const tick = () => { + const elapsed = performance.now() - startTime; + const targetFrame = Math.min( + Math.floor(elapsed / frameDuration), + total - 1, + ); + + if (targetFrame > lastFrame) { + lastFrame = targetFrame; + ctx.clearRect(0, 0, W, H); + drawSteps(ctx, targetFrame, frameMap, imgCache, cursorImg, spinnerImg); + onProgress?.((targetFrame + 1) / total); + } + + if (targetFrame < total - 1) { + requestAnimationFrame(tick); + } else { + setTimeout(() => recorder.stop(), frameDuration * 2); + } + }; + + requestAnimationFrame(tick); + }); +} diff --git a/packages/visualizer/src/component/player/remotion/frame-calculator.ts b/packages/visualizer/src/component/player/remotion/frame-calculator.ts new file mode 100644 index 0000000000..5a27ffdc3a --- /dev/null +++ b/packages/visualizer/src/component/player/remotion/frame-calculator.ts @@ -0,0 +1,225 @@ +import type { LocateResultElement, Rect } from '@midscene/core'; +import type { AnimationScript } from '../../../utils/replay-scripts'; + +export const FPS = 30; + +// ── New ScriptFrame-based data structures ── + +export interface ScriptFrame { + type: + | 'img' + | 'insight' + | 'clear-insight' + | 'pointer' + | 'spinning-pointer' + | 'sleep'; + startFrame: number; // steps-local frame offset (0-based) + durationInFrames: number; // total frames this script occupies + + // screenshot (img/insight) + img?: string; + imageWidth?: number; + imageHeight?: number; + + // camera target (img with camera / insight camera phase) + cameraTarget?: { + left: number; + top: number; + width: number; + pointerLeft: number; + pointerTop: number; + }; + + // insight two-phase: insightPhaseFrames + cameraPhaseFrames = durationInFrames + insightPhaseFrames?: number; + cameraPhaseFrames?: number; + + // insight overlay + highlightElement?: LocateResultElement; + searchArea?: Rect; + + // pointer type + pointerImg?: string; + + // metadata + title?: string; + subTitle?: string; + taskId?: string; +} + +export interface FrameMap { + scriptFrames: ScriptFrame[]; + totalDurationInFrames: number; + fps: number; // 30 + stepsDurationInFrames: number; + imageWidth: number; + imageHeight: number; +} + +// ── calculateFrameMap ── + +export function calculateFrameMap(scripts: AnimationScript[]): FrameMap { + // Determine base image dimensions from first img/insight script + let baseImageWidth = 1920; + let baseImageHeight = 1080; + for (const s of scripts) { + if ((s.type === 'img' || s.type === 'insight') && s.img) { + baseImageWidth = s.imageWidth || 1920; + baseImageHeight = s.imageHeight || 1080; + break; + } + } + + const scriptFrames: ScriptFrame[] = []; + let currentFrame = 0; + + for (const script of scripts) { + const durationMs = script.duration; + + switch (script.type) { + case 'sleep': { + const frames = Math.ceil((durationMs / 1000) * FPS); + scriptFrames.push({ + type: 'sleep', + startFrame: currentFrame, + durationInFrames: frames, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }); + currentFrame += frames; + break; + } + + case 'img': { + const frames = Math.max(Math.ceil((durationMs / 1000) * FPS), 1); + const camera = script.camera; + const iw = script.imageWidth || baseImageWidth; + const ih = script.imageHeight || baseImageHeight; + + const sf: ScriptFrame = { + type: 'img', + startFrame: currentFrame, + durationInFrames: frames, + img: script.img, + imageWidth: iw, + imageHeight: ih, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }; + + if (camera) { + sf.cameraTarget = { + left: camera.left, + top: camera.top, + width: camera.width, + pointerLeft: camera.pointerLeft ?? Math.round(iw / 2), + pointerTop: camera.pointerTop ?? Math.round(ih / 2), + }; + } + + scriptFrames.push(sf); + currentFrame += frames; + break; + } + + case 'insight': { + const insightPhaseFrames = Math.max( + Math.ceil((durationMs / 1000) * FPS), + 1, + ); + const cameraDurationMs = script.insightCameraDuration || 0; + const cameraPhaseFrames = Math.ceil((cameraDurationMs / 1000) * FPS); + const totalFrames = insightPhaseFrames + cameraPhaseFrames; + const iw = script.imageWidth || baseImageWidth; + const ih = script.imageHeight || baseImageHeight; + const camera = script.camera; + + const sf: ScriptFrame = { + type: 'insight', + startFrame: currentFrame, + durationInFrames: totalFrames, + img: script.img, + imageWidth: iw, + imageHeight: ih, + insightPhaseFrames, + cameraPhaseFrames, + highlightElement: script.highlightElement, + searchArea: script.searchArea, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }; + + if (camera) { + sf.cameraTarget = { + left: camera.left, + top: camera.top, + width: camera.width, + pointerLeft: camera.pointerLeft ?? Math.round(iw / 2), + pointerTop: camera.pointerTop ?? Math.round(ih / 2), + }; + } + + scriptFrames.push(sf); + currentFrame += totalFrames; + break; + } + + case 'clear-insight': { + const frames = Math.max(Math.ceil((durationMs / 1000) * FPS), 1); + scriptFrames.push({ + type: 'clear-insight', + startFrame: currentFrame, + durationInFrames: frames, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }); + currentFrame += frames; + break; + } + + case 'spinning-pointer': { + const frames = Math.max(Math.ceil((durationMs / 1000) * FPS), 1); + scriptFrames.push({ + type: 'spinning-pointer', + startFrame: currentFrame, + durationInFrames: frames, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }); + currentFrame += frames; + break; + } + + case 'pointer': { + // Instantaneous — 0 frames + scriptFrames.push({ + type: 'pointer', + startFrame: currentFrame, + durationInFrames: 0, + pointerImg: script.img, + title: script.title, + subTitle: script.subTitle, + taskId: script.taskId, + }); + // No frame advancement + break; + } + } + } + + const stepsDurationInFrames = Math.max(currentFrame, 1); + + return { + scriptFrames, + totalDurationInFrames: stepsDurationInFrames, + fps: FPS, + stepsDurationInFrames, + imageWidth: baseImageWidth, + imageHeight: baseImageHeight, + }; +} diff --git a/packages/visualizer/src/component/player/use-frame-player.ts b/packages/visualizer/src/component/player/use-frame-player.ts new file mode 100644 index 0000000000..ddfdced680 --- /dev/null +++ b/packages/visualizer/src/component/player/use-frame-player.ts @@ -0,0 +1,108 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; + +interface UseFramePlayerOptions { + durationInFrames: number; + fps: number; + autoPlay?: boolean; + loop?: boolean; + playbackRate?: number; +} + +interface FramePlayer { + currentFrame: number; + playing: boolean; + play: () => void; + pause: () => void; + toggle: () => void; + seekTo: (frame: number) => void; +} + +export function useFramePlayer(options: UseFramePlayerOptions): FramePlayer { + const { durationInFrames, fps, autoPlay = false, loop = false } = options; + const [currentFrame, setCurrentFrame] = useState(0); + const [playing, setPlaying] = useState(autoPlay); + + const playingRef = useRef(playing); + const frameRef = useRef(currentFrame); + const rateRef = useRef(options.playbackRate ?? 1); + const durationRef = useRef(durationInFrames); + const fpsRef = useRef(fps); + const loopRef = useRef(loop); + + playingRef.current = playing; + frameRef.current = currentFrame; + rateRef.current = options.playbackRate ?? 1; + durationRef.current = durationInFrames; + fpsRef.current = fps; + loopRef.current = loop; + + useEffect(() => { + if (!playing) return; + + let rafId: number; + let lastTime: number | null = null; + let accumulated = 0; + + const tick = (now: number) => { + if (lastTime !== null) { + const delta = (now - lastTime) * rateRef.current; + accumulated += delta; + const frameDuration = 1000 / fpsRef.current; + + while (accumulated >= frameDuration) { + accumulated -= frameDuration; + const next = frameRef.current + 1; + if (next >= durationRef.current) { + if (loopRef.current) { + frameRef.current = 0; + setCurrentFrame(0); + } else { + frameRef.current = durationRef.current - 1; + setCurrentFrame(durationRef.current - 1); + setPlaying(false); + return; + } + } else { + frameRef.current = next; + setCurrentFrame(next); + } + } + } + lastTime = now; + rafId = requestAnimationFrame(tick); + }; + + rafId = requestAnimationFrame(tick); + return () => cancelAnimationFrame(rafId); + }, [playing]); + + const play = useCallback(() => { + if (frameRef.current >= durationRef.current - 1) { + frameRef.current = 0; + setCurrentFrame(0); + } + setPlaying(true); + }, []); + + const pause = useCallback(() => setPlaying(false), []); + + const toggle = useCallback(() => { + if (playingRef.current) { + setPlaying(false); + } else { + if (frameRef.current >= durationRef.current - 1) { + frameRef.current = 0; + setCurrentFrame(0); + } + setPlaying(true); + } + }, []); + + const seekTo = useCallback((frame: number) => { + const clamped = Math.max(0, Math.min(frame, durationRef.current - 1)); + frameRef.current = clamped; + setCurrentFrame(clamped); + }, []); + + return { currentFrame, playing, play, pause, toggle, seekTo }; +} diff --git a/packages/visualizer/src/component/universal-playground/index.tsx b/packages/visualizer/src/component/universal-playground/index.tsx index 496ff04132..ebfac8ff14 100644 --- a/packages/visualizer/src/component/universal-playground/index.tsx +++ b/packages/visualizer/src/component/universal-playground/index.tsx @@ -149,9 +149,9 @@ export function UniversalPlayground({ handleRun: executeAction, handleStop, canStop, - } = usePlaygroundExecution( + } = usePlaygroundExecution({ playgroundSDK, - effectiveStorage, + storage: effectiveStorage, actionSpace, loading, setLoading, @@ -161,7 +161,8 @@ export function UniversalPlayground({ verticalMode, currentRunningIdRef, interruptedFlagRef, - ); + deviceType: componentConfig.deviceType, + }); // Override SDK config when environment config changes useEffect(() => { diff --git a/packages/visualizer/src/hooks/usePlaygroundExecution.ts b/packages/visualizer/src/hooks/usePlaygroundExecution.ts index 60b8bbd867..b100cfb755 100644 --- a/packages/visualizer/src/hooks/usePlaygroundExecution.ts +++ b/packages/visualizer/src/hooks/usePlaygroundExecution.ts @@ -50,6 +50,7 @@ function buildProgressContent(task: any): string { */ function wrapExecutionDumpForReplay( dump: ExecutionDump | IExecutionDump, + deviceType?: string, ): IGroupedActionDump { const modelBriefsSet = new Set(); @@ -77,25 +78,43 @@ function wrapExecutionDumpForReplay( groupName: 'Playground Execution', modelBriefs, executions: [dump], + deviceType, }; } +export interface UsePlaygroundExecutionOptions { + playgroundSDK: PlaygroundSDKLike | null; + storage: StorageProvider | undefined | null; + actionSpace: DeviceAction[]; + loading: boolean; + setLoading: (loading: boolean) => void; + setInfoList: React.Dispatch>; + replayCounter: number; + setReplayCounter: React.Dispatch>; + verticalMode: boolean; + currentRunningIdRef: React.MutableRefObject; + interruptedFlagRef: React.MutableRefObject>; + deviceType?: string; +} + /** * Hook for handling playground execution logic */ -export function usePlaygroundExecution( - playgroundSDK: PlaygroundSDKLike | null, - storage: StorageProvider | undefined | null, - actionSpace: DeviceAction[], - loading: boolean, - setLoading: (loading: boolean) => void, - setInfoList: React.Dispatch>, - replayCounter: number, - setReplayCounter: React.Dispatch>, - verticalMode: boolean, - currentRunningIdRef: React.MutableRefObject, - interruptedFlagRef: React.MutableRefObject>, -) { +export function usePlaygroundExecution(options: UsePlaygroundExecutionOptions) { + const { + playgroundSDK, + storage, + actionSpace, + loading, + setLoading, + setInfoList, + replayCounter, + setReplayCounter, + verticalMode, + currentRunningIdRef, + interruptedFlagRef, + deviceType, + } = options; // Get execution options from environment config const { deepLocate, deepThink, screenshotIncluded, domIncluded } = useEnvConfig(); @@ -261,7 +280,10 @@ export function usePlaygroundExecution( // This allows noReplayAPIs to display both output and report if (result?.dump) { if (result.dump.tasks && Array.isArray(result.dump.tasks)) { - const groupedDump = wrapExecutionDumpForReplay(result.dump); + const groupedDump = wrapExecutionDumpForReplay( + result.dump, + deviceType, + ); const info = allScriptsFromDump(groupedDump); setReplayCounter((c) => c + 1); replayInfo = info; @@ -333,6 +355,7 @@ export function usePlaygroundExecution( deepThink, screenshotIncluded, domIncluded, + deviceType, ], ); @@ -399,7 +422,10 @@ export function usePlaygroundExecution( executionData.dump?.tasks && Array.isArray(executionData.dump.tasks) ) { - const groupedDump = wrapExecutionDumpForReplay(executionData.dump); + const groupedDump = wrapExecutionDumpForReplay( + executionData.dump, + deviceType, + ); replayInfo = allScriptsFromDump(groupedDump); setReplayCounter((c) => c + 1); counter = replayCounter + 1; @@ -454,6 +480,7 @@ export function usePlaygroundExecution( setInfoList, verticalMode, replayCounter, + deviceType, ]); // Check if execution can be stopped diff --git a/packages/visualizer/src/store/store.tsx b/packages/visualizer/src/store/store.tsx index 304b9da22e..f5dba8e8dd 100644 --- a/packages/visualizer/src/store/store.tsx +++ b/packages/visualizer/src/store/store.tsx @@ -8,6 +8,7 @@ const ELEMENTS_VISIBLE_KEY = 'midscene-elements-visible'; const MODEL_CALL_DETAILS_KEY = 'midscene-model-call-details'; const DARK_MODE_KEY = 'midscene-dark-mode'; const PLAYBACK_SPEED_KEY = 'midscene-playback-speed'; +const SUBTITLE_ENABLED_KEY = 'midscene-subtitle-enabled'; const parseBooleanParam = (value: string | null): boolean | undefined => { if (value === null) { @@ -45,12 +46,14 @@ export const useGlobalPreference = create<{ modelCallDetailsEnabled: boolean; darkModeEnabled: boolean; playbackSpeed: PlaybackSpeedType; + subtitleEnabled: boolean; setBackgroundVisible: (visible: boolean) => void; setElementsVisible: (visible: boolean) => void; setAutoZoom: (enabled: boolean) => void; setModelCallDetailsEnabled: (enabled: boolean) => void; setDarkModeEnabled: (enabled: boolean) => void; setPlaybackSpeed: (speed: PlaybackSpeedType) => void; + setSubtitleEnabled: (enabled: boolean) => void; }>((set) => { const savedAutoZoom = localStorage.getItem(AUTO_ZOOM_KEY) !== 'false'; const savedBackgroundVisible = @@ -67,6 +70,8 @@ export const useGlobalPreference = create<{ const savedPlaybackSpeed = ( Number.isNaN(parsedPlaybackSpeed) ? 1 : parsedPlaybackSpeed ) as PlaybackSpeedType; + const savedSubtitleEnabled = + localStorage.getItem(SUBTITLE_ENABLED_KEY) !== 'false'; const autoZoomFromQuery = getQueryPreference('focusOnCursor'); const elementsVisibleFromQuery = getQueryPreference('showElementMarkers'); const darkModeFromQuery = getQueryPreference('darkMode'); @@ -113,6 +118,11 @@ export const useGlobalPreference = create<{ set({ playbackSpeed: speed }); localStorage.setItem(PLAYBACK_SPEED_KEY, speed.toString()); }, + subtitleEnabled: savedSubtitleEnabled, + setSubtitleEnabled: (enabled: boolean) => { + set({ subtitleEnabled: enabled }); + localStorage.setItem(SUBTITLE_ENABLED_KEY, enabled.toString()); + }, }; }); diff --git a/packages/visualizer/src/utils/pixi-loader.ts b/packages/visualizer/src/utils/pixi-loader.ts deleted file mode 100644 index 5310120470..0000000000 --- a/packages/visualizer/src/utils/pixi-loader.ts +++ /dev/null @@ -1,24 +0,0 @@ -import 'pixi.js/unsafe-eval'; -import * as PIXI from 'pixi.js'; - -const globalTextureMap = new Map(); - -export const loadTexture = async (img: string) => { - if (globalTextureMap.has(img)) return; - return PIXI.Assets.load(img).then((texture) => { - globalTextureMap.set(img, texture); - }); -}; - -export const getTextureFromCache = (name: string) => { - return globalTextureMap.get(name); -}; - -export const getTexture = async (name: string) => { - if (globalTextureMap.has(name)) { - return globalTextureMap.get(name); - } - - await loadTexture(name); - return globalTextureMap.get(name); -}; diff --git a/packages/visualizer/src/utils/replay-scripts.ts b/packages/visualizer/src/utils/replay-scripts.ts index 7bb3ee036e..159fe67f5a 100644 --- a/packages/visualizer/src/utils/replay-scripts.ts +++ b/packages/visualizer/src/utils/replay-scripts.ts @@ -137,6 +137,7 @@ export interface ReplayScriptsInfo { height?: number; sdkVersion?: string; modelBriefs: string[]; + deviceType?: string; } const capitalizeFirstLetter = (str: string) => { @@ -271,6 +272,7 @@ export const allScriptsFromDump = ( height: firstHeight, sdkVersion, modelBriefs, + deviceType: (normalizedDump as IGroupedActionDump).deviceType, }; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8789219895..688d9dc779 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -490,12 +490,6 @@ importers: antd: specifier: ^5.21.6 version: 5.21.6(date-fns@2.30.0)(moment@2.30.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - pixi-filters: - specifier: 6.0.5 - version: 6.0.5(pixi.js@8.1.1) - pixi.js: - specifier: 8.1.1 - version: 8.1.1 react: specifier: 18.3.1 version: 18.3.1 @@ -1526,9 +1520,6 @@ importers: specifier: ^1.11.11 version: 1.11.13 devDependencies: - '@pixi/unsafe-eval': - specifier: 7.4.2 - version: 7.4.2(@pixi/core@7.4.3) '@rsbuild/plugin-less': specifier: ^1.5.0 version: 1.5.0(@rsbuild/core@2.0.0-beta.1(core-js@3.47.0)) @@ -1562,12 +1553,6 @@ importers: http-server: specifier: 14.1.1 version: 14.1.1 - pixi-filters: - specifier: 6.0.5 - version: 6.0.5(pixi.js@8.1.1) - pixi.js: - specifier: 8.1.1 - version: 8.1.1 query-string: specifier: 9.1.1 version: 9.1.1 @@ -3526,41 +3511,6 @@ packages: resolution: {integrity: sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==} engines: {node: '>=14'} - '@pixi/color@7.4.3': - resolution: {integrity: sha512-a6R+bXKeXMDcRmjYQoBIK+v2EYqxSX49wcjAY579EYM/WrFKS98nSees6lqVUcLKrcQh2DT9srJHX7XMny3voQ==} - - '@pixi/colord@2.9.6': - resolution: {integrity: sha512-nezytU2pw587fQstUu1AsJZDVEynjskwOL+kibwcdxsMBFqPsFFNA7xl0ii/gXuDi6M0xj3mfRJj8pBSc2jCfA==} - - '@pixi/constants@7.4.3': - resolution: {integrity: sha512-QGmwJUNQy/vVEHzL6VGQvnwawLZ1wceZMI8HwJAT4/I2uAzbBeFDdmCS8WsTpSWLZjF/DszDc1D8BFp4pVJ5UQ==} - - '@pixi/core@7.4.3': - resolution: {integrity: sha512-5YDs11faWgVVTL8VZtLU05/Fl47vaP5Tnsbf+y/WRR0VSW3KhRRGTBU1J3Gdc2xEWbJhUK07KGP7eSZpvtPVgA==} - - '@pixi/extensions@7.4.3': - resolution: {integrity: sha512-FhoiYkHQEDYHUE7wXhqfsTRz6KxLXjuMbSiAwnLb9uG1vAgp6q6qd6HEsf4X30YaZbLFY8a4KY6hFZWjF+4Fdw==} - - '@pixi/math@7.4.3': - resolution: {integrity: sha512-/uJOVhR2DOZ+zgdI6Bs/CwcXT4bNRKsS+TqX3ekRIxPCwaLra+Qdm7aDxT5cTToDzdxbKL5+rwiLu3Y1egILDw==} - - '@pixi/runner@7.4.3': - resolution: {integrity: sha512-TJyfp7y23u5vvRAyYhVSa7ytq0PdKSvPLXu4G3meoFh1oxTLHH6g/RIzLuxUAThPG2z7ftthuW3qWq6dRV+dhw==} - - '@pixi/settings@7.4.3': - resolution: {integrity: sha512-SmGK8smc0PxRB9nr0UJioEtE9hl4gvj9OedCvZx3bxBwA3omA5BmP3CyhQfN8XJ29+o2OUL01r3zAPVol4l4lA==} - - '@pixi/ticker@7.4.3': - resolution: {integrity: sha512-tHsAD0iOUb6QSGGw+c8cyRBvxsq/NlfzIFBZLEHhWZ+Bx4a0MmXup6I/yJDGmyPCYE+ctCcAfY13wKAzdiVFgQ==} - - '@pixi/unsafe-eval@7.4.2': - resolution: {integrity: sha512-45LM2mpqziNTeIORjgJl042CyssfZ17gfHHWcPZIZIGtiXSBPBy+mKvtHh5PraG0wBxAk/Bcr+nCYtAl8yuwgw==} - peerDependencies: - '@pixi/core': 7.4.2 - - '@pixi/utils@7.4.3': - resolution: {integrity: sha512-NO3Y9HAn2UKS1YdxffqsPp+kDpVm8XWvkZcS/E+rBzY9VTLnNOI7cawSRm+dacdET3a8Jad3aDKEDZ0HmAqAFA==} - '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -4780,15 +4730,9 @@ packages: '@types/cors@2.8.19': resolution: {integrity: sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==} - '@types/css-font-loading-module@0.0.12': - resolution: {integrity: sha512-x2tZZYkSxXqWvTDgveSynfjq/T2HyiZHXb00j/+gy19yp70PHCizM48XFdjBCWH7eHBD0R5i/pw9yMBP/BH5uA==} - '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} - '@types/earcut@2.1.4': - resolution: {integrity: sha512-qp3m9PPz4gULB9MhjGID7wpo3gJ4bTGXm7ltNDsmOvsPduTeHp8wSW9YckBj3mljeOh4F0m2z/0JKAALRKbmLQ==} - '@types/eslint-scope@3.7.7': resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} @@ -4825,9 +4769,6 @@ packages: '@types/fs-extra@11.0.4': resolution: {integrity: sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==} - '@types/gradient-parser@0.1.5': - resolution: {integrity: sha512-r7K3NkJz3A95WkVVmjs0NcchhHstC2C/VIYNX4JC6tieviUNo774FFeOHjThr3Vw/WCeMP9kAT77MKbIRlO/4w==} - '@types/har-format@1.2.16': resolution: {integrity: sha512-fluxdy7ryD3MV6h8pTfTYpy/xQzCFC7m89nOH9y94cNqJ1mDIDPut7MnRHI3F6qRmh/cT2fUjG1MLdCNb4hE9A==} @@ -5061,9 +5002,6 @@ packages: '@webassemblyjs/wast-printer@1.14.1': resolution: {integrity: sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==} - '@webgpu/types@0.1.49': - resolution: {integrity: sha512-NMmS8/DofhH/IFeW+876XrHVWel+J/vdcFCHLDqeJgkH9x0DeiwjVd8LcBdaxdG/T7Rf8VUAYsA8X1efMzLjRQ==} - '@xhmikosr/archive-type@6.0.1': resolution: {integrity: sha512-PB3NeJL8xARZt52yDBupK0dNPn8uIVQDe15qNehUpoeeLWCZyAOam4vGXnoZGz2N9D1VXtjievJuCsXam2TmbQ==} engines: {node: ^14.14.0 || >=16.0.0} @@ -6505,9 +6443,6 @@ packages: duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} - earcut@2.2.4: - resolution: {integrity: sha512-/pjZsA1b4RPHbeWZQn66SWS8nZZWLQQ23oE3Eam7aroEFGEvwKAsJfZ9ytiEMycfzXWpca4FA9QIOehf7PocBQ==} - eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -6777,9 +6712,6 @@ packages: eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} - eventemitter3@5.0.1: - resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} - events-universal@1.0.1: resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} @@ -7935,9 +7867,6 @@ packages: resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==} engines: {node: '>=16'} - ismobilejs@1.1.1: - resolution: {integrity: sha512-VaFW53yt8QO61k2WJui0dHf4SlL8lxBofUuUmwBo0ljPk0Drz2TiuDW4jo3wDcv41qy/SxrJ+VAzJ/qYqsmzRw==} - isobject@3.0.1: resolution: {integrity: sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==} engines: {node: '>=0.10.0'} @@ -9159,9 +9088,6 @@ packages: resolution: {integrity: sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==} engines: {node: '>=0.10.0'} - parse-svg-path@0.1.2: - resolution: {integrity: sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ==} - parse5-htmlparser2-tree-adapter@7.1.0: resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} @@ -9286,14 +9212,6 @@ packages: resolution: {integrity: sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==} hasBin: true - pixi-filters@6.0.5: - resolution: {integrity: sha512-M7F+Xu4Ysp2iy7bNZxt6U4hxO+X8c3msVKOQAq/tf3sLt99XOiY54iH2Ixj5VRIesmjdLWA5CaUQ+TUfyBVe2g==} - peerDependencies: - pixi.js: '>=8.0.0-0' - - pixi.js@8.1.1: - resolution: {integrity: sha512-/4D1HokubR2TlA/3JdeAb/EsGlkSRt5SmmMiPnsw9QB1PHDdzR7Td1m401fGIjTRq5edl5Zrz29hYXPuVwzdIw==} - pkce-challenge@4.1.0: resolution: {integrity: sha512-ZBmhE1C9LcPoH9XZSdwiPtbPHZROwAnMy+kIFQVrnMCxY4Cudlz3gBOpzilgc0jOgRaiT3sIWfpMomW2ar2orQ==} engines: {node: '>=16.20.0'} @@ -14025,57 +13943,6 @@ snapshots: '@opentelemetry/semantic-conventions@1.38.0': optional: true - '@pixi/color@7.4.3': - dependencies: - '@pixi/colord': 2.9.6 - - '@pixi/colord@2.9.6': {} - - '@pixi/constants@7.4.3': {} - - '@pixi/core@7.4.3': - dependencies: - '@pixi/color': 7.4.3 - '@pixi/constants': 7.4.3 - '@pixi/extensions': 7.4.3 - '@pixi/math': 7.4.3 - '@pixi/runner': 7.4.3 - '@pixi/settings': 7.4.3 - '@pixi/ticker': 7.4.3 - '@pixi/utils': 7.4.3 - - '@pixi/extensions@7.4.3': {} - - '@pixi/math@7.4.3': {} - - '@pixi/runner@7.4.3': {} - - '@pixi/settings@7.4.3': - dependencies: - '@pixi/constants': 7.4.3 - '@types/css-font-loading-module': 0.0.12 - ismobilejs: 1.1.1 - - '@pixi/ticker@7.4.3': - dependencies: - '@pixi/extensions': 7.4.3 - '@pixi/settings': 7.4.3 - '@pixi/utils': 7.4.3 - - '@pixi/unsafe-eval@7.4.2(@pixi/core@7.4.3)': - dependencies: - '@pixi/core': 7.4.3 - - '@pixi/utils@7.4.3': - dependencies: - '@pixi/color': 7.4.3 - '@pixi/constants': 7.4.3 - '@pixi/settings': 7.4.3 - '@types/earcut': 2.1.4 - earcut: 2.2.4 - eventemitter3: 4.0.7 - url: 0.11.4 - '@pkgjs/parseargs@0.11.0': optional: true @@ -15602,14 +15469,10 @@ snapshots: dependencies: '@types/node': 18.19.118 - '@types/css-font-loading-module@0.0.12': {} - '@types/debug@4.1.12': dependencies: '@types/ms': 0.7.34 - '@types/earcut@2.1.4': {} - '@types/eslint-scope@3.7.7': dependencies: '@types/eslint': 9.6.1 @@ -15657,8 +15520,6 @@ snapshots: '@types/jsonfile': 6.1.4 '@types/node': 18.19.118 - '@types/gradient-parser@0.1.5': {} - '@types/har-format@1.2.16': {} '@types/hast@3.0.4': @@ -15947,8 +15808,6 @@ snapshots: '@webassemblyjs/ast': 1.14.1 '@xtuc/long': 4.2.2 - '@webgpu/types@0.1.49': {} - '@xhmikosr/archive-type@6.0.1': dependencies: file-type: 18.7.0 @@ -17640,8 +17499,6 @@ snapshots: duplexer@0.1.2: {} - earcut@2.2.4: {} - eastasianwidth@0.2.0: {} edit-json-file@1.8.1: @@ -18064,8 +17921,6 @@ snapshots: eventemitter3@4.0.7: {} - eventemitter3@5.0.1: {} - events-universal@1.0.1: dependencies: bare-events: 2.8.2 @@ -19429,8 +19284,6 @@ snapshots: isexe@3.1.1: {} - ismobilejs@1.1.1: {} - isobject@3.0.1: {} isomorphic-fetch@3.0.0: @@ -20941,8 +20794,6 @@ snapshots: parse-passwd@1.0.0: {} - parse-svg-path@0.1.2: {} - parse5-htmlparser2-tree-adapter@7.1.0: dependencies: domhandler: 5.0.3 @@ -21050,23 +20901,6 @@ snapshots: dependencies: pngjs: 3.4.0 - pixi-filters@6.0.5(pixi.js@8.1.1): - dependencies: - '@types/gradient-parser': 0.1.5 - pixi.js: 8.1.1 - - pixi.js@8.1.1: - dependencies: - '@pixi/colord': 2.9.6 - '@types/css-font-loading-module': 0.0.12 - '@types/earcut': 2.1.4 - '@webgpu/types': 0.1.49 - '@xmldom/xmldom': 0.8.10 - earcut: 2.2.4 - eventemitter3: 5.0.1 - ismobilejs: 1.1.1 - parse-svg-path: 0.1.2 - pkce-challenge@4.1.0: {} pkce-challenge@5.0.0: {}