fix(visualizer): add await to async screenshot.getData() calls

- Make generateAnimationScripts async and return Promise
- Add await to all getData() calls in replay-scripts.ts
- Make allScriptsFromDump async with Promise return type
- Convert forEach to for...of loop to support await
- Use useState + useEffect for async data in blackboard
- Add await to allScriptsFromDump calls in usePlaygroundExecution

Author: quanru

packages/visualizer/src/component/blackboard/index.tsx

@@ -101,7 +101,12 @@ export const Blackboard = (props: {
 
   const context = props.uiContext;
   const { size, screenshot } = context;
-  const screenshotBase64 = screenshot.getData();
+  const [screenshotBase64, setScreenshotBase64] = useState<string>('');
+
+  // Load screenshot data asynchronously
+  useEffect(() => {
+    screenshot.getData().then(setScreenshotBase64);
+  }, [screenshot]);
 
   const screenWidth = size.width;
   const screenHeight = size.height;

packages/visualizer/src/hooks/usePlaygroundExecution.ts

@@ -245,7 +245,7 @@ export function usePlaygroundExecution(
       if (result?.dump) {
         if (result.dump.tasks && Array.isArray(result.dump.tasks)) {
           const groupedDump = wrapExecutionDumpForReplay(result.dump);
-          const info = allScriptsFromDump(groupedDump);
+          const info = await allScriptsFromDump(groupedDump);
           setReplayCounter((c) => c + 1);
           replayInfo = info;
           counter = replayCounter + 1;
@@ -382,7 +382,7 @@ export function usePlaygroundExecution(
             Array.isArray(executionData.dump.tasks)
           ) {
             const groupedDump = wrapExecutionDumpForReplay(executionData.dump);
-            replayInfo = allScriptsFromDump(groupedDump);
+            replayInfo = await allScriptsFromDump(groupedDump);
             setReplayCounter((c) => c + 1);
             counter = replayCounter + 1;
           }

packages/visualizer/src/utils/replay-scripts.ts

@@ -145,14 +145,14 @@ const capitalizeFirstLetter = (str: string) => {
   return str.charAt(0).toUpperCase() + str.slice(1);
 };
 
-export const allScriptsFromDump = (
+export const allScriptsFromDump = async (
   dump:
     | GroupedActionDump
     | IGroupedActionDump
     | ExecutionDump
     | null
     | undefined,
-): ReplayScriptsInfo | null => {
+): Promise<ReplayScriptsInfo | null> => {
   if (!dump) {
     console.warn('[allScriptsFromDump] dump is empty');
     return {
@@ -205,8 +205,9 @@ export const allScriptsFromDump = (
 
   // Use first dimensions as default for the overall player size
   const allScripts: AnimationScript[] = [];
-  normalizedDump.executions?.filter(Boolean).forEach((execution) => {
-    const scripts = generateAnimationScripts(
+  const executions = normalizedDump.executions?.filter(Boolean) || [];
+  for (const execution of executions) {
+    const scripts = await generateAnimationScripts(
       execution,
       -1,
       firstWidth!,
@@ -227,7 +228,7 @@ export const allScriptsFromDump = (
         }
       }
     });
-  });
+  }
 
   const allScriptsWithoutIntermediateDoneFrame = allScripts.filter(
     (script, index) => {
@@ -270,12 +271,12 @@ export const allScriptsFromDump = (
   };
 };
 
-export const generateAnimationScripts = (
+export const generateAnimationScripts = async (
   execution: ExecutionDump | IExecutionDump | null,
   task: ExecutionTask | number,
   imageWidth: number,
   imageHeight: number,
-): AnimationScript[] | null => {
+): Promise<AnimationScript[] | null> => {
   if (!execution || !execution.tasks.length) return null;
   if (imageWidth === 0 || imageHeight === 0) {
     return null;
@@ -342,8 +343,9 @@ export const generateAnimationScripts = (
   const taskCount = tasksIncluded.length;
   let initSubTitle = '';
   let errorStateFlag = false;
-  tasksIncluded.forEach((task, index) => {
-    if (errorStateFlag) return;
+  for (let index = 0; index < tasksIncluded.length; index++) {
+    const task = tasksIncluded[index];
+    if (errorStateFlag) continue;
 
     if (index === 0) {
       initSubTitle = paramStr(task);
@@ -395,7 +397,7 @@ export const generateAnimationScripts = (
         const screenshotData =
           typeof context.screenshot === 'string'
             ? context.screenshot
-            : context.screenshot.getData();
+            : await context.screenshot.getData();
         scripts.push({
           type: 'img',
           img: screenshotData,
@@ -450,8 +452,8 @@ export const generateAnimationScripts = (
 
       const planningTask = task as ExecutionTaskPlanning;
       if (planningTask.recorder && planningTask.recorder.length > 0) {
-        const screenshotData =
-          planningTask.recorder[0]?.screenshot?.getData() || '';
+        const screenshot = planningTask.recorder[0]?.screenshot;
+        const screenshotData = screenshot ? await screenshot.getData() : '';
         scripts.push({
           type: 'img',
           img: screenshotData,
@@ -488,8 +490,10 @@ export const generateAnimationScripts = (
 
       currentCameraState = insightCameraState ?? fullPageCameraState;
       // const ifLastTask = index === taskCount - 1;
-      const actionScreenshotData =
-        task.recorder?.[0]?.screenshot?.getData() || '';
+      const actionScreenshot = task.recorder?.[0]?.screenshot;
+      const actionScreenshotData = actionScreenshot
+        ? await actionScreenshot.getData()
+        : '';
       scripts.push({
         type: 'img',
         img: actionScreenshotData,
@@ -508,7 +512,7 @@ export const generateAnimationScripts = (
 
       if (screenshot) {
         const screenshotData =
-          typeof screenshot === 'string' ? screenshot : screenshot.getData();
+          typeof screenshot === 'string' ? screenshot : await screenshot.getData();
         scripts.push({
           type: 'img',
           img: screenshotData,
@@ -529,10 +533,13 @@ export const generateAnimationScripts = (
         errorMsg.indexOf('NOT_IMPLEMENTED_AS_DESIGNED') > 0
           ? 'Further actions cannot be performed in the current environment'
           : errorMsg;
-      const errorScreenshotData =
+      const errorScreenshot =
         task.recorder && task.recorder.length > 0
-          ? task.recorder[task.recorder.length - 1].screenshot?.getData() || ''
-          : '';
+          ? task.recorder[task.recorder.length - 1].screenshot
+          : undefined;
+      const errorScreenshotData = errorScreenshot
+        ? await errorScreenshot.getData()
+        : '';
       scripts.push({
         type: 'img',
         img: errorScreenshotData,