feat(visualizer): device-aware shell rendering with Remotion Player

apps/report/src/App.less

@@ -81,12 +81,31 @@ footer.mt-8 {
   padding: 0 8px;
 }
 
+.main-layout {
+  display: flex;
+  flex-direction: row;
+  flex-grow: 1;
+  height: 100%;
+  overflow: hidden;
+}
+
 .page-side {
   height: calc(100% - 8px);
   background: #f2f4f7;
-  padding-right: 8px;
   margin-bottom: 8px;
-  overflow-x: auto;
+  flex-shrink: 0;
+  overflow-x: hidden;
+  overflow-y: auto;
+}
+
+.resize-handle {
+  width: 8px;
+  flex-shrink: 0;
+  cursor: col-resize;
+
+  &:hover {
+    background: rgba(0, 0, 0, 0.06);
+  }
 }
 
 [data-theme='dark'] .page-side {
@@ -181,6 +200,8 @@ footer.mt-8 {
   border-radius: 16px;
   background: #fff;
   margin-bottom: 8px;
+  min-width: 0;
+  overflow: hidden;
 
   .main-right-header {
     height: 50px;
@@ -295,6 +316,10 @@ footer.mt-8 {
     color: rgba(255, 255, 255, 0.45) !important;
   }
 
+  .resize-handle:hover {
+    background: rgba(255, 255, 255, 0.06);
+  }
+
   .main-right {
     background: #1f1f1f;
 

apps/report/src/App.tsx

@@ -31,6 +31,8 @@ import type {
 } from './types';
 
 let globalRenderCount = 1;
+const SIDEBAR_WIDTH_KEY = 'midscene-sidebar-width';
+const DEFAULT_SIDEBAR_WIDTH = 280;
 
 function Visualizer(props: VisualizerProps): JSX.Element {
   const { dumps } = props;
@@ -53,7 +55,10 @@ function Visualizer(props: VisualizerProps): JSX.Element {
   const modelBriefs = useExecutionDump((store) => store.modelBriefs);
   const reset = useExecutionDump((store) => store.reset);
   const [mainLayoutChangeFlag, setMainLayoutChangeFlag] = useState(0);
-  const mainLayoutChangedRef = useRef(false);
+  const [sidebarWidth, setSidebarWidth] = useState(() => {
+    const saved = localStorage.getItem(SIDEBAR_WIDTH_KEY);
+    return saved ? Number(saved) : DEFAULT_SIDEBAR_WIDTH;
+  });
   const dump = useExecutionDump((store) => store.dump);
   const [timelineCollapsed, setTimelineCollapsed] = useState(false);
   const {
@@ -153,62 +158,65 @@ function Visualizer(props: VisualizerProps): JSX.Element {
     );
 
     mainContent = (
-      <PanelGroup
-        autoSaveId="main-page-layout"
-        direction="horizontal"
-        onLayout={() => {
-          if (!mainLayoutChangedRef.current) {
-            setMainLayoutChangeFlag((prev) => prev + 1);
-          }
-        }}
-      >
-        <Panel maxSize={95} defaultSize={25} minSize={15}>
-          <div className="page-side">
-            <Sidebar
-              dumps={dumps}
-              proModeEnabled={proModeEnabled}
-              onProModeChange={setProModeEnabled}
-              replayAllScripts={replayAllScripts}
-              setReplayAllMode={setReplayAllMode}
-            />
-          </div>
-        </Panel>
-        <PanelResizeHandle
+      <div className="main-layout">
+        <div className="page-side" style={{ width: sidebarWidth }}>
+          <Sidebar
+            dumps={dumps}
+            proModeEnabled={proModeEnabled}
+            onProModeChange={setProModeEnabled}
+            replayAllScripts={replayAllScripts}
+            setReplayAllMode={setReplayAllMode}
+          />
+        </div>
+        <div
           className="resize-handle"
-          onDragging={(isChanging) => {
-            if (mainLayoutChangedRef.current && !isChanging) {
+          onMouseDown={(e) => {
+            e.preventDefault();
+            const startX = e.clientX;
+            const startWidth = sidebarWidth;
+            let latestWidth = startWidth;
+            const onMouseMove = (ev: MouseEvent) => {
+              latestWidth = Math.max(
+                200,
+                Math.min(500, startWidth + ev.clientX - startX),
+              );
+              setSidebarWidth(latestWidth);
+            };
+            const onMouseUp = () => {
+              document.removeEventListener('mousemove', onMouseMove);
+              document.removeEventListener('mouseup', onMouseUp);
+              localStorage.setItem(SIDEBAR_WIDTH_KEY, String(latestWidth));
               setMainLayoutChangeFlag((prev) => prev + 1);
-            }
-            mainLayoutChangedRef.current = isChanging;
+            };
+            document.addEventListener('mousemove', onMouseMove);
+            document.addEventListener('mouseup', onMouseUp);
           }}
         />
-        <Panel defaultSize={75} maxSize={95}>
-          <div className="main-right">
-            <div
-              className="main-right-header"
-              onClick={() => setTimelineCollapsed(!timelineCollapsed)}
-              style={{ cursor: 'pointer', userSelect: 'none' }}
+        <div className="main-right">
+          <div
+            className="main-right-header"
+            onClick={() => setTimelineCollapsed(!timelineCollapsed)}
+            style={{ cursor: 'pointer', userSelect: 'none' }}
+          >
+            <span
+              className="timeline-collapse-icon"
+              style={{
+                display: 'inline-block',
+                marginRight: 8,
+                transition: 'transform 0.2s',
+                transform: timelineCollapsed
+                  ? 'rotate(-90deg)'
+                  : 'rotate(0deg)',
+              }}
             >
-              <span
-                className="timeline-collapse-icon"
-                style={{
-                  display: 'inline-block',
-                  marginRight: 8,
-                  transition: 'transform 0.2s',
-                  transform: timelineCollapsed
-                    ? 'rotate(-90deg)'
-                    : 'rotate(0deg)',
-                }}
-              >
-                ▼
-              </span>
-              Record
-            </div>
-            {!timelineCollapsed && <Timeline key={mainLayoutChangeFlag} />}
-            <div className="main-content">{content}</div>
+              ▼
+            </span>
+            Record
           </div>
-        </Panel>
-      </PanelGroup>
+          {!timelineCollapsed && <Timeline key={mainLayoutChangeFlag} />}
+          <div className="main-content">{content}</div>
+        </div>
+      </div>
     );
   }
 

apps/report/src/components/sidebar/index.less

@@ -37,9 +37,13 @@
           cursor: pointer;
           display: flex;
           align-items: center;
+          justify-content: center;
+          padding: 6px;
+          border-radius: 8px;
+          transition: all 0.2s;
 
           &:hover {
-            background: #f5f5f5;
+            background: #e6e8eb;
           }
         }
       }
@@ -456,7 +460,7 @@
       .page-nav-left {
         .page-nav-toolbar .icon-button {
           &:hover {
-            background: rgba(255, 255, 255, 0.08);
+            background: #2c2e31;
           }
 
           svg {

apps/report/src/components/store/index.tsx

@@ -59,6 +59,7 @@ export interface DumpStoreType {
   modelBriefs: string[];
   insightWidth: number | null;
   insightHeight: number | null;
+  deviceType: string | undefined;
   activeExecution: ExecutionDump | null;
   activeExecutionAnimation: AnimationScript[] | null;
   activeTask: ExecutionTask | null;
@@ -94,6 +95,7 @@ export const useExecutionDump = create<DumpStoreType>((set, get) => {
     modelBriefs: [],
     insightWidth: null,
     insightHeight: null,
+    deviceType: undefined,
     activeTask: null,
     activeExecution: null,
     activeExecutionAnimation: null,
@@ -159,6 +161,7 @@ export const useExecutionDump = create<DumpStoreType>((set, get) => {
           height,
           modelBriefs,
           sdkVersion,
+          deviceType,
         } = allScriptsInfo;
 
         set({
@@ -167,6 +170,7 @@ export const useExecutionDump = create<DumpStoreType>((set, get) => {
           insightHeight: height,
           modelBriefs,
           sdkVersion,
+          deviceType,
         });
 
         const replayAvailable = allScripts.length > 0;

packages/core/src/agent/agent.ts

@@ -409,6 +409,7 @@ export class Agent<
       groupDescription: this.opts.groupDescription,
       executions: [],
       modelBriefs: [],
+      deviceType: this.interface.interfaceType,
     });
     this.executionDumpIndexByRunner = new WeakMap<TaskRunner, number>();
 

packages/core/src/types.ts

@@ -705,6 +705,7 @@ export interface IGroupedActionDump {
   groupDescription?: string;
   modelBriefs: string[];
   executions: IExecutionDump[];
+  deviceType?: string;
 }
 
 /**
@@ -716,6 +717,7 @@ export class GroupedActionDump implements IGroupedActionDump {
   groupDescription?: string;
   modelBriefs: string[];
   executions: ExecutionDump[];
+  deviceType?: string;
 
   constructor(data: IGroupedActionDump) {
     this.sdkVersion = data.sdkVersion;
@@ -725,6 +727,7 @@ export class GroupedActionDump implements IGroupedActionDump {
     this.executions = data.executions.map((exec) =>
       exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),
     );
+    this.deviceType = data.deviceType;
   }
 
   /**
@@ -771,6 +774,7 @@ export class GroupedActionDump implements IGroupedActionDump {
       groupDescription: this.groupDescription,
       modelBriefs: this.modelBriefs,
       executions: this.executions.map((exec) => exec.toJSON()),
+      deviceType: this.deviceType,
     };
   }
 

packages/visualizer/package.json

@@ -25,7 +25,6 @@
     "react-dom": ">=19.1.0"
   },
   "devDependencies": {
-    "@pixi/unsafe-eval": "7.4.2",
     "@rsbuild/plugin-less": "^1.5.0",
     "@rsbuild/plugin-node-polyfill": "1.4.2",
     "@rsbuild/plugin-react": "^1.4.1",
@@ -37,8 +36,6 @@
     "@types/react-dom": "^18.3.1",
     "execa": "9.3.0",
     "http-server": "14.1.1",
-    "pixi-filters": "6.0.5",
-    "pixi.js": "8.1.1",
     "query-string": "9.1.1",
     "react": "18.3.1",
     "react-dom": "18.3.1",
@@ -58,9 +55,11 @@
     "@midscene/playground": "workspace:*",
     "@midscene/shared": "workspace:*",
     "@midscene/web": "workspace:*",
+    "@remotion/player": "4.0.419",
     "antd": "^5.21.6",
     "buffer": "6.0.3",
-    "dayjs": "^1.11.11"
+    "dayjs": "^1.11.11",
+    "remotion": "4.0.419"
   },
   "license": "MIT"
 }