Skip to content

Commit dfd6180

Browse files
devhelprdevhelpr
committed
Add offscreenCanvas support and improve WebcamViewer performance
1 parent b77d4b2 commit dfd6180

File tree

12 files changed

+358
-24
lines changed

12 files changed

+358
-24
lines changed

‎apps/vps-web/src/app/ai-flow-engine-worker/ai-flow-engine-worker-message.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export interface AIWorkerMessage {
1010
nodeId?: string;
1111
input?: any;
1212
inputPayload?: any;
13+
offscreenCanvases?: OffscreenCanvasNodes;
1314
}
1415

1516
export interface AIWorkerMessageResponse {
@@ -18,9 +19,16 @@ export interface AIWorkerMessageResponse {
1819
}
1920

2021
export interface AIWorkerWorker extends Omit<Worker, 'postMessage'> {
21-
postMessage: (message: AIWorkerMessage) => void;
22+
postMessage: (message: AIWorkerMessage, transfer?: Transferable[]) => void;
2223
}
2324

2425
export interface AIWorkerWorkerSelf extends Omit<Worker, 'postMessage'> {
2526
postMessage: (message: AIWorkerMessageResponse) => void;
2627
}
28+
29+
export interface OffscreenCanvasNode {
30+
id: string;
31+
offscreenCanvas: OffscreenCanvas;
32+
}
33+
34+
export type OffscreenCanvasNodes = OffscreenCanvasNode[];

‎apps/vps-web/src/app/ai-flow-engine-worker/ai-flow-engine-worker.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,27 @@ import { RuntimeFlowEngine } from '../flow-engine/flow-engine';
88
import {
99
AIWorkerMessage,
1010
AIWorkerWorkerSelf,
11+
OffscreenCanvasNodes,
1112
} from './ai-flow-engine-worker-message';
1213
import { NodeInfo } from '@devhelpr/web-flow-executor';
1314
import { registerWorkerNodes } from '../custom-nodes/register-worker-nodes';
1415

1516
declare let self: AIWorkerWorkerSelf;
1617
let flowEngine: RuntimeFlowEngine;
18+
let offscreenCanvases: OffscreenCanvasNodes = [];
19+
20+
// function getGradientColor(percent: number, canvas: OffscreenCanvas) {
21+
// const ctx = canvas.getContext('2d');
22+
// if (!ctx) {
23+
// return;
24+
// }
25+
// const gradient = ctx.createLinearGradient(0, 0, canvas.width, 0);
26+
// gradient.addColorStop(0, 'red');
27+
// gradient.addColorStop(1, 'blue');
28+
// ctx.fillStyle = gradient;
29+
// ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
30+
// }
31+
1732
// Message event handler
1833
self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
1934
try {
@@ -56,9 +71,16 @@ self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
5671
}
5772
}
5873
} else if (data.message === 'start') {
74+
offscreenCanvases = [];
5975
if (!data.flow) {
6076
throw new Error('Flow not provided');
6177
}
78+
if (data.offscreenCanvases) {
79+
offscreenCanvases = data.offscreenCanvases;
80+
// offscreenCanvases.forEach((canvasNode) => {
81+
// getGradientColor(40, canvasNode.offscreenCanvas);
82+
// });
83+
}
6284
flowEngine = new RuntimeFlowEngine();
6385
flowEngine.onSendUpdateToNode = (data, node) => {
6486
//console.log('onSendUpdateToNode', data, node);
@@ -74,6 +96,17 @@ self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
7496
});
7597
};
7698
flowEngine.initialize(data.flow.flows['flow'].nodes, registerWorkerNodes);
99+
flowEngine.canvasApp.elements.forEach((node) => {
100+
if (node && node.nodeInfo) {
101+
node.nodeInfo.offscreenCanvas = undefined;
102+
const offscreenCanvas = offscreenCanvases.find(
103+
(canvasNode) => canvasNode.id === node.id
104+
);
105+
if (offscreenCanvas) {
106+
node.nodeInfo.offscreenCanvas = offscreenCanvas.offscreenCanvas;
107+
}
108+
}
109+
});
77110
flowEngine
78111
.run()
79112
.then((output) => {

‎apps/vps-web/src/app/custom-nodes/canvas-node-worker.ts‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
import {
2+
FormField,
3+
IComputeResult,
4+
InitialValues,
5+
INodeComponent,
6+
NodeTask,
7+
ThumbConnectionType,
8+
ThumbType,
9+
visualNodeFactory,
10+
} from '@devhelpr/visual-programming-system';
11+
import { NodeInfo } from '@devhelpr/web-flow-executor';
12+
13+
const fieldName = 'canvas-rect-node';
14+
const nodeTitle = 'Canvas Node';
15+
export const canvasNodeName = 'canvas-rect-node';
16+
const familyName = 'flow-canvas';
17+
const thumbs = [
18+
{
19+
thumbType: ThumbType.StartConnectorCenter,
20+
thumbIndex: 0,
21+
connectionType: ThumbConnectionType.start,
22+
color: 'white',
23+
label: ' ',
24+
name: 'output',
25+
maxConnections: -1,
26+
},
27+
{
28+
thumbType: ThumbType.EndConnectorCenter,
29+
thumbIndex: 0,
30+
connectionType: ThumbConnectionType.end,
31+
color: 'white',
32+
label: ' ',
33+
name: 'input',
34+
maxConnections: 1,
35+
},
36+
];
37+
38+
export const getCanvasNode =
39+
() =>
40+
(_updated: () => void): NodeTask<NodeInfo> => {
41+
let node: INodeComponent<NodeInfo> | undefined = undefined;
42+
const initializeCompute = () => {
43+
return;
44+
};
45+
const computeAsync = (
46+
input: string,
47+
_loopIndex?: number,
48+
_payload?: any
49+
) => {
50+
return new Promise<IComputeResult>((resolve) => {
51+
// console.log(
52+
// 'canvas node compute',
53+
// input,
54+
// node?.nodeInfo?.offscreenCanvas
55+
// );
56+
// if (node?.nodeInfo?.offscreenCanvas) {
57+
// getGradientColor(40, node.nodeInfo.offscreenCanvas);
58+
// }
59+
const data = input as any;
60+
if (data.frame && node?.nodeInfo?.offscreenCanvas) {
61+
const helper: number[][] = data.frame as number[][];
62+
63+
const canvas = node?.nodeInfo?.offscreenCanvas;
64+
const context = canvas.getContext('2d');
65+
context!.imageSmoothingEnabled = false;
66+
67+
const frameHeight = helper.length;
68+
const frameWidth = helper[0].length / 4;
69+
70+
// Convert 2D array to flat Uint8ClampedArray
71+
const flatPixels = new Uint8ClampedArray(
72+
frameWidth * frameHeight * 4
73+
);
74+
75+
for (let y = 0; y < frameHeight; y++) {
76+
for (let x = 0; x < frameWidth; x++) {
77+
const i = (y * frameWidth + x) * 4;
78+
const j = x * 4;
79+
80+
flatPixels[i] = helper[y][j]; // R
81+
flatPixels[i + 1] = helper[y][j + 1]; // G
82+
flatPixels[i + 2] = helper[y][j + 2]; // B
83+
flatPixels[i + 3] = helper[y][j + 3]; // A
84+
}
85+
}
86+
87+
const imageData = new ImageData(flatPixels, frameWidth, frameHeight);
88+
89+
// Create a temporary offscreen canvas to draw the frame
90+
const tempCanvas = new OffscreenCanvas(frameWidth, frameHeight);
91+
const tempCtx = tempCanvas.getContext('2d');
92+
tempCtx!.putImageData(imageData, 0, 0);
93+
94+
// "cover" logic: scale up and crop overflow
95+
const scale = Math.max(
96+
canvas.width / frameWidth,
97+
canvas.height / frameHeight
98+
);
99+
100+
const displayWidth = frameWidth * scale;
101+
const displayHeight = frameHeight * scale;
102+
const offsetX = (canvas.width - displayWidth) / 2;
103+
const offsetY = (canvas.height - displayHeight) / 2;
104+
105+
context!.clearRect(0, 0, canvas.width, canvas.height);
106+
context!.save();
107+
108+
// mirror horizontally
109+
context!.translate(canvas.width, 0);
110+
context!.scale(-1, 1);
111+
112+
// draw image at correct mirrored position
113+
context!.drawImage(
114+
tempCanvas,
115+
0,
116+
0,
117+
frameWidth,
118+
frameHeight,
119+
canvas.width - offsetX - displayWidth, // flipped X
120+
offsetY,
121+
displayWidth,
122+
displayHeight
123+
);
124+
125+
context!.restore();
126+
}
127+
resolve({
128+
result: input,
129+
output: input,
130+
followPath: undefined,
131+
});
132+
});
133+
};
134+
135+
return visualNodeFactory(
136+
canvasNodeName,
137+
nodeTitle,
138+
familyName,
139+
fieldName,
140+
computeAsync,
141+
initializeCompute,
142+
false,
143+
200,
144+
100,
145+
thumbs,
146+
(_values?: InitialValues): FormField[] => {
147+
return [];
148+
},
149+
(nodeInstance) => {
150+
if (!nodeInstance.node.nodeInfo) {
151+
nodeInstance.node.nodeInfo = {};
152+
}
153+
nodeInstance.node.nodeInfo.shouldNotSendOutputFromWorkerToMainThread =
154+
true;
155+
156+
node = nodeInstance.node;
157+
},
158+
{
159+
category: 'default',
160+
},
161+
undefined,
162+
true
163+
);
164+
};

‎apps/vps-web/src/app/custom-nodes/classes/base-rect-node-class.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ export class BaseRectNode {
3737

3838
static readonly text: string = 'rect';
3939

40+
static initialWidth = 200;
41+
static intialHeight = 100;
42+
4043
static readonly disableManualResize: boolean = true;
4144
flowEngine: FlowEngine | undefined = undefined;
4245
constructor(

‎apps/vps-web/src/app/custom-nodes/classes/canvas-node-class.tsx‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import {
2+
createJSXElement,
3+
FlowNode,
4+
IComputeResult,
5+
IRectNodeComponent,
6+
} from '@devhelpr/visual-programming-system';
7+
import { NodeInfo } from '@devhelpr/web-flow-executor';
8+
import { BaseRectNode } from './base-rect-node-class';
9+
10+
export class CanvasNode extends BaseRectNode {
11+
static readonly nodeTypeName: string = 'canvas-rect-node';
12+
static readonly nodeTitle: string = 'Canvas node';
13+
static readonly category: string = 'Default';
14+
15+
static readonly text: string = 'rect';
16+
17+
static readonly disableManualResize: boolean = false;
18+
19+
static initialWidth = 300;
20+
static intialHeight = 300;
21+
22+
constructor(
23+
id: string,
24+
updated: () => void,
25+
node: IRectNodeComponent<NodeInfo>
26+
) {
27+
super(id, updated, node);
28+
}
29+
compute = (
30+
input: string,
31+
_loopIndex?: number,
32+
_payload?: any
33+
): Promise<IComputeResult> => {
34+
return new Promise<IComputeResult>((resolve) => {
35+
resolve({
36+
result: input,
37+
output: input,
38+
followPath: undefined,
39+
});
40+
});
41+
};
42+
/*
43+
w-min h-min
44+
min-w-min min-h-min
45+
p-0
46+
flex items-center justify-center
47+
*/
48+
49+
childElementSelector = '.child-node-wrapper > canvas'; // '.child-node-wrapper > *:first-child'
50+
render(node: FlowNode<NodeInfo>) {
51+
const nodeInfo = node.nodeInfo as any;
52+
console.log(
53+
'render rect-node',
54+
node.width,
55+
node.height,
56+
(node?.nodeInfo as any)?.text
57+
);
58+
return (
59+
<div>
60+
<div
61+
getElement={(element: HTMLElement) => {
62+
this.rectElement = element;
63+
}}
64+
class={`w-full h-full relative rounded overflow-clip justify-center items-center text-center whitespace-pre inline-flex`}
65+
style={`min-width:${node.width ?? 50}px;min-height:${
66+
node.height ?? 50
67+
}px;background:${nodeInfo?.fillColor ?? 'black'};border: ${
68+
nodeInfo?.strokeWidth ?? '2'
69+
}px ${nodeInfo?.strokeColor ?? 'white'} solid;color:${
70+
nodeInfo?.strokeColor ?? 'white'
71+
}`}
72+
>
73+
<canvas
74+
width={node.width ?? 50}
75+
height={node.height ?? 50}
76+
style={`width:${node.width ?? 50}px;height:${node.height ?? 50}px`}
77+
class="absolute top-0 left-0 right-0 bottom-0 canvas-node pointer-events-none"
78+
></canvas>
79+
</div>
80+
</div>
81+
);
82+
}
83+
onResize: ((width: number, height: number) => void) | undefined = undefined;
84+
}

‎apps/vps-web/src/app/custom-nodes/classes/webcam-node.tsx‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class WebcamViewer {
2323
private isStreaming = false;
2424
private canvas: HTMLCanvasElement | null = null;
2525
private frameInterval: number | null = null;
26-
private readonly FPS = 2;
26+
private readonly FPS = 30;
2727
private readonly FRAME_INTERVAL = 1000 / this.FPS;
2828

2929
constructor(
@@ -82,9 +82,11 @@ class WebcamViewer {
8282
// Set canvas dimensions to match video
8383
this.canvas.width = this.videoElement.videoWidth;
8484
this.canvas.height = this.videoElement.videoHeight;
85-
85+
if (this.canvas.width === 0 || this.canvas.height === 0) {
86+
return;
87+
}
8688
// Draw current video frame to canvas
87-
const ctx = this.canvas.getContext('2d');
89+
const ctx = this.canvas.getContext('2d', { willReadFrequently: true });
8890
if (ctx) {
8991
ctx.drawImage(this.videoElement, 0, 0);
9092

@@ -212,7 +214,6 @@ export class WebcamViewerNode extends BaseRectNode {
212214
_loopIndex?: number,
213215
_payload?: any
214216
): Promise<IComputeResult> => {
215-
console.log('drawgrid compute', input);
216217
return new Promise<IComputeResult>((resolve) => {
217218
if (this.webcamViewer) {
218219
resolve({
@@ -247,7 +248,6 @@ export class WebcamViewerNode extends BaseRectNode {
247248
return;
248249
}
249250
if (this.flowEngine?.runNode) {
250-
console.log('run webcam node');
251251
this.flowEngine?.runNode(
252252
undefined,
253253
this.node,
@@ -298,8 +298,6 @@ export class WebcamViewerNode extends BaseRectNode {
298298

299299
childElementSelector = '.child-node-wrapper > *:first-child';
300300
render = (node: FlowNode<NodeInfo>) => {
301-
//const nodeInfo = node.nodeInfo as any;
302-
console.log('render rect-node', node.width, node.height);
303301
return (
304302
<div
305303
style={`min-width:${node.width ?? 50}px;min-height:${

0 commit comments

Comments
 (0)