Skip to content

Commit b9965cc

Browse files
devhelprdevhelpr
committed
Add WebcamViewerNode and related logic; update NodeInfo interface
1 parent ac29ab3 commit b9965cc

File tree

9 files changed

+546
-19
lines changed

9 files changed

+546
-19
lines changed

‎apps/vps-web/src/app/ai-flow-engine-worker/ai-flow-engine-worker.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,17 @@ import {
99
AIWorkerMessage,
1010
AIWorkerWorkerSelf,
1111
} from './ai-flow-engine-worker-message';
12-
import { NodeInfo, run } from '@devhelpr/web-flow-executor';
12+
import { NodeInfo } from '@devhelpr/web-flow-executor';
1313
import { registerWorkerNodes } from '../custom-nodes/register-worker-nodes';
1414

1515
declare let self: AIWorkerWorkerSelf;
16-
console.log('WORKER RuntimeFlowContext', run);
1716
let flowEngine: RuntimeFlowEngine;
1817
// Message event handler
1918
self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
2019
try {
2120
const { data } = event;
2221
if (data.message === 'start-node') {
23-
console.log('start-node', data);
22+
//console.log('start-node', data);
2423
const nodeId = data.nodeId;
2524
if (flowEngine && nodeId) {
2625
const node = flowEngine.canvasApp.elements.get(nodeId);
@@ -42,6 +41,9 @@ self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
4241
undefined,
4342
undefined,
4443
(output, node) => {
44+
if (node.nodeInfo?.shouldNotSendOutputFromWorkerToMainThread) {
45+
return;
46+
}
4547
self.postMessage({
4648
message: 'node-update',
4749
result: {
@@ -59,7 +61,10 @@ self.addEventListener('message', (event: MessageEvent<AIWorkerMessage>) => {
5961
}
6062
flowEngine = new RuntimeFlowEngine();
6163
flowEngine.onSendUpdateToNode = (data, node) => {
62-
console.log('onSendUpdateToNode', data, node);
64+
//console.log('onSendUpdateToNode', data, node);
65+
if (node.nodeInfo?.shouldNotSendOutputFromWorkerToMainThread) {
66+
return;
67+
}
6368
self.postMessage({
6469
message: 'node-update',
6570
result: {

‎apps/vps-web/src/app/custom-nodes/classes/webcam-node.tsx‎

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
import {
2+
createElement,
3+
createJSXElement,
4+
FlowNode,
5+
IComputeResult,
6+
renderElement,
7+
} from '@devhelpr/visual-programming-system';
8+
import { getRunIndex, NodeInfo, runNode } from '@devhelpr/web-flow-executor';
9+
10+
import { BaseRectNode } from './base-rect-node-class';
11+
12+
interface WebcamViewerSettings {
13+
fadeRadius: number;
14+
fadePower: number;
15+
gamma: number;
16+
gridSize: number;
17+
}
18+
19+
class WebcamViewer {
20+
private rectNode: BaseRectNode;
21+
private videoElement: HTMLVideoElement | null = null;
22+
private stream: MediaStream | null = null;
23+
private isStreaming = false;
24+
private canvas: HTMLCanvasElement | null = null;
25+
private frameInterval: number | null = null;
26+
private readonly FPS = 2;
27+
private readonly FRAME_INTERVAL = 1000 / this.FPS;
28+
29+
constructor(
30+
gridContainer: HTMLDivElement,
31+
rectNode: BaseRectNode,
32+
settings: WebcamViewerSettings
33+
) {
34+
this.rectNode = rectNode;
35+
this.initializeWebcam(gridContainer);
36+
}
37+
38+
private async initializeWebcam(container: HTMLDivElement) {
39+
try {
40+
// Create video element
41+
this.videoElement = document.createElement('video');
42+
this.videoElement.autoplay = true;
43+
this.videoElement.playsInline = true;
44+
this.videoElement.style.width = '100%';
45+
this.videoElement.style.height = '100%';
46+
this.videoElement.style.objectFit = 'cover';
47+
48+
// Create canvas for frame capture
49+
this.canvas = document.createElement('canvas');
50+
this.canvas.style.display = 'none';
51+
52+
// Add elements to container
53+
container.appendChild(this.videoElement);
54+
container.appendChild(this.canvas);
55+
56+
// Get webcam stream
57+
this.stream = await navigator.mediaDevices.getUserMedia({
58+
video: true,
59+
audio: false,
60+
});
61+
62+
// Attach stream to video element
63+
if (this.videoElement) {
64+
this.videoElement.srcObject = this.stream;
65+
this.isStreaming = true;
66+
67+
// Start frame capture
68+
this.startFrameCapture();
69+
}
70+
} catch (error) {
71+
console.error('Error accessing webcam:', error);
72+
}
73+
}
74+
75+
private startFrameCapture() {
76+
if (this.frameInterval) {
77+
clearInterval(this.frameInterval);
78+
}
79+
80+
this.frameInterval = window.setInterval(() => {
81+
if (this.videoElement && this.canvas && this.isStreaming) {
82+
// Set canvas dimensions to match video
83+
this.canvas.width = this.videoElement.videoWidth;
84+
this.canvas.height = this.videoElement.videoHeight;
85+
86+
// Draw current video frame to canvas
87+
const ctx = this.canvas.getContext('2d');
88+
if (ctx) {
89+
ctx.drawImage(this.videoElement, 0, 0);
90+
91+
// Get image data as RGBA array
92+
const imageData = ctx.getImageData(
93+
0,
94+
0,
95+
this.canvas.width,
96+
this.canvas.height
97+
);
98+
const rgbaArray = imageData.data;
99+
100+
// Convert flat array to 2D array
101+
const frameData: number[][] = [];
102+
for (let y = 0; y < this.canvas.height; y++) {
103+
const row: number[] = [];
104+
for (let x = 0; x < this.canvas.width; x++) {
105+
const i = (y * this.canvas.width + x) * 4;
106+
row.push(
107+
rgbaArray[i], // R
108+
rgbaArray[i + 1], // G
109+
rgbaArray[i + 2], // B
110+
rgbaArray[i + 3] // A
111+
);
112+
}
113+
frameData.push(row);
114+
}
115+
116+
// Call the callback with frame data
117+
if (this.onReceiveWebcamFrame) {
118+
this.onReceiveWebcamFrame(frameData);
119+
}
120+
}
121+
}
122+
}, this.FRAME_INTERVAL);
123+
}
124+
125+
public cleanup() {
126+
if (this.frameInterval) {
127+
clearInterval(this.frameInterval);
128+
this.frameInterval = null;
129+
}
130+
131+
if (this.stream) {
132+
this.stream.getTracks().forEach((track) => track.stop());
133+
this.stream = null;
134+
this.isStreaming = false;
135+
}
136+
137+
if (this.videoElement) {
138+
this.videoElement.srcObject = null;
139+
this.videoElement.remove();
140+
this.videoElement = null;
141+
}
142+
143+
if (this.canvas) {
144+
this.canvas.remove();
145+
this.canvas = null;
146+
}
147+
}
148+
149+
public setupControlPanel(panel: HTMLDivElement) {
150+
panel.className = 'control-panel';
151+
152+
const controlsContainer = document.createElement('div');
153+
controlsContainer.className = 'controls-container';
154+
155+
renderElement(
156+
<element:Fragment>
157+
<div>Webcam Controls</div>
158+
<button onclick={() => this.toggleWebcam()}>
159+
{this.isStreaming ? 'Stop Webcam' : 'Start Webcam'}
160+
</button>
161+
</element:Fragment>,
162+
controlsContainer
163+
);
164+
165+
panel.appendChild(controlsContainer);
166+
}
167+
168+
private async toggleWebcam() {
169+
if (this.isStreaming) {
170+
this.cleanup();
171+
} else {
172+
if (this.videoElement) {
173+
await this.initializeWebcam(
174+
this.videoElement.parentElement as HTMLDivElement
175+
);
176+
}
177+
}
178+
}
179+
180+
onReceiveWebcamFrame: undefined | ((frameData: number[][]) => void) =
181+
undefined;
182+
onStorageChange: undefined | ((storageObject: WebcamViewerSettings) => void) =
183+
undefined;
184+
}
185+
186+
export class WebcamViewerNode extends BaseRectNode {
187+
static readonly nodeTypeName = 'webcam-viewer-node';
188+
static readonly nodeTitle = 'Webcam viewer';
189+
static readonly category = 'Default';
190+
static readonly text = 'webcam viewer';
191+
static readonly disableManualResize: boolean = true;
192+
193+
webcamViewer: WebcamViewer | undefined;
194+
195+
getSettingsPopup = (popupContainer: HTMLElement) => {
196+
const popupInstance = createElement(
197+
'div',
198+
{ class: 'max-h-[380px] h-[fit-content] p-3 pb-6' },
199+
popupContainer,
200+
undefined
201+
);
202+
if (popupInstance?.domElement && this.webcamViewer) {
203+
this.webcamViewer.setupControlPanel(
204+
popupInstance.domElement as HTMLDivElement
205+
);
206+
}
207+
return popupInstance;
208+
};
209+
210+
compute = (
211+
input: string,
212+
_loopIndex?: number,
213+
_payload?: any
214+
): Promise<IComputeResult> => {
215+
console.log('drawgrid compute', input);
216+
return new Promise<IComputeResult>((resolve) => {
217+
if (this.webcamViewer) {
218+
resolve({
219+
result: [],
220+
output: [],
221+
followPath: undefined,
222+
});
223+
} else {
224+
resolve({
225+
result: undefined,
226+
output: undefined,
227+
followPath: undefined,
228+
stop: true,
229+
});
230+
}
231+
});
232+
};
233+
234+
changeTimeout: NodeJS.Timeout | null = null;
235+
onReceiveWebcamFrame = (frameData: number[][]) => {
236+
// console.log('Received webcam frame:', {
237+
// width: frameData[0].length / 4,
238+
// height: frameData.length,
239+
// samplePixel: frameData[0].slice(0, 4), // Log first pixel's RGBA values
240+
// });
241+
242+
if (
243+
!this.node ||
244+
!this.canvasAppInstance ||
245+
!this.createRunCounterContext
246+
) {
247+
return;
248+
}
249+
if (this.flowEngine?.runNode) {
250+
console.log('run webcam node');
251+
this.flowEngine?.runNode(
252+
undefined,
253+
this.node,
254+
this.canvasAppInstance,
255+
() => {
256+
//
257+
},
258+
JSON.stringify(frameData),
259+
undefined,
260+
undefined,
261+
getRunIndex(),
262+
undefined,
263+
undefined,
264+
this.createRunCounterContext(false, false),
265+
false,
266+
{
267+
trigger: true,
268+
}
269+
);
270+
} else {
271+
runNode(
272+
this.node,
273+
this.canvasAppInstance,
274+
() => {
275+
//
276+
},
277+
JSON.stringify(frameData),
278+
undefined,
279+
undefined,
280+
getRunIndex(),
281+
undefined,
282+
undefined,
283+
this.createRunCounterContext(false, false),
284+
false,
285+
{
286+
trigger: true,
287+
}
288+
);
289+
}
290+
};
291+
292+
onStorageChange = (storageObject: WebcamViewerSettings) => {
293+
if (this.node && this.node.nodeInfo) {
294+
(this.node.nodeInfo as any).drawGridSettings = storageObject;
295+
this.updated();
296+
}
297+
};
298+
299+
childElementSelector = '.child-node-wrapper > *:first-child';
300+
render = (node: FlowNode<NodeInfo>) => {
301+
//const nodeInfo = node.nodeInfo as any;
302+
console.log('render rect-node', node.width, node.height);
303+
return (
304+
<div
305+
style={`min-width:${node.width ?? 50}px;min-height:${
306+
node.height ?? 50
307+
}px;`}
308+
>
309+
<div
310+
getElement={(element: HTMLDivElement) => {
311+
this.rectElement = element;
312+
}}
313+
renderElement={(element: HTMLElement) => {
314+
this.webcamViewer = new WebcamViewer(
315+
element as HTMLDivElement,
316+
this,
317+
(node.nodeInfo as any)?.drawGridSettings ?? {
318+
fadeRadius: 5, // 11x11 brush (5 cells in each direction from center)
319+
fadePower: 1.0,
320+
gamma: 2.0,
321+
gridSize: 28,
322+
}
323+
);
324+
325+
this.webcamViewer.onReceiveWebcamFrame = this.onReceiveWebcamFrame;
326+
this.webcamViewer.onStorageChange = this.onStorageChange;
327+
}}
328+
class={`draw-grid text-center whitespace-nowrap`}
329+
data-disable-interaction={true}
330+
></div>
331+
</div>
332+
);
333+
};
334+
}

‎apps/vps-web/src/app/custom-nodes/register-nodes.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
NodeRegistration,
1414
} from './utils/register-helpers';
1515
import { RectNode } from './classes/rect-node-class';
16+
import { WebcamViewerNode } from './classes/webcam-node';
1617

1718
const nodes: NodeRegistration[] = [
1819
() => ({
@@ -22,6 +23,7 @@ const nodes: NodeRegistration[] = [
2223
RectNode,
2324
OvalNode,
2425
DrawGridNode,
26+
WebcamViewerNode,
2527
];
2628

2729
export const registerNodes = (

0 commit comments

Comments
 (0)