Skip to content

Commit cae2068

Browse files
devhelprdevhelpr
committed
AI canvas create image node
1 parent cfe1d62 commit cae2068

File tree

8 files changed

+631
-2
lines changed

8 files changed

+631
-2
lines changed

‎apps/vps-web/src/app/custom-nodes/classes/base-rect-node-class.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ export class BaseRectNode {
6464
this.flowEngine = flowEngine;
6565
}
6666

67+
updateVisual: ((data: any) => void) | undefined = undefined;
68+
6769
getSettingsPopup:
6870
| ((popupContainer: HTMLElement) => IDOMElement | undefined)
6971
| undefined = (popupContainer: HTMLElement) => {

‎apps/vps-web/src/app/custom-nodes/classes/prompt-image-class.tsx‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import {
2+
createJSXElement,
3+
FlowNode,
4+
IComputeResult,
5+
IRectNodeComponent,
6+
InitialValues,
7+
} from '@devhelpr/visual-programming-system';
8+
import { NodeInfo } from '@devhelpr/web-flow-executor';
9+
import { BaseRectNode } from './base-rect-node-class';
10+
11+
export class PromptImageNode extends BaseRectNode {
12+
static readonly nodeTypeName: string = 'prompt-image-rect-node';
13+
static readonly nodeTitle: string = 'Prompt image';
14+
static readonly category: string = 'Default';
15+
16+
static readonly text: string = 'rect';
17+
18+
static readonly disableManualResize: boolean = true;
19+
20+
static initialWidth = 100;
21+
static intialHeight = 100;
22+
static getFormFields = (
23+
_getNode: () => IRectNodeComponent<NodeInfo>,
24+
_updated: () => void,
25+
_values?: InitialValues
26+
) => {
27+
return [];
28+
};
29+
constructor(
30+
id: string,
31+
updated: () => void,
32+
node: IRectNodeComponent<NodeInfo>
33+
) {
34+
super(id, updated, node);
35+
}
36+
compute = (
37+
_input: string,
38+
_loopIndex?: number,
39+
_payload?: any
40+
): Promise<IComputeResult> => {
41+
return new Promise<IComputeResult>((resolve) => {
42+
resolve({
43+
result: (this.node?.nodeInfo as any)?.prompt ?? 'prompt',
44+
output: (this.node?.nodeInfo as any)?.prompt ?? 'prompt',
45+
followPath: undefined,
46+
});
47+
});
48+
};
49+
/*
50+
w-min h-min
51+
min-w-min min-h-min
52+
p-0
53+
flex items-center justify-center
54+
*/
55+
imageElement: HTMLImageElement | undefined;
56+
childElementSelector = '.child-node-wrapper > article'; // '.child-node-wrapper > *:first-child'
57+
render(node: FlowNode<NodeInfo>) {
58+
const nodeInfo = node.nodeInfo as any;
59+
console.log(
60+
'render prompt image rect-node',
61+
node.width,
62+
node.height,
63+
(node?.nodeInfo as any)?.text
64+
);
65+
return (
66+
<div>
67+
<div
68+
getElement={(element: HTMLElement) => {
69+
this.rectElement = element;
70+
}}
71+
class={`w-full h-full relative rounded overflow-clip justify-center items-center text-center whitespace-pre inline-flex`}
72+
style={`min-width:${node.width ?? 50}px;min-height:${
73+
node.height ?? 50
74+
}px;background:${nodeInfo?.fillColor ?? 'black'};border: ${
75+
nodeInfo?.strokeWidth ?? '2'
76+
}px ${nodeInfo?.strokeColor ?? 'white'} solid;color:${
77+
nodeInfo?.strokeColor ?? 'white'
78+
}`}
79+
>
80+
<img
81+
class="w-full h-full cover"
82+
getElement={(element: HTMLImageElement) => {
83+
this.imageElement = element;
84+
}}
85+
>
86+
prompt image
87+
</img>
88+
</div>
89+
</div>
90+
);
91+
}
92+
onResize: ((width: number, height: number) => void) | undefined = undefined;
93+
94+
updateVisual = (data: any) => {
95+
if (this.imageElement && data && data.image && data.isImage) {
96+
this.imageElement.src = `data:${
97+
data.mimeType ? data.mimeType : 'image/png'
98+
};base64,${data.image}`;
99+
}
100+
};
101+
}

‎apps/vps-web/src/app/custom-nodes/classes/prompt-node-class.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import {
22
createJSXElement,
33
FlowNode,
4-
FormField,
54
IComputeResult,
65
IRectNodeComponent,
76
InitialValues,
@@ -73,7 +72,7 @@ export class PromptNode extends BaseRectNode {
7372
super(id, updated, node);
7473
}
7574
compute = (
76-
input: string,
75+
_input: string,
7776
_loopIndex?: number,
7877
_payload?: any
7978
): Promise<IComputeResult> => {

‎apps/vps-web/src/app/custom-nodes/prompt-image-worker.ts‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
import {
2+
FormField,
3+
IComputeResult,
4+
IConnectionNodeComponent,
5+
InitialValues,
6+
IFlowCanvasBase,
7+
IRectNodeComponent,
8+
NodeTask,
9+
ThumbConnectionType,
10+
ThumbType,
11+
visualNodeFactory,
12+
} from '@devhelpr/visual-programming-system';
13+
import { NodeInfo } from '@devhelpr/web-flow-executor';
14+
import { promptNodeName } from './prompt-worker';
15+
16+
const fieldName = 'prompt-input';
17+
const nodeTitle = 'Prompt image';
18+
export const promptImageNodeName = 'prompt-image-rect-node';
19+
const familyName = 'flow-canvas';
20+
const thumbs = [
21+
{
22+
thumbType: ThumbType.StartConnectorCenter,
23+
thumbIndex: 0,
24+
connectionType: ThumbConnectionType.start,
25+
color: 'white',
26+
label: ' ',
27+
name: 'output',
28+
maxConnections: -1,
29+
},
30+
{
31+
thumbType: ThumbType.EndConnectorCenter,
32+
thumbIndex: 0,
33+
connectionType: ThumbConnectionType.end,
34+
color: 'white',
35+
label: ' ',
36+
name: 'input',
37+
maxConnections: -1,
38+
},
39+
];
40+
41+
export const getPromptImageNode =
42+
() =>
43+
(_updated: () => void): NodeTask<NodeInfo> => {
44+
let node: IRectNodeComponent<NodeInfo> | undefined = undefined;
45+
let canvasAppInstance: IFlowCanvasBase<NodeInfo> | undefined = undefined;
46+
47+
const initializeCompute = () => {
48+
connectedNodeInputs = {};
49+
return;
50+
};
51+
let connectedNodeInputs: Record<string, string> = {};
52+
const computeAsync = (
53+
input: string,
54+
_loopIndex?: number,
55+
_payload?: any,
56+
_dummy1?: any,
57+
_dummy2?: any,
58+
_dummy3?: any,
59+
connection?: IConnectionNodeComponent<NodeInfo>
60+
) => {
61+
if (connection?.startNode) {
62+
connectedNodeInputs[connection?.startNode.id] = input;
63+
}
64+
let allSet = true;
65+
let passThrough = true;
66+
if (node && node.connections) {
67+
node.connections.forEach((connectionNode) => {
68+
if (
69+
connectionNode?.endNode?.id &&
70+
connectionNode?.endNode?.id === node?.id
71+
) {
72+
if (connectionNode?.startNode?.id) {
73+
if (!connectedNodeInputs[connectionNode.startNode.id]) {
74+
allSet = false;
75+
}
76+
} else {
77+
allSet = false;
78+
}
79+
}
80+
if (
81+
connectionNode?.startNode?.id &&
82+
connectionNode?.startNode?.id === node?.id &&
83+
connectionNode?.endNode?.nodeInfo?.taskType !== promptNodeName
84+
) {
85+
passThrough = false;
86+
}
87+
});
88+
let hasOutputs = false;
89+
node.connections.forEach((connectionNode) => {
90+
if (
91+
connectionNode?.startNode?.id &&
92+
connectionNode?.startNode?.id === node?.id
93+
) {
94+
hasOutputs = true;
95+
}
96+
});
97+
if (!hasOutputs) {
98+
passThrough = false;
99+
}
100+
}
101+
102+
/*
103+
https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key=[googleGeminiAI-key]
104+
105+
{
106+
"contents": [{
107+
"parts":[{"text": "prompt"}]
108+
}]
109+
}
110+
111+
output: candidates[0].content.parts[0].text
112+
*/
113+
if (!allSet) {
114+
return Promise.resolve({
115+
output: '',
116+
result: '',
117+
followPath: undefined,
118+
stop: true,
119+
});
120+
}
121+
122+
return new Promise<IComputeResult>((resolve) => {
123+
let prompt = ''; //(node?.nodeInfo as any).formValues.prompt ?? 'prompt';
124+
125+
if (node && node.connections) {
126+
node.connections.forEach((connectionNode) => {
127+
if (
128+
connectionNode?.startNode?.id &&
129+
connectionNode?.endNode?.id &&
130+
connectionNode?.endNode?.id === node?.id
131+
) {
132+
if (connectedNodeInputs[connectionNode.startNode.id]) {
133+
prompt = `${prompt} - ${
134+
connectedNodeInputs[connectionNode.startNode.id]
135+
}`;
136+
}
137+
}
138+
});
139+
140+
const promptTask = `Create an image with the following specs: ${
141+
node?.nodeInfo?.formValues.prompt ?? ''
142+
}`;
143+
if (!passThrough) {
144+
const googleGeminiAIKey =
145+
canvasAppInstance?.getTempData('googleGeminiAI-key') ?? '';
146+
147+
let url =
148+
'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp-image-generation:generateContent?key=[googleGeminiAI-key]';
149+
150+
url = url.replace('[googleGeminiAI-key]', googleGeminiAIKey);
151+
152+
prompt = `${promptTask}${promptTask ? ': ' : ''}${prompt}`;
153+
console.log('prompt image node', prompt);
154+
155+
fetch(url, {
156+
method: 'post',
157+
158+
body: JSON.stringify({
159+
contents: [
160+
{
161+
parts: [{ text: prompt }],
162+
},
163+
],
164+
generationConfig: { responseModalities: ['Text', 'Image'] },
165+
}),
166+
mode: 'cors',
167+
})
168+
.then((response) => response.json())
169+
.then((result) => {
170+
const output = {
171+
image: result.candidates[0].content.parts[0].inlineData.data,
172+
mimeType:
173+
result.candidates[0].content.parts[0].inlineData.mimeType ??
174+
'',
175+
isImage: true,
176+
};
177+
//result.candidates[0].content.parts[0].text ?? '-';
178+
console.log('prompt output', output);
179+
180+
resolve({
181+
output: output,
182+
result: output,
183+
followPath: undefined,
184+
});
185+
});
186+
187+
return;
188+
}
189+
prompt = `${promptTask} - ${prompt}`;
190+
}
191+
192+
console.log('prompt worker', prompt);
193+
try {
194+
resolve({
195+
output: prompt,
196+
result: prompt,
197+
followPath: undefined,
198+
});
199+
} catch (error) {
200+
console.error('Error prompting:', error);
201+
resolve({
202+
output: '',
203+
result: '',
204+
followPath: undefined,
205+
stop: true,
206+
});
207+
}
208+
});
209+
};
210+
211+
return visualNodeFactory(
212+
promptImageNodeName,
213+
nodeTitle,
214+
familyName,
215+
fieldName,
216+
computeAsync as any,
217+
initializeCompute,
218+
false,
219+
100,
220+
100,
221+
thumbs,
222+
(_values?: InitialValues): FormField[] => {
223+
return [];
224+
},
225+
(nodeInstance) => {
226+
canvasAppInstance = nodeInstance.contextInstance;
227+
if (!nodeInstance.node.nodeInfo) {
228+
nodeInstance.node.nodeInfo = {};
229+
}
230+
node = nodeInstance.node as IRectNodeComponent<NodeInfo>;
231+
// nodeInstance.node.nodeInfo.shouldNotSendOutputFromWorkerToMainThread =
232+
// true;
233+
},
234+
{
235+
category: 'default',
236+
},
237+
undefined,
238+
true
239+
);
240+
};

‎apps/vps-web/src/app/custom-nodes/rect-node.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,9 @@ export const getRectNode =
175175
node.nodeInfo.isSettingsPopup = true;
176176
}
177177
}
178+
if (rectNode.updateVisual && node.nodeInfo) {
179+
node.nodeInfo.updateVisual = rectNode.updateVisual;
180+
}
178181

179182
const childNodeWrapper = (nodeRenderElement = (
180183
rect?.nodeComponent?.domElement as HTMLElement

‎apps/vps-web/src/app/custom-nodes/register-nodes.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { RectNode } from './classes/rect-node-class';
1616
import { WebcamViewerNode } from './classes/webcam-node';
1717
import { CanvasNode } from './classes/canvas-node-class';
1818
import { PromptNode } from './classes/prompt-node-class';
19+
import { PromptImageNode } from './classes/prompt-image-class';
1920

2021
const nodes: NodeRegistration[] = [
2122
() => ({
@@ -28,6 +29,7 @@ const nodes: NodeRegistration[] = [
2829
WebcamViewerNode,
2930
CanvasNode,
3031
PromptNode,
32+
PromptImageNode,
3133
];
3234

3335
export const registerNodes = (

0 commit comments

Comments
 (0)