diff --git a/editor/src/assistant/ai-assistant-client.ts b/editor/src/assistant/ai-assistant-client.ts new file mode 100644 index 000000000..4073c8ab8 --- /dev/null +++ b/editor/src/assistant/ai-assistant-client.ts @@ -0,0 +1,145 @@ +// @ts-ignore - ES module imports in CommonJS context +import { ChatOllama } from "@langchain/ollama"; +// @ts-ignore - ES module imports in CommonJS context +import { HumanMessage, SystemMessage, BaseMessage } from "@langchain/core/messages"; +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +// @ts-ignore - ES module imports in CommonJS context +import { StateGraph, END, Annotation } from "@langchain/langgraph"; +// @ts-ignore - ES module imports in CommonJS context +import { ToolNode } from "@langchain/langgraph/prebuilt"; + +const DEFAULT_MODEL = "qwen2.5:3b"; +const DEFAULT_TEMPERATURE = 0.7; +const DEFAULT_MAX_TOKENS = 2048; +const DEFAULT_BASE_URL = "http://localhost:11434"; + +const StateAnnotation = Annotation.Root({ + messages: Annotation({ + reducer: (x, y) => x.concat(y), + }), +}); + +export interface AIAssistantConfig { + baseUrl?: string; + model?: string; + temperature?: number; + maxTokens?: number; +} + +export interface ChatOptions { + prompt: string; + systemPrompt?: string; +} + +export interface ChatResult { + response: string; + toolCalls: any[]; +} + +export class AIAssistantClient { + private config: AIAssistantConfig; + private tools: DynamicStructuredTool[]; + + constructor(tools: DynamicStructuredTool[], config: AIAssistantConfig = {}) { + this.tools = tools; + this.config = { + baseUrl: config.baseUrl || DEFAULT_BASE_URL, + model: config.model || DEFAULT_MODEL, + temperature: config.temperature ?? DEFAULT_TEMPERATURE, + maxTokens: config.maxTokens || DEFAULT_MAX_TOKENS, + }; + } + + async chat(options: ChatOptions): Promise { + const llm = new ChatOllama({ + model: this.config.model!, + baseUrl: this.config.baseUrl!, + temperature: this.config.temperature!, + numPredict: this.config.maxTokens!, + }); + + const llmWithTools = llm.bindTools(this.tools as any); + const toolNode = new ToolNode(this.tools as any); + + const callModel = async (state: typeof StateAnnotation.State) => { + const response = await llmWithTools.invoke(state.messages); + return { messages: [response] }; + }; + + const shouldContinue = (state: typeof StateAnnotation.State) => { + if (state.messages.length === 0) { + return END; + } + const lastMessage = state.messages[state.messages.length - 1]; + if (!lastMessage) { + return END; + } + return ("tool_calls" in lastMessage && + Array.isArray(lastMessage.tool_calls) && + lastMessage.tool_calls.length > 0) ? "tools" : END; + }; + + const workflow = new StateGraph(StateAnnotation) + .addNode("agent", callModel) + .addNode("tools", toolNode) + .addEdge("__start__", "agent") + .addConditionalEdges("agent", shouldContinue, { + tools: "tools", + [END]: END, + }) + .addEdge("tools", "agent"); + + const app = workflow.compile(); + + const messages: BaseMessage[] = []; + + const systemPrompt = options.systemPrompt || `You are an AI assistant for the Babylon.js Editor. You can help users modify their 3D scenes by changing properties of meshes, materials, lights, and cameras. + +When users ask you to modify objects in the scene: +1. First, list the available objects to see what's in the scene +2. Get the properties of specific objects if needed +3. Make the requested changes using the appropriate tools + +Always provide clear feedback about what changes you made.`; + + messages.push(new SystemMessage(systemPrompt)); + messages.push(new HumanMessage(options.prompt)); + + const result = await app.invoke({ messages }); + + if (result.messages.length === 0) { + throw new Error('No messages in response'); + } + + const lastMessage = result.messages[result.messages.length - 1]; + if (!lastMessage) { + throw new Error('Last message is undefined'); + } + + const responseContent = typeof lastMessage.content === 'string' + ? lastMessage.content + : JSON.stringify(lastMessage.content); + + const toolCalls: any[] = []; + result.messages.forEach((msg: any) => { + if (msg.tool_calls && Array.isArray(msg.tool_calls)) { + toolCalls.push(...msg.tool_calls); + } + }); + + return { + response: responseContent, + toolCalls, + }; + } + + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + getConfig(): AIAssistantConfig { + return { ...this.config }; + } +} + diff --git a/editor/src/assistant/ai-assistant-settings.tsx b/editor/src/assistant/ai-assistant-settings.tsx new file mode 100644 index 000000000..9795b21c7 --- /dev/null +++ b/editor/src/assistant/ai-assistant-settings.tsx @@ -0,0 +1,269 @@ +import { Component, ReactNode } from "react"; +import { IoCloseOutline, IoRefreshOutline } from "react-icons/io5"; +import { VscLoading } from "react-icons/vsc"; +import axios from "axios"; + +import { Button } from "../ui/shadcn/ui/button"; +import { Input } from "../ui/shadcn/ui/input"; +import { Label } from "../ui/shadcn/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../ui/shadcn/ui/select"; +import { Slider } from "../ui/shadcn/ui/slider"; +import { Switch } from "../ui/shadcn/ui/switch"; + +const DEFAULT_OLLAMA_URL = "http://localhost:11434"; +const SETTINGS_KEY = "ai-assistant-settings"; + +export interface AIAssistantSettings { + ollamaUrl: string; + model: string; + temperature: number; + messageHistoryLimit: number; + keepAllHistory: boolean; +} + +export interface IAIAssistantSettingsProps { + onClose: () => void; + onSave: (settings: AIAssistantSettings) => void; + currentSettings: AIAssistantSettings; +} + +interface IAIAssistantSettingsState { + ollamaUrl: string; + model: string; + temperature: number; + messageHistoryLimit: number; + keepAllHistory: boolean; + availableModels: string[]; + loading: boolean; + error: string | null; +} + +export class AIAssistantSettingsDialog extends Component { + constructor(props: IAIAssistantSettingsProps) { + super(props); + + this.state = { + ollamaUrl: props.currentSettings.ollamaUrl, + model: props.currentSettings.model, + temperature: props.currentSettings.temperature, + messageHistoryLimit: props.currentSettings.messageHistoryLimit, + keepAllHistory: props.currentSettings.keepAllHistory, + availableModels: [], + loading: false, + error: null, + }; + } + + componentDidMount(): void { + this._fetchModels(); + } + + private async _fetchModels(): Promise { + this.setState({ loading: true, error: null }); + + try { + const response = await axios.get(`${this.state.ollamaUrl}/api/tags`, { + timeout: 5000, + }); + + const models = response.data.models?.map((m: any) => m.name) || []; + + this.setState({ + availableModels: models, + loading: false, + model: models.length > 0 && !models.includes(this.state.model) ? models[0] : this.state.model, + }); + } catch (error) { + this.setState({ + error: `Failed to fetch models: ${error instanceof Error ? error.message : String(error)}`, + loading: false, + availableModels: [], + }); + } + } + + private _handleSave(): void { + const settings: AIAssistantSettings = { + ollamaUrl: this.state.ollamaUrl, + model: this.state.model, + temperature: this.state.temperature, + messageHistoryLimit: this.state.messageHistoryLimit, + keepAllHistory: this.state.keepAllHistory, + }; + + localStorage.setItem(SETTINGS_KEY, JSON.stringify(settings)); + + this.props.onSave(settings); + this.props.onClose(); + } + + public render(): ReactNode { + return ( +
+
+
+

AI Assistant Settings

+ +
+ +
+
+ +
+ this.setState({ ollamaUrl: e.target.value })} + placeholder="http://localhost:11434" + /> + +
+

+ URL of your Ollama instance +

+
+ + {this.state.error && ( +
+ {this.state.error} +
+ )} + +
+ + {this.state.availableModels.length > 0 ? ( +
+ +

+ {this.state.availableModels.length} models available +

+
+ ) : ( +
+ this.setState({ model: e.target.value })} + placeholder="qwen2.5:3b" + /> +

+ Enter model name manually or refresh to load from Ollama +

+
+ )} +
+ +
+
+ + + {this.state.temperature.toFixed(2)} + +
+ this.setState({ temperature: value })} + min={0} + max={2} + step={0.1} + /> +

+ Lower values are more focused, higher values are more creative +

+
+ +
+
+ + this.setState({ keepAllHistory: checked })} + /> +
+

+ When enabled, all messages are kept. When disabled, only the most recent messages are kept. +

+
+ + {!this.state.keepAllHistory && ( +
+
+ + + {this.state.messageHistoryLimit} + +
+ this.setState({ messageHistoryLimit: value })} + min={5} + max={100} + step={5} + /> +

+ Number of messages to keep in history +

+
+ )} +
+ +
+ + +
+
+
+ ); + } +} + +export function loadAIAssistantSettings(): AIAssistantSettings { + try { + const stored = localStorage.getItem(SETTINGS_KEY); + if (stored) { + return JSON.parse(stored); + } + } catch (error) { + console.error("Failed to load AI Assistant settings:", error); + } + + return { + ollamaUrl: DEFAULT_OLLAMA_URL, + model: "qwen2.5:3b", + temperature: 0.7, + messageHistoryLimit: 30, + keepAllHistory: false, + }; +} + diff --git a/editor/src/assistant/ai-assistant-ui.tsx b/editor/src/assistant/ai-assistant-ui.tsx new file mode 100644 index 000000000..6516de27d --- /dev/null +++ b/editor/src/assistant/ai-assistant-ui.tsx @@ -0,0 +1,341 @@ +import { Component, ReactNode } from "react"; +import { IoCloseOutline, IoSend, IoSettingsOutline, IoCheckmarkCircle, IoCloseCircle, IoAddOutline } from "react-icons/io5"; +import { VscLoading } from "react-icons/vsc"; + +import { Button } from "../ui/shadcn/ui/button"; +import { Input } from "../ui/shadcn/ui/input"; + +import { AIAssistantClient } from "./ai-assistant-client"; +import { createMeshTools } from "./tools/mesh-tool"; +import { createMaterialTools } from "./tools/material-tool"; +import { createLightTools } from "./tools/light-tool"; +import { createCameraTools } from "./tools/camera-tool"; +import { AIAssistantSettingsDialog, AIAssistantSettings, loadAIAssistantSettings } from "./ai-assistant-settings"; + +import { Editor } from "../editor/main"; + +const MESSAGES_KEY = "ai-assistant-messages"; + +export interface IAIAssistantUIProps { + editor: Editor; + onClose: () => void; +} + +interface Message { + role: "user" | "assistant"; + content: string; + timestamp: Date; +} + +export interface IAIAssistantUIState { + messages: Message[]; + input: string; + loading: boolean; + settings: AIAssistantSettings; + showSettings: boolean; + ollamaStatus: "checking" | "available" | "unavailable"; +} + +export class AIAssistantUI extends Component { + private client: AIAssistantClient | null = null; + private scrollRef: HTMLDivElement | null = null; + + constructor(props: IAIAssistantUIProps) { + super(props); + + this.state = { + messages: this._loadMessages(), + input: "", + loading: false, + settings: loadAIAssistantSettings(), + showSettings: false, + ollamaStatus: "checking", + }; + } + + componentDidMount(): void { + this._initializeClient(); + this._checkOllamaStatus(); + } + + private _loadMessages(): Message[] { + try { + const stored = localStorage.getItem(MESSAGES_KEY); + if (stored) { + const parsed = JSON.parse(stored); + return parsed.map((msg: any) => ({ + ...msg, + timestamp: new Date(msg.timestamp), + })); + } + } catch (error) { + console.error("Failed to load message history:", error); + } + return []; + } + + private _saveMessages(messages: Message[]): void { + try { + const settings = this.state.settings; + let messagesToSave = messages; + + if (!settings.keepAllHistory && messages.length > settings.messageHistoryLimit) { + messagesToSave = messages.slice(-settings.messageHistoryLimit); + } + + localStorage.setItem(MESSAGES_KEY, JSON.stringify(messagesToSave)); + } catch (error) { + console.error("Failed to save message history:", error); + } + } + + private _clearMessages(): void { + this.setState({ messages: [] }); + localStorage.removeItem(MESSAGES_KEY); + } + + componentDidUpdate(_prevProps: IAIAssistantUIProps, prevState: IAIAssistantUIState): void { + if ( + prevState.settings.model !== this.state.settings.model || + prevState.settings.temperature !== this.state.settings.temperature || + prevState.settings.ollamaUrl !== this.state.settings.ollamaUrl + ) { + this._initializeClient(); + } + + if (prevState.messages.length !== this.state.messages.length) { + this._scrollToBottom(); + this._saveMessages(this.state.messages); + } + } + + private _initializeClient(): void { + const scene = this.props.editor.layout.preview.scene; + if (!scene) return; + + const onSceneChanged = () => { + this.props.editor.layout.graph.refresh(); + this.props.editor.layout.inspector.forceUpdate(); + }; + + const tools = [ + ...createMeshTools(scene, onSceneChanged), + ...createMaterialTools(scene, onSceneChanged), + ...createLightTools(scene, onSceneChanged), + ...createCameraTools(scene, onSceneChanged), + ]; + + this.client = new AIAssistantClient(tools, { + baseUrl: this.state.settings.ollamaUrl, + model: this.state.settings.model, + temperature: this.state.settings.temperature, + }); + } + + private _scrollToBottom(): void { + if (this.scrollRef) { + this.scrollRef.scrollTop = this.scrollRef.scrollHeight; + } + } + + private async _checkOllamaStatus(): Promise { + this.setState({ ollamaStatus: "checking" }); + + try { + const response = await fetch(`${this.state.settings.ollamaUrl}/api/tags`, { + method: "GET", + signal: AbortSignal.timeout(3000), + }); + + if (response.ok) { + this.setState({ ollamaStatus: "available" }); + } else { + this.setState({ ollamaStatus: "unavailable" }); + } + } catch (error) { + this.setState({ ollamaStatus: "unavailable" }); + } + } + + private async _handleSend(): Promise { + if (!this.state.input.trim() || this.state.loading || !this.client) { + return; + } + + const userMessage: Message = { + role: "user", + content: this.state.input, + timestamp: new Date(), + }; + + this.setState({ + messages: [...this.state.messages, userMessage], + input: "", + loading: true, + }); + + try { + const result = await this.client.chat({ + prompt: this.state.input, + }); + + const assistantMessage: Message = { + role: "assistant", + content: result.response, + timestamp: new Date(), + }; + + this.setState({ + messages: [...this.state.messages, assistantMessage], + loading: false, + }); + } catch (error) { + const errorMessage: Message = { + role: "assistant", + content: `Error: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }; + + this.setState({ + messages: [...this.state.messages, errorMessage], + loading: false, + }); + } + } + + private _handleKeyPress(e: React.KeyboardEvent): void { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + this._handleSend(); + } + } + + public render(): ReactNode { + return ( +
+
+
+
+

AI Assistant

+ +
+ {this.state.ollamaStatus === "checking" && ( + + )} + {this.state.ollamaStatus === "available" && ( + + )} + {this.state.ollamaStatus === "unavailable" && ( + + )} + {this.state.settings.model} + • + T: {this.state.settings.temperature} +
+
+ +
+ + + + + +
+
+ +
(this.scrollRef = ref)} + className="flex-1 overflow-y-auto p-4 space-y-4" + > + {this.state.messages.length === 0 && ( +
+

Welcome to the AI Assistant!

+

Ask me to modify meshes, materials, lights, or cameras in your scene.

+
+ )} + + {this.state.messages.map((message, index) => ( +
+
+

{message.content}

+

+ {message.timestamp.toLocaleTimeString()} +

+
+
+ ))} + + {this.state.loading && ( +
+
+ +
+
+ )} +
+ +
+
+ this.setState({ input: e.target.value })} + onKeyPress={(e) => this._handleKeyPress(e)} + placeholder="Ask me to modify your scene..." + disabled={this.state.loading} + /> + +
+
+
+ + {this.state.showSettings && ( + this.setState({ showSettings: false })} + onSave={(settings) => { + this.setState({ settings }); + this._checkOllamaStatus(); + }} + /> + )} +
+ ); + } +} + diff --git a/editor/src/assistant/langgraph-llm-client.ts b/editor/src/assistant/langgraph-llm-client.ts new file mode 100644 index 000000000..c36be5886 --- /dev/null +++ b/editor/src/assistant/langgraph-llm-client.ts @@ -0,0 +1,596 @@ +// @ts-ignore - ES module imports in CommonJS context +import { ChatOllama } from "@langchain/ollama"; +// @ts-ignore - ES module imports in CommonJS context +import { HumanMessage, SystemMessage, BaseMessage } from "@langchain/core/messages"; +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +// @ts-ignore - ES module imports in CommonJS context +import { StateGraph, END, Annotation } from "@langchain/langgraph"; +// @ts-ignore - ES module imports in CommonJS context +import { ToolNode } from "@langchain/langgraph/prebuilt"; +import { z } from "zod"; +import axios from "axios"; + +interface ToolParameter { + name: string; + type: string; + description?: string; + required?: boolean; + enum?: string[]; +} + +interface ToolDefinitionResponse { + name: string; + description: string; + endpoint: string; + method: "GET" | "POST" | "PUT" | "DELETE"; + parameters: ToolParameter[]; +} + +interface ToolsDefinitionResponse { + tools: ToolDefinitionResponse[]; +} + +const DEFAULT_MAX_TOKENS = 2048; + +/** + * LLM Client Interface + * Defines the contract for LLM client implementations + */ +export interface LLMClient { + chat(options: { + model: string; + temperature: number; + prompt: string; + systemPrompt?: string; + toolsUrl?: string; + image?: { + fileName: string; + mimeType: string; + data: string; // Base64 encoded image data + }; + }): Promise; +} + +/** + * LLM Configuration Interface + */ +export interface LLMConfig { + baseUrl?: string; + defaultModel?: string; + defaultTemperature?: number; + defaultMaxTokens?: number; +} + +const StateAnnotation = Annotation.Root({ + messages: Annotation({ + reducer: (x, y) => x.concat(y), + }), +}); + +function createZodSchema(parameters: ToolDefinitionResponse["parameters"]): z.ZodObject { + const schemaObj: Record = {}; + + // Validate that parameters is an array + if (!Array.isArray(parameters)) { + console.warn('Tool parameters is not an array, using empty schema', parameters); + return z.object({}) as z.ZodObject; + } + + parameters.forEach(param => { + // Validate parameter structure + if (!param || typeof param !== 'object' || !param.name) { + console.warn('Invalid parameter structure, skipping', param); + return; + } + + let field: z.ZodTypeAny; + if (param.enum) { + field = z.enum(param.enum as [string, ...string[]]); + } else if (param.type === 'number') { + field = z.number(); + } else if (param.type === 'boolean') { + field = z.boolean(); + } else { + field = z.string(); + } + field = field.describe(param.description || ''); + if (!param.required) { + field = field.optional(); + } + schemaObj[param.name] = field; + }); + return z.object(schemaObj) as z.ZodObject; +} + +function createDynamicTool(toolDef: ToolDefinitionResponse, baseUrl: string): any { + // Validate required fields + if (!toolDef.endpoint || typeof toolDef.endpoint !== 'string') { + throw new Error(`Tool ${toolDef.name} is missing a valid endpoint`); + } + if (!toolDef.method || !['GET', 'POST', 'PUT', 'DELETE'].includes(toolDef.method)) { + throw new Error(`Tool ${toolDef.name} has invalid method: ${toolDef.method}`); + } + + // @ts-ignore - Type instantiation is excessively deep (TypeScript limitation with complex generics) + const schema = createZodSchema(toolDef.parameters); + // @ts-ignore - Type instantiation is excessively deep (TypeScript limitation with complex generics) + return new DynamicStructuredTool({ + name: toolDef.name, + description: toolDef.description, + schema, + func: async (input: Record) => { + // Resolve endpoint relative to base URL + const endpointUrl = toolDef.endpoint.startsWith('http') + ? toolDef.endpoint + : `${baseUrl}${toolDef.endpoint}`; + let response; + switch (toolDef.method) { + case "GET": + response = await axios.get(endpointUrl, { params: input }); + break; + case "POST": + response = await axios.post(endpointUrl, input); + break; + case "PUT": + response = await axios.put(endpointUrl, input); + break; + case "DELETE": + response = await axios.delete(endpointUrl, { data: input }); + break; + default: + throw new Error(`Unsupported HTTP method: ${toolDef.method}`); + } + return JSON.stringify(response.data); + }, + }); +} + +export class LangGraphLLMClient implements LLMClient { + private config: LLMConfig; + + constructor(config: LLMConfig = {}) { + this.config = config; + } + + async chat(options: { + model: string; + temperature: number; + prompt: string; + systemPrompt?: string; + maxTokens?: number; + image?: { + fileName: string; + mimeType: string; + data: string; // Base64 encoded image data + }; + toolsUrl?: string; + }): Promise { + try { + // Check if tools should be used + if (options.toolsUrl && options.toolsUrl.length > 0) { + return await this.chatWithTools({ + model: options.model, + temperature: options.temperature, + prompt: options.prompt, + systemPrompt: options.systemPrompt, + maxTokens: options.maxTokens, + image: options.image, + toolsUrl: options.toolsUrl + }, false) as string; + } else { + return await this.chatWithoutTools(options, false) as string; + } + } catch (error) { + console.error('LangGraph LLM call failed:', error); + throw error; + } + } + + async chatWithDetails(options: { + model: string; + temperature: number; + prompt: string; + systemPrompt?: string; + maxTokens?: number; + image?: { + fileName: string; + mimeType: string; + data: string; + }; + toolsUrl?: string; + }): Promise<{ + response: string; + metadata: { + model: string; + temperature: number; + maxTokens?: number; + messages: any[]; + toolCalls?: any[]; + executionTime: number; + timestamp: string; + }; + }> { + const startTime = Date.now(); + const timestamp = new Date().toISOString(); + + try { + let response: string; + let allMessages: any[] = []; + let toolCalls: any[] = []; + + // Check if tools should be used + if (options.toolsUrl && options.toolsUrl.length > 0) { + const result = await this.chatWithTools({ + model: options.model, + temperature: options.temperature, + prompt: options.prompt, + systemPrompt: options.systemPrompt, + maxTokens: options.maxTokens, + image: options.image, + toolsUrl: options.toolsUrl + }, true) as { response: string; messages: any[]; toolCalls: any[] }; + response = result.response; + allMessages = result.messages; + toolCalls = result.toolCalls; + } else { + const result = await this.chatWithoutTools({ + model: options.model, + temperature: options.temperature, + prompt: options.prompt, + systemPrompt: options.systemPrompt, + maxTokens: options.maxTokens, + image: options.image + }, true) as { response: string; messages: any[] }; + response = result.response; + allMessages = result.messages; + } + + const executionTime = Date.now() - startTime; + + return { + response, + metadata: { + model: options.model.trim(), + temperature: options.temperature, + maxTokens: options.maxTokens, + messages: allMessages, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + executionTime, + timestamp, + }, + }; + } catch (error) { + const executionTime = Date.now() - startTime; + + // Create a detailed error object + const errorDetails: any = { + message: error instanceof Error ? error.message : String(error), + type: error instanceof Error ? error.name : 'Error', + stack: error instanceof Error ? error.stack : undefined, + cause: error instanceof Error ? error.cause : undefined, + metadata: { + model: options.model.trim(), + temperature: options.temperature, + maxTokens: options.maxTokens, + messages: [], + executionTime, + timestamp, + }, + }; + + // If error is already an object with metadata, merge it + if (typeof error === 'object' && error !== null && 'metadata' in error) { + errorDetails.metadata = { ...errorDetails.metadata, ...(error as any).metadata }; + } + + // Create a proper Error object with the details attached + const enhancedError = new Error(errorDetails.message); + (enhancedError as any).details = errorDetails; + throw enhancedError; + } + } + + /** + * Create an LLM instance with the specified configuration + */ + private createLLMInstance(options: { + model: string; + temperature: number; + maxTokens?: number; + }): ChatOllama { + return new ChatOllama({ + model: options.model || this.config.defaultModel || "qwen3:1.7b", + baseUrl: this.config.baseUrl || "http://localhost:11434", + temperature: options.temperature !== undefined ? options.temperature : this.config.defaultTemperature || 0.7, + numPredict: options.maxTokens || this.config.defaultMaxTokens || DEFAULT_MAX_TOKENS, + }); + } + + /** + * Build messages array from prompt, system prompt, and optional image + */ + private buildMessages(options: { + prompt: string; + systemPrompt?: string; + image?: { + fileName: string; + mimeType: string; + data: string; + }; + }): BaseMessage[] { + const messages: BaseMessage[] = []; + + if (options.systemPrompt) { + messages.push(new SystemMessage(options.systemPrompt)); + } + + if (options.image) { + const imageDataUri = `data:${options.image.mimeType};base64,${options.image.data}`; + messages.push(new HumanMessage({ + content: [ + { type: 'text', text: options.prompt }, + { type: 'image_url', image_url: { url: imageDataUri } } + ] as any + })); + } else { + messages.push(new HumanMessage(options.prompt)); + } + + return messages; + } + + /** + * Serialize messages for metadata + */ + private serializeMessages(messages: BaseMessage[], response?: any): any[] { + const serialized = messages.map(msg => ({ + type: msg.constructor.name, + content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), + additional_kwargs: (msg as any).additional_kwargs || {}, + })); + + if (response) { + const responseContent = typeof response.content === 'string' ? response.content : JSON.stringify(response.content); + serialized.push({ + type: 'AIMessage', + content: responseContent, + additional_kwargs: (response as any).additional_kwargs || {}, + response_metadata: (response as any).response_metadata || {}, + } as any); + } + + return serialized; + } + + private async chatWithoutTools( + options: { + model: string; + temperature: number; + prompt: string; + systemPrompt?: string; + maxTokens?: number; + image?: { + fileName: string; + mimeType: string; + data: string; + }; + }, + detailed: boolean = false + ): Promise { + const llm = this.createLLMInstance({ + model: options.model, + temperature: options.temperature, + maxTokens: options.maxTokens, + }); + + const messages = this.buildMessages({ + prompt: options.prompt, + systemPrompt: options.systemPrompt, + image: options.image, + }); + + // @ts-ignore + const response = await llm.invoke(messages); + const responseContent = typeof response.content === 'string' ? response.content : JSON.stringify(response.content); + + if (detailed) { + return { + response: responseContent, + messages: this.serializeMessages(messages, response), + }; + } + + return responseContent; + } + + private async chatWithTools( + options: { + model: string; + temperature: number; + prompt: string; + systemPrompt?: string; + maxTokens?: number; + image?: { + fileName: string; + mimeType: string; + data: string; + }; + toolsUrl: string; + }, + detailed: boolean = false + ): Promise { + // Fetch tool definitions + const response = await axios.get(options.toolsUrl); + const toolDefs = response.data.tools; + + // Extract base URL from toolsUrl + const toolsUrlObj = new URL(options.toolsUrl); + const baseUrl = `${toolsUrlObj.protocol}//${toolsUrlObj.host}`; + + // Create tools with error handling + const tools: any[] = []; + const toolErrors: string[] = []; + for (const def of toolDefs) { + try { + const tool = createDynamicTool(def, baseUrl); + tools.push(tool); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(`Failed to create tool ${def.name}:`, errorMessage); + toolErrors.push(`Tool ${def.name}: ${errorMessage}`); + } + } + + if (tools.length === 0) { + throw new Error(`No valid tools could be created. Errors: ${toolErrors.join('; ')}`); + } + + if (toolErrors.length > 0) { + console.warn(`Some tools failed to load: ${toolErrors.join('; ')}`); + } + + // Create LLM with tools + const llm = this.createLLMInstance({ + model: options.model, + temperature: options.temperature, + maxTokens: options.maxTokens, + }); + + // Type assertions needed due to TypeScript's limitations with complex generic types in LangChain + const llmWithTools = llm.bindTools(tools as any); + const toolNode = new ToolNode(tools as any); + + const callModel = async (state: typeof StateAnnotation.State) => { + const response = await llmWithTools.invoke(state.messages); + return { messages: [response] }; + }; + + const shouldContinue = (state: typeof StateAnnotation.State) => { + if (state.messages.length === 0) { + return END; + } + const lastMessage = state.messages[state.messages.length - 1]; + if (!lastMessage) { + return END; + } + return ("tool_calls" in lastMessage && + Array.isArray(lastMessage.tool_calls) && + lastMessage.tool_calls.length > 0) ? "tools" : END; + }; + + const workflow = new StateGraph(StateAnnotation) + .addNode("agent", callModel) + .addNode("tools", toolNode) + .addEdge("__start__", "agent") + .addConditionalEdges("agent", shouldContinue, { + tools: "tools", + [END]: END, + }) + .addEdge("tools", "agent"); + + const app = workflow.compile(); + + // Enhanced system prompt with date handling rules + let enhancedSystemPrompt = options.systemPrompt || ''; + + // Add critical date handling instructions if tools are being used + if (tools.length > 0 && !enhancedSystemPrompt.includes('get_current_date')) { + const dateHandlingRules = ` + +CRITICAL DATE HANDLING RULES - YOU MUST FOLLOW THESE: +1. When the user mentions "today", "tomorrow", scheduling, or creating appointments: + - YOU MUST call get_current_date or get_current_datetime FIRST + - Extract the "today" or "now" field from the response + - Use THAT EXACT date value to create activities/appointments + - NEVER use hardcoded dates, dates from training data, or dates with years before 2024 + +2. Example workflow: + - User: "Create an appointment for today" + - Step 1: Call get_current_date → Get today="2025-11-09" + - Step 2: Use "2025-11-09T14:00:00Z" (or appropriate time) for create_activity + +3. WRONG: Using "2023-10-15T14:00:00Z" (old year, hardcoded) + RIGHT: Call get_current_date first, then use the returned date + +If you try to create an activity with a date from 2023 or earlier, it will be REJECTED.`; + + enhancedSystemPrompt = enhancedSystemPrompt + dateHandlingRules; + } + + // Build messages + const messages = this.buildMessages({ + prompt: options.prompt, + systemPrompt: enhancedSystemPrompt, + image: options.image, + }); + + const result = await app.invoke({ messages }); + + // Extract the final response + if (result.messages.length === 0) { + throw new Error('No messages in response'); + } + const lastMessage = result.messages[result.messages.length - 1]; + if (!lastMessage) { + throw new Error('Last message is undefined'); + } + const responseContent = typeof lastMessage.content === 'string' ? lastMessage.content : JSON.stringify(lastMessage.content); + + // Collect all tool calls + const toolCalls: any[] = []; + result.messages.forEach((msg: any) => { + if (msg.tool_calls && Array.isArray(msg.tool_calls)) { + toolCalls.push(...msg.tool_calls); + } + }); + + // Serialize messages for metadata (with tool calls support) + const serializedMessages = result.messages.map((msg: any) => ({ + type: msg.constructor?.name || 'Message', + content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), + tool_calls: msg.tool_calls || undefined, + additional_kwargs: msg.additional_kwargs || {}, + response_metadata: msg.response_metadata || {}, + })); + + const result_obj = { + response: responseContent, + messages: serializedMessages, + toolCalls, + }; + + if (detailed) { + return result_obj; + } + + return responseContent; + } + + getConfig(): LLMConfig { + return { ...this.config }; + } + + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + getBaseUrl(): string { + return this.config.baseUrl || 'http://localhost:11434'; + } + + setBaseUrl(baseUrl: string): void { + this.config.baseUrl = baseUrl; + } + + async checkStatus(): Promise { + try { + // Check if Ollama is accessible by making a simple request + const baseUrl = this.config.baseUrl || 'http://localhost:11434'; + await axios.get(`${baseUrl}/api/tags`); + return true; + } catch (error) { + return false; + } + } +} diff --git a/editor/src/assistant/tools/camera-tool.ts b/editor/src/assistant/tools/camera-tool.ts new file mode 100644 index 000000000..34a20fb4b --- /dev/null +++ b/editor/src/assistant/tools/camera-tool.ts @@ -0,0 +1,178 @@ +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { Scene, ArcRotateCamera, FreeCamera, Vector3 } from "babylonjs"; + +export function createCameraTools(scene: Scene, onChange?: () => void): DynamicStructuredTool[] { + return [ + new DynamicStructuredTool({ + name: "list_cameras", + description: "Lists all cameras in the scene with their basic properties", + schema: z.object({}), + func: async () => { + const cameras = scene.cameras.map((camera) => ({ + name: camera.name, + id: camera.id, + type: camera.getClassName(), + isActive: scene.activeCamera === camera, + })); + return JSON.stringify({ cameras, count: cameras.length }); + }, + }), + + new DynamicStructuredTool({ + name: "get_camera_properties", + description: "Gets detailed properties of a specific camera by name or id", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the camera"), + }), + func: async ({ nameOrId }) => { + const camera = scene.getCameraByName(nameOrId) || scene.getCameraById(nameOrId); + if (!camera) { + return JSON.stringify({ error: `Camera "${nameOrId}" not found` }); + } + + const baseProps: any = { + name: camera.name, + id: camera.id, + type: camera.getClassName(), + isActive: scene.activeCamera === camera, + fov: camera.fov, + minZ: camera.minZ, + maxZ: camera.maxZ, + }; + + if (camera instanceof ArcRotateCamera) { + baseProps.alpha = camera.alpha; + baseProps.beta = camera.beta; + baseProps.radius = camera.radius; + baseProps.target = { + x: camera.target.x, + y: camera.target.y, + z: camera.target.z, + }; + } + + if (camera instanceof FreeCamera) { + baseProps.position = { + x: camera.position.x, + y: camera.position.y, + z: camera.position.z, + }; + } + + return JSON.stringify(baseProps); + }, + }), + + new DynamicStructuredTool({ + name: "set_camera_fov", + description: "Sets the field of view (FOV) of a camera in radians", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the camera"), + fov: z.number().min(0).max(Math.PI).describe("Field of view in radians"), + }), + func: async ({ nameOrId, fov }) => { + const camera = scene.getCameraByName(nameOrId) || scene.getCameraById(nameOrId); + if (!camera) { + return JSON.stringify({ error: `Camera "${nameOrId}" not found` }); + } + + camera.fov = fov; + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `FOV of camera "${camera.name}" set to ${fov} radians`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_arc_rotate_camera_position", + description: "Sets the position parameters (alpha, beta, radius) of an ArcRotateCamera", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the ArcRotateCamera"), + alpha: z.number().optional().describe("Horizontal rotation angle in radians"), + beta: z.number().optional().describe("Vertical rotation angle in radians"), + radius: z.number().optional().describe("Distance from target"), + }), + func: async ({ nameOrId, alpha, beta, radius }) => { + const camera = scene.getCameraByName(nameOrId) || scene.getCameraById(nameOrId); + if (!camera) { + return JSON.stringify({ error: `Camera "${nameOrId}" not found` }); + } + + if (!(camera instanceof ArcRotateCamera)) { + return JSON.stringify({ error: `Camera "${camera.name}" is not an ArcRotateCamera` }); + } + + if (alpha !== undefined) camera.alpha = alpha; + if (beta !== undefined) camera.beta = beta; + if (radius !== undefined) camera.radius = radius; + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `ArcRotateCamera "${camera.name}" position updated`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_arc_rotate_camera_target", + description: "Sets the target position that an ArcRotateCamera looks at", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the ArcRotateCamera"), + x: z.number().describe("X coordinate of target"), + y: z.number().describe("Y coordinate of target"), + z: z.number().describe("Z coordinate of target"), + }), + func: async ({ nameOrId, x, y, z }) => { + const camera = scene.getCameraByName(nameOrId) || scene.getCameraById(nameOrId); + if (!camera) { + return JSON.stringify({ error: `Camera "${nameOrId}" not found` }); + } + + if (!(camera instanceof ArcRotateCamera)) { + return JSON.stringify({ error: `Camera "${camera.name}" is not an ArcRotateCamera` }); + } + + camera.target.set(x, y, z); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `ArcRotateCamera "${camera.name}" target set to (${x}, ${y}, ${z})`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_free_camera_position", + description: "Sets the position of a FreeCamera", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the FreeCamera"), + x: z.number().describe("X coordinate"), + y: z.number().describe("Y coordinate"), + z: z.number().describe("Z coordinate"), + }), + func: async ({ nameOrId, x, y, z }) => { + const camera = scene.getCameraByName(nameOrId) || scene.getCameraById(nameOrId); + if (!camera) { + return JSON.stringify({ error: `Camera "${nameOrId}" not found` }); + } + + if (!(camera instanceof FreeCamera)) { + return JSON.stringify({ error: `Camera "${camera.name}" is not a FreeCamera` }); + } + + camera.position = new Vector3(x, y, z); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `FreeCamera "${camera.name}" position set to (${x}, ${y}, ${z})`, + }); + }, + }), + ]; +} + diff --git a/editor/src/assistant/tools/light-tool.ts b/editor/src/assistant/tools/light-tool.ts new file mode 100644 index 000000000..5d6e4286d --- /dev/null +++ b/editor/src/assistant/tools/light-tool.ts @@ -0,0 +1,251 @@ +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { Scene, PointLight, DirectionalLight, SpotLight, HemisphericLight, Color3, Vector3 } from "babylonjs"; + +export function createLightTools(scene: Scene, onChange?: () => void): DynamicStructuredTool[] { + return [ + new DynamicStructuredTool({ + name: "list_lights", + description: "Lists all lights in the scene with their basic properties", + schema: z.object({}), + func: async () => { + const lights = scene.lights.map((light) => ({ + name: light.name, + id: light.id, + type: light.getClassName(), + isEnabled: light.isEnabled(), + intensity: light.intensity, + })); + return JSON.stringify({ lights, count: lights.length }); + }, + }), + + new DynamicStructuredTool({ + name: "get_light_properties", + description: "Gets detailed properties of a specific light by name or id", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light"), + }), + func: async ({ nameOrId }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + const baseProps: any = { + name: light.name, + id: light.id, + type: light.getClassName(), + isEnabled: light.isEnabled(), + intensity: light.intensity, + }; + + if (light instanceof PointLight || light instanceof DirectionalLight || light instanceof SpotLight) { + baseProps.diffuse = light.diffuse + ? { r: light.diffuse.r, g: light.diffuse.g, b: light.diffuse.b } + : null; + baseProps.specular = light.specular + ? { r: light.specular.r, g: light.specular.g, b: light.specular.b } + : null; + } + + if (light instanceof PointLight || light instanceof SpotLight) { + baseProps.position = { + x: light.position.x, + y: light.position.y, + z: light.position.z, + }; + } + + if (light instanceof DirectionalLight || light instanceof SpotLight) { + baseProps.direction = { + x: light.direction.x, + y: light.direction.y, + z: light.direction.z, + }; + } + + return JSON.stringify(baseProps); + }, + }), + + new DynamicStructuredTool({ + name: "set_light_intensity", + description: "Sets the intensity of a light", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light"), + intensity: z.number().min(0).describe("Light intensity"), + }), + func: async ({ nameOrId, intensity }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + light.intensity = intensity; + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Intensity of light "${light.name}" set to ${intensity}`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_light_color", + description: "Sets the diffuse color of a light", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light"), + r: z.number().min(0).max(1).describe("Red component (0-1)"), + g: z.number().min(0).max(1).describe("Green component (0-1)"), + b: z.number().min(0).max(1).describe("Blue component (0-1)"), + }), + func: async ({ nameOrId, r, g, b }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + if (light instanceof PointLight || light instanceof DirectionalLight || light instanceof SpotLight) { + light.diffuse = new Color3(r, g, b); + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Color of light "${light.name}" set to RGB(${r}, ${g}, ${b})`, + }); + } + + return JSON.stringify({ + error: `Light "${light.name}" does not support color modification`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_light_position", + description: "Sets the position of a point light or spot light", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light"), + x: z.number().describe("X coordinate"), + y: z.number().describe("Y coordinate"), + z: z.number().describe("Z coordinate"), + }), + func: async ({ nameOrId, x, y, z }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + if (light instanceof PointLight || light instanceof SpotLight) { + light.position = new Vector3(x, y, z); + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Position of light "${light.name}" set to (${x}, ${y}, ${z})`, + }); + } + + return JSON.stringify({ + error: `Light "${light.name}" does not support position modification`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "enable_disable_light", + description: "Enables or disables a light", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light"), + enabled: z.boolean().describe("Whether the light should be enabled"), + }), + func: async ({ nameOrId, enabled }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + light.setEnabled(enabled); + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Light "${light.name}" ${enabled ? "enabled" : "disabled"}`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "create_light", + description: "Creates a new light in the scene (point, directional, spot, or hemispheric)", + schema: z.object({ + type: z.enum(["point", "directional", "spot", "hemispheric"]).describe("Type of light to create"), + name: z.string().describe("Name for the new light"), + x: z.number().optional().describe("X position/direction"), + y: z.number().optional().describe("Y position/direction"), + z: z.number().optional().describe("Z position/direction"), + intensity: z.number().optional().describe("Light intensity (default: 1)"), + }), + func: async ({ type, name, x, y, z, intensity }) => { + const vector = new Vector3(x || 0, y || 1, z || 0); + const lightIntensity = intensity || 1; + let light; + + switch (type) { + case "point": + light = new PointLight(name, vector, scene); + break; + case "directional": + light = new DirectionalLight(name, vector, scene); + break; + case "spot": + light = new SpotLight(name, vector, new Vector3(0, -1, 0), Math.PI / 3, 2, scene); + break; + case "hemispheric": + light = new HemisphericLight(name, vector, scene); + break; + default: + return JSON.stringify({ error: `Unknown light type: ${type}` }); + } + + light.intensity = lightIntensity; + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `Created ${type} light "${name}" with intensity ${lightIntensity}`, + lightId: light.id, + }); + }, + }), + + new DynamicStructuredTool({ + name: "delete_light", + description: "Deletes a light from the scene", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the light to delete"), + }), + func: async ({ nameOrId }) => { + const light = scene.getLightByName(nameOrId) || scene.getLightById(nameOrId); + if (!light) { + return JSON.stringify({ error: `Light "${nameOrId}" not found` }); + } + + const lightName = light.name; + light.dispose(); + scene.markAllMaterialsAsDirty(2); + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `Deleted light "${lightName}"`, + }); + }, + }), + ]; +} + diff --git a/editor/src/assistant/tools/material-tool.ts b/editor/src/assistant/tools/material-tool.ts new file mode 100644 index 000000000..3b0fa332f --- /dev/null +++ b/editor/src/assistant/tools/material-tool.ts @@ -0,0 +1,173 @@ +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { Scene, PBRMaterial, StandardMaterial, Color3 } from "babylonjs"; + +export function createMaterialTools(scene: Scene, onChange?: () => void): DynamicStructuredTool[] { + return [ + new DynamicStructuredTool({ + name: "list_materials", + description: "Lists all materials in the scene with their basic properties", + schema: z.object({}), + func: async () => { + const materials = scene.materials.map((material) => ({ + name: material.name, + id: material.id, + type: material.getClassName(), + alpha: material.alpha, + })); + return JSON.stringify({ materials, count: materials.length }); + }, + }), + + new DynamicStructuredTool({ + name: "get_material_properties", + description: "Gets detailed properties of a specific material by name or id", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the material"), + }), + func: async ({ nameOrId }) => { + const material = scene.getMaterialByName(nameOrId) || scene.getMaterialById(nameOrId); + if (!material) { + return JSON.stringify({ error: `Material "${nameOrId}" not found` }); + } + + const baseProps: any = { + name: material.name, + id: material.id, + type: material.getClassName(), + alpha: material.alpha, + backFaceCulling: material.backFaceCulling, + }; + + if (material instanceof PBRMaterial) { + baseProps.metallic = material.metallic; + baseProps.roughness = material.roughness; + baseProps.albedoColor = material.albedoColor + ? { r: material.albedoColor.r, g: material.albedoColor.g, b: material.albedoColor.b } + : null; + } else if (material instanceof StandardMaterial) { + baseProps.diffuseColor = material.diffuseColor + ? { r: material.diffuseColor.r, g: material.diffuseColor.g, b: material.diffuseColor.b } + : null; + baseProps.specularColor = material.specularColor + ? { r: material.specularColor.r, g: material.specularColor.g, b: material.specularColor.b } + : null; + } + + return JSON.stringify(baseProps); + }, + }), + + new DynamicStructuredTool({ + name: "set_material_alpha", + description: "Sets the alpha (transparency) of a material", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the material"), + alpha: z.number().min(0).max(1).describe("Alpha value (0 = transparent, 1 = opaque)"), + }), + func: async ({ nameOrId, alpha }) => { + const material = scene.getMaterialByName(nameOrId) || scene.getMaterialById(nameOrId); + if (!material) { + return JSON.stringify({ error: `Material "${nameOrId}" not found` }); + } + + material.alpha = alpha; + material.markAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Alpha of material "${material.name}" set to ${alpha}`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_pbr_material_color", + description: "Sets the albedo color of a PBR material", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the PBR material"), + r: z.number().min(0).max(1).describe("Red component (0-1)"), + g: z.number().min(0).max(1).describe("Green component (0-1)"), + b: z.number().min(0).max(1).describe("Blue component (0-1)"), + }), + func: async ({ nameOrId, r, g, b }) => { + const material = scene.getMaterialByName(nameOrId) || scene.getMaterialById(nameOrId); + if (!material) { + return JSON.stringify({ error: `Material "${nameOrId}" not found` }); + } + + if (!(material instanceof PBRMaterial)) { + return JSON.stringify({ error: `Material "${material.name}" is not a PBR material` }); + } + + material.albedoColor = new Color3(r, g, b); + material.markAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Albedo color of material "${material.name}" set to RGB(${r}, ${g}, ${b})`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_pbr_material_metallic_roughness", + description: "Sets the metallic and roughness values of a PBR material", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the PBR material"), + metallic: z.number().min(0).max(1).describe("Metallic value (0-1)"), + roughness: z.number().min(0).max(1).describe("Roughness value (0-1)"), + }), + func: async ({ nameOrId, metallic, roughness }) => { + const material = scene.getMaterialByName(nameOrId) || scene.getMaterialById(nameOrId); + if (!material) { + return JSON.stringify({ error: `Material "${nameOrId}" not found` }); + } + + if (!(material instanceof PBRMaterial)) { + return JSON.stringify({ error: `Material "${material.name}" is not a PBR material` }); + } + + material.metallic = metallic; + material.roughness = roughness; + material.markAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Metallic and roughness of material "${material.name}" set to ${metallic} and ${roughness}`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_standard_material_color", + description: "Sets the diffuse color of a Standard material", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the Standard material"), + r: z.number().min(0).max(1).describe("Red component (0-1)"), + g: z.number().min(0).max(1).describe("Green component (0-1)"), + b: z.number().min(0).max(1).describe("Blue component (0-1)"), + }), + func: async ({ nameOrId, r, g, b }) => { + const material = scene.getMaterialByName(nameOrId) || scene.getMaterialById(nameOrId); + if (!material) { + return JSON.stringify({ error: `Material "${nameOrId}" not found` }); + } + + if (!(material instanceof StandardMaterial)) { + return JSON.stringify({ error: `Material "${material.name}" is not a Standard material` }); + } + + material.diffuseColor = new Color3(r, g, b); + material.markAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Diffuse color of material "${material.name}" set to RGB(${r}, ${g}, ${b})`, + }); + }, + }), + ]; +} + diff --git a/editor/src/assistant/tools/mesh-tool.ts b/editor/src/assistant/tools/mesh-tool.ts new file mode 100644 index 000000000..61dae3620 --- /dev/null +++ b/editor/src/assistant/tools/mesh-tool.ts @@ -0,0 +1,274 @@ +// @ts-ignore - ES module imports in CommonJS context +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { Scene, Vector3, MeshBuilder, StandardMaterial, Color3 } from "babylonjs"; + +export function createMeshTools(scene: Scene, onChange?: () => void): DynamicStructuredTool[] { + return [ + new DynamicStructuredTool({ + name: "list_meshes", + description: "Lists all meshes in the scene with their basic properties", + schema: z.object({}), + func: async () => { + const meshes = scene.meshes.map((mesh) => ({ + name: mesh.name, + id: mesh.id, + isVisible: mesh.isVisible, + isEnabled: mesh.isEnabled(), + position: { + x: mesh.position.x, + y: mesh.position.y, + z: mesh.position.z, + }, + scaling: { + x: mesh.scaling.x, + y: mesh.scaling.y, + z: mesh.scaling.z, + }, + })); + return JSON.stringify({ meshes, count: meshes.length }); + }, + }), + + new DynamicStructuredTool({ + name: "get_mesh_properties", + description: "Gets detailed properties of a specific mesh by name or id", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh"), + }), + func: async ({ nameOrId }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + return JSON.stringify({ + name: mesh.name, + id: mesh.id, + isVisible: mesh.isVisible, + isEnabled: mesh.isEnabled(), + position: { + x: mesh.position.x, + y: mesh.position.y, + z: mesh.position.z, + }, + rotation: { + x: mesh.rotation.x, + y: mesh.rotation.y, + z: mesh.rotation.z, + }, + scaling: { + x: mesh.scaling.x, + y: mesh.scaling.y, + z: mesh.scaling.z, + }, + material: mesh.material?.name || "none", + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_mesh_position", + description: "Sets the position of a mesh", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh"), + x: z.number().describe("X coordinate"), + y: z.number().describe("Y coordinate"), + z: z.number().describe("Z coordinate"), + }), + func: async ({ nameOrId, x, y, z }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + mesh.position = new Vector3(x, y, z); + mesh.computeWorldMatrix(true); + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Position of mesh "${mesh.name}" set to (${x}, ${y}, ${z})`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_mesh_scaling", + description: "Sets the scaling of a mesh", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh"), + x: z.number().describe("X scale factor"), + y: z.number().describe("Y scale factor"), + z: z.number().describe("Z scale factor"), + }), + func: async ({ nameOrId, x, y, z }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + mesh.scaling = new Vector3(x, y, z); + mesh.computeWorldMatrix(true); + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Scaling of mesh "${mesh.name}" set to (${x}, ${y}, ${z})`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_mesh_rotation", + description: "Sets the rotation of a mesh in radians", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh"), + x: z.number().describe("X rotation in radians"), + y: z.number().describe("Y rotation in radians"), + z: z.number().describe("Z rotation in radians"), + }), + func: async ({ nameOrId, x, y, z }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + mesh.rotation = new Vector3(x, y, z); + mesh.computeWorldMatrix(true); + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Rotation of mesh "${mesh.name}" set to (${x}, ${y}, ${z}) radians`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "set_mesh_visibility", + description: "Sets the visibility of a mesh", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh"), + visible: z.boolean().describe("Whether the mesh should be visible"), + }), + func: async ({ nameOrId, visible }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + mesh.isVisible = visible; + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + return JSON.stringify({ + success: true, + message: `Visibility of mesh "${mesh.name}" set to ${visible}`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "create_mesh", + description: "Creates a new mesh in the scene (box, sphere, cylinder, plane, torus, or ground)", + schema: z.object({ + type: z.enum(["box", "sphere", "cylinder", "plane", "torus", "ground"]).describe("Type of mesh to create"), + name: z.string().describe("Name for the new mesh"), + x: z.number().optional().describe("X position (default: 0)"), + y: z.number().optional().describe("Y position (default: 0)"), + z: z.number().optional().describe("Z position (default: 0)"), + }), + func: async ({ type, name, x, y, z }) => { + const position = new Vector3(x || 0, y || 0, z || 0); + let mesh; + + switch (type) { + case "box": + mesh = MeshBuilder.CreateBox(name, { size: 1 }, scene); + break; + case "sphere": + mesh = MeshBuilder.CreateSphere(name, { diameter: 1 }, scene); + break; + case "cylinder": + mesh = MeshBuilder.CreateCylinder(name, { height: 2, diameter: 1 }, scene); + break; + case "plane": + mesh = MeshBuilder.CreatePlane(name, { size: 1 }, scene); + break; + case "torus": + mesh = MeshBuilder.CreateTorus(name, { diameter: 1, thickness: 0.3 }, scene); + break; + case "ground": + mesh = MeshBuilder.CreateGround(name, { width: 10, height: 10 }, scene); + break; + default: + return JSON.stringify({ error: `Unknown mesh type: ${type}` }); + } + + mesh.position = position; + + const material = new StandardMaterial(name + "_material", scene); + material.diffuseColor = new Color3(0.7, 0.7, 0.7); + mesh.material = material; + + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `Created ${type} mesh "${name}" at position (${x || 0}, ${y || 0}, ${z || 0})`, + meshId: mesh.id, + }); + }, + }), + + new DynamicStructuredTool({ + name: "rename_mesh", + description: "Renames a mesh in the scene", + schema: z.object({ + nameOrId: z.string().describe("The current name or id of the mesh"), + newName: z.string().describe("The new name for the mesh"), + }), + func: async ({ nameOrId, newName }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + const oldName = mesh.name; + mesh.name = newName; + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `Renamed mesh from "${oldName}" to "${newName}"`, + }); + }, + }), + + new DynamicStructuredTool({ + name: "delete_mesh", + description: "Deletes a mesh from the scene", + schema: z.object({ + nameOrId: z.string().describe("The name or id of the mesh to delete"), + }), + func: async ({ nameOrId }) => { + const mesh = scene.getMeshByName(nameOrId) || scene.getMeshById(nameOrId); + if (!mesh) { + return JSON.stringify({ error: `Mesh "${nameOrId}" not found` }); + } + + const meshName = mesh.name; + mesh.dispose(); + scene.markAllMaterialsAsDirty(1); + if (onChange) onChange(); + + return JSON.stringify({ + success: true, + message: `Deleted mesh "${meshName}"`, + }); + }, + }), + ]; +} + diff --git a/editor/src/editor/layout/toolbar.tsx b/editor/src/editor/layout/toolbar.tsx index 7e23c7a7a..beb0847c2 100644 --- a/editor/src/editor/layout/toolbar.tsx +++ b/editor/src/editor/layout/toolbar.tsx @@ -201,6 +201,19 @@ export class EditorToolbar extends Component { + {/* Tools */} + + Tools + + { + await this.props.editor.loadAIAssistant(); + this.props.editor.setState({ showAIAssistant: true }); + }}> + AI Assistant... + + + + {/* Window */} Window diff --git a/editor/src/editor/main.tsx b/editor/src/editor/main.tsx index 32d478553..f0b6978fe 100644 --- a/editor/src/editor/main.tsx +++ b/editor/src/editor/main.tsx @@ -118,6 +118,14 @@ export interface IEditorState { * Defines wether or not Visual Studio Code is available. */ visualStudioCodeAvailable: boolean; + /** + * Defines whether the AI Assistant is open. + */ + showAIAssistant: boolean; + /** + * The AI Assistant component (lazy loaded) + */ + AIAssistantComponent?: any; } export class Editor extends Component { @@ -153,11 +161,24 @@ export class Editor extends Component { nodeJSAvailable: false, visualStudioCodeAvailable: false, + showAIAssistant: false, + AIAssistantComponent: null, }; webFrame.setZoomFactor(0.8); } + public async loadAIAssistant(): Promise { + if (!this.state.AIAssistantComponent) { + try { + const module = await import("../assistant/ai-assistant-ui"); + this.setState({ AIAssistantComponent: module.AIAssistantUI }); + } catch (error) { + console.error("Failed to load AI Assistant:", error); + } + } + } + public render(): ReactNode { return ( <> @@ -193,6 +214,10 @@ export class Editor extends Component { this.setState({ editPreferences: false })} /> + {this.state.showAIAssistant && this.state.AIAssistantComponent && ( + this.setState({ showAIAssistant: false })} /> + )} + (this.commandPalette = r!)} editor={this} /> diff --git a/package.json b/package.json index 739972eaa..4ed9deb47 100644 --- a/package.json +++ b/package.json @@ -61,10 +61,15 @@ "dotenv": "16.4.5", "eslint": "9.29.0", "minimist": "1.2.8", - "rimraf": "6.0.1", - "prettier": "3.6.2" + "prettier": "3.6.2", + "rimraf": "6.0.1" + }, + "dependencies": { + "@langchain/core": "^1.0.5", + "@langchain/langgraph": "^1.0.2", + "@langchain/ollama": "^1.0.1", + "zod": "^4.1.12" }, - "dependencies": {}, "resolutions": { "braces": "3.0.3", "node-abi": "4.14.0", diff --git a/yarn.lock b/yarn.lock index bb13e632b..ae1a56df9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1201,6 +1201,11 @@ classnames "^2.3.1" tslib "~2.6.2" +"@cfworker/json-schema@^4.0.2": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@cfworker/json-schema/-/json-schema-4.1.1.tgz#4a2a3947ee9fa7b7c24be981422831b8674c3be6" + integrity sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og== + "@corex/deepmerge@^4.0.43": version "4.0.43" resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-4.0.43.tgz#9bd42559ebb41cc5a7fb7cfeea5f231c20977dca" @@ -2372,6 +2377,56 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@langchain/core@^1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@langchain/core/-/core-1.0.5.tgz#1e20ecce80fa4d0b979ea05b24b879b8357d8092" + integrity sha512-9Hy/b9+j+mm0Bhnm8xD9B0KpBYTidroLrDHdbrHoMC2DqXoY2umvi1M3M/9D744qsMSaIMP0ZwFcy5YbqI/dGw== + dependencies: + "@cfworker/json-schema" "^4.0.2" + ansi-styles "^5.0.0" + camelcase "6" + decamelize "1.2.0" + js-tiktoken "^1.0.12" + langsmith "^0.3.64" + mustache "^4.2.0" + p-queue "^6.6.2" + p-retry "4" + uuid "^10.0.0" + zod "^3.25.76 || ^4" + +"@langchain/langgraph-checkpoint@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.0.0.tgz#ece2ede439d0d0b0b532c4be7817fd5029afe4f8" + integrity sha512-xrclBGvNCXDmi0Nz28t3vjpxSH6UYx6w5XAXSiiB1WEdc2xD2iY/a913I3x3a31XpInUW/GGfXXfePfaghV54A== + dependencies: + uuid "^10.0.0" + +"@langchain/langgraph-sdk@~1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@langchain/langgraph-sdk/-/langgraph-sdk-1.0.0.tgz#16faca6cc426432dee9316428d0aecd94e5b7989" + integrity sha512-g25ti2W7Dl5wUPlNK+0uIGbeNFqf98imhHlbdVVKTTkDYLhi/pI1KTgsSSkzkeLuBIfvt2b0q6anQwCs7XBlbw== + dependencies: + p-queue "^6.6.2" + p-retry "4" + uuid "^9.0.0" + +"@langchain/langgraph@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@langchain/langgraph/-/langgraph-1.0.2.tgz#62de931edac0dd850daf708bd6f8f3835cf25a5e" + integrity sha512-syxzzWTnmpCL+RhUEvalUeOXFoZy/KkzHa2Da2gKf18zsf9Dkbh3rfnRDrTyUGS1XSTejq07s4rg1qntdEDs2A== + dependencies: + "@langchain/langgraph-checkpoint" "^1.0.0" + "@langchain/langgraph-sdk" "~1.0.0" + uuid "^10.0.0" + +"@langchain/ollama@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@langchain/ollama/-/ollama-1.0.1.tgz#c63ac6db65110beef4020a5e2b167ad0bc678d33" + integrity sha512-Pe32hhTpMvnRlNFJxkdu6r1QzsONGz5uvoLiMU1TpgAUu7EyKr2osymlgjBLqDe2vMKUmqHb+yWRH0IppDBUOg== + dependencies: + ollama "^0.5.12" + uuid "^10.0.0" + "@malept/cross-spawn-promise@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz#d0772de1aa680a0bfb9ba2f32b4c828c7857cb9d" @@ -3931,6 +3986,11 @@ dependencies: "@types/node" "*" +"@types/retry@0.12.0": + version "0.12.0" + resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d" + integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== + "@types/scheduler@*": version "0.16.8" resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" @@ -3951,6 +4011,11 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-3.0.3.tgz#acaab0f919ce69cce629c2d4ed2eb4adc1b6c20c" integrity sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q== +"@types/uuid@^10.0.0": + version "10.0.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d" + integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ== + "@types/verror@^1.10.3": version "1.10.9" resolved "https://registry.yarnpkg.com/@types/verror/-/verror-1.10.9.tgz#420c32adb9a2dd50b3db4c8f96501e05a0e72941" @@ -4749,10 +4814,10 @@ babylonjs-editor-tools@latest: integrity sha512-AREjL0WjtjyOvud0EMG/II3zH73KlSif/u0HV965tPWmUZHrxr+g/4iX6eU0mIYlIjOuepfRAopaF04IYJOaHA== "babylonjs-editor-tools@link:tools": - version "5.1.10" + version "5.2.1" "babylonjs-editor@link:editor": - version "5.1.10" + version "5.2.1" dependencies: "@babylonjs/core" "8.33.2" "@babylonjs/havok" "1.3.10" @@ -5142,16 +5207,16 @@ camelcase-css@^2.0.1: resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5" integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA== +camelcase@6, camelcase@^6.2.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== + camelcase@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== -camelcase@^6.2.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" - integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== - camelize@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/camelize/-/camelize-1.0.1.tgz#89b7e16884056331a35d6b5ad064332c91daa6c3" @@ -5495,6 +5560,13 @@ config-file-ts@0.2.8-rc1: glob "^10.3.12" typescript "^5.4.3" +console-table-printer@^2.12.1: + version "2.15.0" + resolved "https://registry.yarnpkg.com/console-table-printer/-/console-table-printer-2.15.0.tgz#5c808204640b8f024d545bde8aabe5d344dfadc1" + integrity sha512-SrhBq4hYVjLCkBVOWaTzceJalvn5K1Zq5aQA6wXC/cYjI3frKWNPEMK3sZsJfNNQApvCQmgBcc13ZKmFj8qExw== + dependencies: + simple-wcswidth "^1.1.2" + constant-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/constant-case/-/constant-case-3.0.4.tgz#3b84a9aeaf4cf31ec45e6bf5de91bdfb0589faf1" @@ -5659,6 +5731,11 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +decamelize@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== + decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -6569,6 +6646,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +eventemitter3@^4.0.4: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -8246,6 +8328,13 @@ jiti@^2.4.2: resolved "https://registry.yarnpkg.com/jiti/-/jiti-2.4.2.tgz#d19b7732ebb6116b06e2038da74a55366faef560" integrity sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A== +js-tiktoken@^1.0.12: + version "1.0.21" + resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.21.tgz#368a9957591a30a62997dd0c4cf30866f00f8221" + integrity sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g== + dependencies: + base64-js "^1.5.1" + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -8356,6 +8445,19 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== +langsmith@^0.3.64: + version "0.3.79" + resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.3.79.tgz#6c845644da26e7fdd8e9b80706091669fc43bda4" + integrity sha512-j5uiAsyy90zxlxaMuGjb7EdcL51Yx61SpKfDOI1nMPBbemGju+lf47he4e59Hp5K63CY8XWgFP42WeZ+zuIU4Q== + dependencies: + "@types/uuid" "^10.0.0" + chalk "^4.1.2" + console-table-printer "^2.12.1" + p-queue "^6.6.2" + p-retry "4" + semver "^7.6.3" + uuid "^10.0.0" + language-subtag-registry@^0.3.20: version "0.3.22" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz#2e1500861b2e457eba7e7ae86877cbd08fa1fd1d" @@ -8952,6 +9054,11 @@ ms@^2.0.0, ms@^2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +mustache@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64" + integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== + mz@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32" @@ -9187,6 +9294,13 @@ object.values@^1.1.6, object.values@^1.1.7: define-properties "^1.2.0" es-abstract "^1.22.1" +ollama@^0.5.12: + version "0.5.18" + resolved "https://registry.yarnpkg.com/ollama/-/ollama-0.5.18.tgz#10b8ee9e5cd840f2003b7bbea1802dd772e6a564" + integrity sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg== + dependencies: + whatwg-fetch "^3.6.20" + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -9247,6 +9361,11 @@ p-cancelable@^2.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -9282,6 +9401,29 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +p-queue@^6.6.2: + version "6.6.2" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" + integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== + dependencies: + eventemitter3 "^4.0.4" + p-timeout "^3.2.0" + +p-retry@4: + version "4.6.2" + resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" + integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ== + dependencies: + "@types/retry" "0.12.0" + retry "^0.13.1" + +p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -10031,6 +10173,11 @@ retry@^0.12.0: resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" integrity sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow== +retry@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" + integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== + reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" @@ -10353,6 +10500,11 @@ simple-update-notifier@2.0.0: dependencies: semver "^7.5.3" +simple-wcswidth@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz#66722f37629d5203f9b47c5477b1225b85d6525b" + integrity sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw== + sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" @@ -11308,6 +11460,16 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== +uuid@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" + integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== + +uuid@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + v8-to-istanbul@^9.0.1: version "9.3.0" resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz#b9572abfa62bd556c16d75fdebc1a411d5ff3175" @@ -11407,6 +11569,11 @@ webm-muxer@^5.0.2: "@types/dom-webcodecs" "^0.1.4" "@types/wicg-file-system-access" "^2020.9.5" +whatwg-fetch@^3.6.20: + version "3.6.20" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz#580ce6d791facec91d37c72890995a0b48d31c70" + integrity sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg== + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -11573,6 +11740,11 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +"zod@^3.25.76 || ^4", zod@^4.1.12: + version "4.1.12" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.1.12.tgz#64f1ea53d00eab91853195653b5af9eee68970f0" + integrity sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ== + zwitch@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7"