diff --git a/examples/nextjs/pages/simple.tsx b/examples/nextjs/pages/simple.tsx index c099519d8..806db2cbc 100644 --- a/examples/nextjs/pages/simple.tsx +++ b/examples/nextjs/pages/simple.tsx @@ -13,6 +13,8 @@ import { useTracks, SessionEvent, useEvents, + useRpc, + rpc, } from '@livekit/components-react'; import { Track, TokenSource, MediaDeviceFailure } from 'livekit-client'; import type { NextPage } from 'next'; @@ -71,6 +73,23 @@ const SimpleExample: NextPage = () => { ); }, []); + const { performRpc } = useRpc(session, { + getUserLocation: rpc.json(async (payload: { highAccuracy: boolean }, data) => { + const position = await new Promise((resolve, reject) => { + navigator.geolocation.getCurrentPosition(resolve, reject, { + enableHighAccuracy: payload.highAccuracy, + timeout: data.responseTimeout * 1000, + }); + }); + return { + latitude: position.coords.latitude, + longitude: position.coords.longitude, + }; + }), + }); + + const [participantIdentity, setParticipantIdentity] = useState(''); + return (
@@ -82,6 +101,19 @@ const SimpleExample: NextPage = () => { {connect ? 'Disconnect' : 'Connect'} )} + + setParticipantIdentity(e.target.value)} /> + + diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index c47c201af..0fa651934 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -70,3 +70,20 @@ export { } from './useAgent'; export * from './useEvents'; export * from './useSessionMessages'; +export { + type RpcRawHandler, + type RpcMethodDescriptor, + type RpcMethod, + type PerformRpcDescriptor, + type RpcJsonParams, + type UseRpcOptions, + type PerformRpcFn, + type UseRpcReturn, + rpc, + useRpc, +} from './useRpc'; +export { + type SchemaLike, + type ClientToolDefinition, + useClientTools, +} from './useClientTools'; diff --git a/packages/react/src/hooks/useAgent.ts b/packages/react/src/hooks/useAgent.ts index 4767bac9f..5fea067f2 100644 --- a/packages/react/src/hooks/useAgent.ts +++ b/packages/react/src/hooks/useAgent.ts @@ -72,6 +72,8 @@ type AgentStateCommon = { agentParticipant: RemoteParticipant | null; workerParticipant: RemoteParticipant | null; + + session: UseSessionReturn; }; }; @@ -741,6 +743,7 @@ export function useAgent(session?: SessionStub): UseAgentReturn { agentParticipant, workerParticipant, emitter, + session: session! as UseSessionReturn, }, }; @@ -848,6 +851,7 @@ export function useAgent(session?: SessionStub): UseAgentReturn { agentParticipantAttributes, emitter, agentParticipant, + session, state, videoTrack, audioTrack, diff --git a/packages/react/src/hooks/useClientTools.ts b/packages/react/src/hooks/useClientTools.ts new file mode 100644 index 000000000..78b3f581c --- /dev/null +++ b/packages/react/src/hooks/useClientTools.ts @@ -0,0 +1,156 @@ +import * as React from 'react'; +import { type RpcInvocationData } from 'livekit-client'; + +import type { UseAgentReturn } from './useAgent'; +import { useRpc, type RpcMethod } from './useRpc'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** + * Any object with a `parse` method that validates and returns typed data, and a `toJSONSchema` + * method that returns a JSON Schema representation. + * + * This aligns with zod's interface without depending on it. Zod provides `.parse()` natively + * and `.toJSONSchema()` via `z.toJSONSchema(schema)`. Other schema libraries can also satisfy + * this interface with minimal wrapping. + * + * @beta + */ +export type SchemaLike = { + /** Validate and parse the input. Should throw if validation fails. */ + parse: (input: unknown) => T; + /** + * Return a JSON Schema representation of the parameters. + * Must be `{ type: "object", properties: { ... } }` at the top level, since the agent + * framework maps parameters to Python kwargs / JS named arguments. + */ + toJSONSchema: () => Record; +}; + +/** + * Definition for a single client tool. + * @beta + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type ClientToolDefinition = { + /** Description of the tool, presented to the LLM by the agent framework. */ + description: string; + + /** + * Schema for validating/parsing incoming payloads AND generating JSON Schema for the manifest. + * + * Must satisfy the {@link SchemaLike} interface. Compatible with zod schemas and any other + * library with conforming `parse` and `toJSONSchema` methods. + */ + parameters: SchemaLike; + + /** The function called when the agent invokes this tool. */ + execute: (params: TParams, context: RpcInvocationData) => Promise; +}; + +/** Version of the client tool manifest attribute format. */ +const CLIENT_TOOL_MANIFEST_VERSION = 1; + +/** Prefix for participant attributes that advertise client tools. */ +const CLIENT_TOOL_ATTRIBUTE_PREFIX = 'lk.client_tools.'; + +// --------------------------------------------------------------------------- +// useClientTools hook +// --------------------------------------------------------------------------- + +/** + * Declares tools that an AI agent can call on the frontend. + * + * Each tool's description, parameter schema, and implementation are defined here on the client. + * The hook publishes the tool manifest as participant attributes so the agent framework can + * discover them, and registers RPC handlers so the agent can invoke them. + * + * On the agent side, each tool is referenced by name via `client_tool(name="toolName")`. + * + * @example + * ```tsx + * useClientTools(agent, { + * getUserLocation: { + * description: "Get the user's browser geolocation", + * parameters: z.object({ highAccuracy: z.boolean() }), + * execute: async ({ highAccuracy }) => { + * const pos = await getPosition(highAccuracy); + * return { lat: pos.coords.latitude, lng: pos.coords.longitude }; + * }, + * }, + * }); + * ``` + * + * @beta + */ +export function useClientTools( + agent: UseAgentReturn, + tools: Record, +): void { + const session = agent.internal.session; + const { room } = session; + + // --- Ref for latest tool definitions (same pattern as useRpc) --- + const toolsRef = React.useRef(tools); + toolsRef.current = tools; + + // --- Convert tool definitions into RpcMethods for useRpc --- + const rpcMethods = React.useMemo(() => { + const methods: Record = {}; + for (const [name, tool] of Object.entries(tools)) { + methods[name] = { + parse: (raw: string) => { + const parsed = JSON.parse(raw); + return tool.parameters.parse(parsed); + }, + serialize: (val: unknown) => JSON.stringify(val), + handler: async (payload: unknown, data: RpcInvocationData) => { + return toolsRef.current[name]!.execute(payload, data); + }, + }; + } + return methods; + }, [tools]); + + // --- Register RPC handlers via useRpc, scoped to agent participant --- + useRpc(session, rpcMethods, { from: agent.identity }); + + // --- Publish tool manifest as participant attributes --- + const toolNamesKey = React.useMemo(() => Object.keys(tools).sort().join('\0'), [tools]); + + React.useEffect(() => { + const attributes: Record = {}; + for (const [name, tool] of Object.entries(toolsRef.current)) { + let jsonSchema: Record; + try { + jsonSchema = tool.parameters.toJSONSchema(); + } catch (e) { + throw new Error( + `useClientTools: Failed to generate JSON Schema for tool "${name}". ` + + `Ensure your parameters schema implements toJSONSchema() correctly: ${e}`, + ); + } + + const manifest: Record = { + version: CLIENT_TOOL_MANIFEST_VERSION, + description: tool.description, + parameters: jsonSchema, + }; + + attributes[`${CLIENT_TOOL_ATTRIBUTE_PREFIX}${name}`] = JSON.stringify(manifest); + } + + room.localParticipant.setAttributes(attributes); + + return () => { + // Clear attributes on unmount + const cleared: Record = {}; + for (const name of Object.keys(toolsRef.current)) { + cleared[`${CLIENT_TOOL_ATTRIBUTE_PREFIX}${name}`] = ''; + } + room.localParticipant.setAttributes(cleared); + }; + }, [room, toolNamesKey]); +} diff --git a/packages/react/src/hooks/useRpc.ts b/packages/react/src/hooks/useRpc.ts new file mode 100644 index 000000000..f43570eb0 --- /dev/null +++ b/packages/react/src/hooks/useRpc.ts @@ -0,0 +1,302 @@ +import * as React from 'react'; +import { + type Participant, + RpcError, + type RpcInvocationData, + type PerformRpcParams, +} from 'livekit-client'; + +import { useEnsureSession } from '../context'; +import type { UseSessionReturn } from './useSession'; + +/** @beta */ +export type RpcRawHandler = (data: RpcInvocationData) => Promise; + +/** @beta */ +export type RpcMethodDescriptor = { + parse?: (raw: string) => Input; + serialize?: (val: Output) => string; + handler: (payload: Input, context: RpcInvocationData) => Promise; +}; + +/** @beta */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type RpcMethod = + | RpcRawHandler + | RpcMethodDescriptor; + +/** @beta */ +export type PerformRpcDescriptor = Omit< + PerformRpcParams, + 'payload' +> & { + parse?: (raw: string) => Output; + serialize?: (val: Input) => string; + payload: Input; +}; + +/** @beta */ +export type RpcJsonParams = Omit & { payload: Input }; + +/** Options for {@link useRpc}. + * @beta */ +export type UseRpcOptions = { + /** Only accept RPCs from this participant. Others receive UNSUPPORTED_METHOD. */ + from?: string | Participant; +}; + +/** + * Namespace for RPC helpers. + * + * `rpc.json` can be used in two ways: + * + * **Handler mode** (for registering methods via {@link useRpc}): + * ```ts + * useRpc({ + * myMethod: rpc.json(async (payload: MyInput, ctx) => { + * return { result: 'value' }; + * }), + * }); + * ``` + * + * **Payload mode** (for outbound calls via `performRpc`): + * ```ts + * const result = await performRpc(rpc.json( + * { destinationIdentity: '...', method: 'myMethod', payload: { key: 'value' } }, + * )); + * ``` + * + * @beta + */ +export const rpc = (() => { + /* Overload: handler mode (for useRpc methods record) */ + function json( + handler: (payload: Input, data: RpcInvocationData) => Promise, + ): RpcMethodDescriptor; + /* Overload: payload mode (for performRpc) */ + function json(value: RpcJsonParams): PerformRpcDescriptor; + function json( + handlerOrValue: + | RpcJsonParams + | ((payload: Input, data: RpcInvocationData) => Promise), + ): RpcMethodDescriptor | PerformRpcDescriptor { + if (typeof handlerOrValue === 'function') { + return { + parse: (raw: string) => JSON.parse(raw), + serialize: (val: unknown) => JSON.stringify(val), + handler: handlerOrValue as RpcMethodDescriptor['handler'], + }; + } + + return { + ...handlerOrValue, + parse: (raw: string) => JSON.parse(raw), + serialize: (val: unknown) => JSON.stringify(val), + }; + } + + return { + json, + }; +})(); + +/** @beta */ +export type PerformRpcFn = { + (params: PerformRpcDescriptor): Promise; +}; + +/** @beta */ +export type UseRpcReturn = { + performRpc: PerformRpcFn; +}; + +function isUseSessionReturn(value: unknown): value is UseSessionReturn { + return ( + typeof value === 'object' && + value !== null && + 'room' in value && + 'connectionState' in value && + 'internal' in value + ); +} + +async function resolveHandler( + method: RpcMethod, + data: RpcInvocationData, +): Promise { + if (typeof method === 'function') { + return method(data); + } + + let parsed: Input; + if (method.parse) { + try { + parsed = method.parse(data.payload); + } catch (e) { + throw RpcError.builtIn('APPLICATION_ERROR', `Failed to parse RPC payload: ${e}`); + } + } else { + parsed = data.payload as Input; + } + + const result = await method.handler(parsed, data); + + if (method.serialize) { + try { + return method.serialize(result); + } catch (e) { + throw RpcError.builtIn('APPLICATION_ERROR', `Failed to serialize RPC response: ${e}`); + } + } else if (typeof result !== 'string') { + throw RpcError.builtIn( + 'APPLICATION_ERROR', + `Failed to serialize RPC response: return value from handler function not string. Did you mean to include a "serialize" RpcMethod key?`, + ); + } else { + return result; + } +} + +// --------------------------------------------------------------------------- +// useRpc hook +// --------------------------------------------------------------------------- + +/** + * Hook for declarative RPC method registration and outbound RPC calls. + * + * Registers handler functions for incoming RPC method calls and returns a `performRpc` function + * for making outbound RPC calls to other participants. + * + * Handlers are registered on mount and unregistered on unmount. The effect lifecycle is driven + * by the set of method names — handler function identity does not matter (they are captured by + * ref), so inline functions work without `useCallback`. + * + * @example + * ```tsx + * const { performRpc } = useRpc({ + * // JSON handler via preset + * getUserLocation: rpc.json(async (payload: { highAccuracy: boolean }, ctx) => { + * const pos = await getPosition(payload.highAccuracy); + * return { lat: pos.coords.latitude, lng: pos.coords.longitude }; + * }), + * + * // Raw string handler + * getTimezone: async (data) => Intl.DateTimeFormat().resolvedOptions().timeZone, + * }, { from: session.agent }); + * ``` + * + * @beta + */ +export function useRpc( + session: UseSessionReturn, + methods?: Record, + options?: UseRpcOptions, +): UseRpcReturn; +export function useRpc(methods?: Record, options?: UseRpcOptions): UseRpcReturn; +export function useRpc( + methodsOrSession?: Record | UseSessionReturn, + optionsOrMethods?: UseRpcOptions | Record, + maybeOptions?: UseRpcOptions, +): UseRpcReturn { + let methods: Record | undefined; + let options: UseRpcOptions | undefined; + let session: UseSessionReturn | undefined; + + if (isUseSessionReturn(methodsOrSession)) { + session = methodsOrSession; + methods = optionsOrMethods as Record | undefined; + options = maybeOptions; + } else { + methods = methodsOrSession; + options = optionsOrMethods as UseRpcOptions | undefined; + } + + const { room } = useEnsureSession(session); + + // Ref that always holds the latest handlers — updated synchronously on render + const handlersRef = React.useRef(methods); + handlersRef.current = methods; + + // Ref that always holds the latest options (for participant filter) + const optionsRef = React.useRef(options); + optionsRef.current = options; + + // Derive a stable string from the sorted method name set for the effect dependency. + // The effect only re-runs when methods are added or removed, not when handler bodies change. + const methodNamesEffectKey = React.useMemo( + () => + Object.keys(methods ?? {}) + .sort() + .join('\0'), + [methods], + ); + + React.useEffect(() => { + const currentMethods = handlersRef.current ?? {}; + const names = Object.keys(currentMethods); + + for (const name of names) { + room.registerRpcMethod(name, async (data: RpcInvocationData) => { + // Participant filter + const from = optionsRef.current?.from; + const fromIdentity = typeof from === 'string' ? from : from?.identity; + if (fromIdentity && data.callerIdentity !== fromIdentity) { + throw RpcError.builtIn( + 'UNSUPPORTED_METHOD', + `Method not available for caller ${data.callerIdentity}`, + ); + } + + // Resolve the latest handler from the ref + const handler = handlersRef.current?.[name]; + if (!handler) { + throw RpcError.builtIn('APPLICATION_ERROR', `No handler registered for method "${name}"`); + } + + return resolveHandler(handler, data); + }); + } + + return () => { + for (const name of names) { + room.unregisterRpcMethod(name); + } + }; + }, [room, methodNamesEffectKey]); + + // Stable performRpc function with overloads + const performRpc: PerformRpcFn = React.useCallback( + async (params: PerformRpcDescriptor) => { + let serialized: string; + if (params.serialize) { + try { + serialized = params.serialize(params.payload); + } catch (e) { + throw RpcError.builtIn('APPLICATION_ERROR', `Failed to serialize RPC payload: ${e}`); + } + } else { + serialized = params.payload as string; + } + + const rawResponse = await room.localParticipant.performRpc({ + destinationIdentity: params.destinationIdentity, + method: params.method, + payload: serialized, + responseTimeout: params.responseTimeout, + }); + + if (params.parse) { + try { + return params.parse(rawResponse); + } catch (e) { + throw RpcError.builtIn('APPLICATION_ERROR', `Failed to serialize RPC response: ${e}`); + } + } else { + return rawResponse as Output; + } + }, + [room], + ); + + return React.useMemo(() => ({ performRpc }), [performRpc]); +}