Skip to content

Commit 172305d

Browse files
toubatbriantoubatbrian
authored
Add Hedra Avatar Plugin (#957)
1 parent 1d35967 commit 172305d

File tree

15 files changed

+700
-0
lines changed

15 files changed

+700
-0
lines changed

‎.changeset/hip-brooms-stand.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@livekit/agents-plugin-hedra': patch
3+
---
4+
5+
Built hedra avatar plugin for livekit agent js

‎REUSE.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,15 @@ SPDX-License-Identifier = "Apache-2.0"
4242
path = ["agents/resources/*.ogg", "agents/resources/NOTICE"]
4343
SPDX-FileCopyrightText = "2024 LiveKit, Inc."
4444
SPDX-License-Identifier = "Apache-2.0"
45+
46+
# image assets
47+
[[annotations]]
48+
path = ["**/*.png", "**/*.jpg", "**/*.jpeg"]
49+
SPDX-FileCopyrightText = "2026 LiveKit, Inc."
50+
SPDX-License-Identifier = "Apache-2.0"
51+
52+
# README files
53+
[[annotations]]
54+
path = ["**/README.md"]
55+
SPDX-FileCopyrightText = "2026 LiveKit, Inc."
56+
SPDX-License-Identifier = "Apache-2.0"

‎examples/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
"@livekit/agents-plugin-deepgram": "workspace:*",
3232
"@livekit/agents-plugin-elevenlabs": "workspace:*",
3333
"@livekit/agents-plugin-google": "workspace:*",
34+
"@livekit/agents-plugin-hedra": "workspace:*",
3435
"@livekit/agents-plugin-inworld": "workspace:*",
3536
"@livekit/agents-plugin-livekit": "workspace:*",
3637
"@livekit/agents-plugin-neuphonic": "workspace:*",

‎examples/src/hedra/hedra_avatar.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
import {
5+
type JobContext,
6+
type JobProcess,
7+
ServerOptions,
8+
cli,
9+
defineAgent,
10+
inference,
11+
metrics,
12+
voice,
13+
} from '@livekit/agents';
14+
import * as hedra from '@livekit/agents-plugin-hedra';
15+
import * as livekit from '@livekit/agents-plugin-livekit';
16+
import * as silero from '@livekit/agents-plugin-silero';
17+
import { fileURLToPath } from 'node:url';
18+
19+
// import { readFileSync } from 'node:fs';
20+
// import { dirname, join } from 'node:path';
21+
// const __filename = fileURLToPath(import.meta.url);
22+
// const __dirname = dirname(__filename);
23+
24+
export default defineAgent({
25+
prewarm: async (proc: JobProcess) => {
26+
proc.userData.vad = await silero.VAD.load();
27+
},
28+
entry: async (ctx: JobContext) => {
29+
const agent = new voice.Agent({
30+
instructions: 'You are a helpful assistant. Speak clearly and concisely.',
31+
});
32+
33+
const session = new voice.AgentSession({
34+
stt: new inference.STT({
35+
model: 'deepgram/nova-3',
36+
language: 'en',
37+
}),
38+
llm: new inference.LLM({
39+
model: 'openai/gpt-4o-mini',
40+
}),
41+
tts: new inference.TTS({
42+
model: 'cartesia/sonic-3',
43+
}),
44+
vad: ctx.proc.userData.vad! as silero.VAD,
45+
turnDetection: new livekit.turnDetector.MultilingualModel(),
46+
});
47+
48+
await session.start({
49+
agent,
50+
room: ctx.room,
51+
});
52+
53+
const avatar = new hedra.AvatarSession({
54+
avatarId: process.env.HEDRA_AVATAR_ID,
55+
// API key is read from HEDRA_API_KEY environment variable by default
56+
57+
// Alternatively, use a custom avatar image:
58+
// const avatarImageData = readFileSync(join(__dirname, 'avatar.png'));
59+
// avatarImage: {
60+
// data: avatarImageData,
61+
// mimeType: 'image/png',
62+
// filename: 'avatar.png',
63+
// },
64+
});
65+
await avatar.start(session, ctx.room);
66+
67+
const usageCollector = new metrics.UsageCollector();
68+
69+
session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
70+
metrics.logMetrics(ev.metrics);
71+
usageCollector.collect(ev.metrics);
72+
});
73+
74+
session.generateReply({
75+
instructions: 'Greet the user briefly and confirm you are ready.',
76+
});
77+
},
78+
});
79+
80+
cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));

‎plugins/hedra/README.md‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Hedra plugin for LiveKit Agents
2+
3+
Support for avatar generation and animation with [Hedra](https://hedra.ai/).
4+
5+
See [https://docs.livekit.io/agents/integrations/avatar/hedra/](https://docs.livekit.io/agents/integrations/avatar/hedra/) for more information.
6+
7+
## Installation
8+
9+
```bash
10+
npm install @livekit/agents-plugin-hedra
11+
# or
12+
pnpm add @livekit/agents-plugin-hedra
13+
# or
14+
yarn add @livekit/agents-plugin-hedra
15+
```
16+
17+
## Pre-requisites
18+
19+
You'll need an API key from Hedra. It can be set as an environment variable: `HEDRA_API_KEY`
20+
21+
## Usage
22+
23+
### Using an Avatar ID
24+
25+
```typescript
26+
import { AvatarSession } from '@livekit/agents-plugin-hedra';
27+
28+
// Create an avatar session with an avatar ID
29+
const avatar = new AvatarSession({
30+
avatarId: 'your-avatar-id',
31+
apiKey: 'your-hedra-api-key', // or set HEDRA_API_KEY env var
32+
});
33+
34+
// Start the avatar session after creating your agent session
35+
await avatar.start(agentSession, room);
36+
```
37+
38+
### Using a Custom Avatar Image
39+
40+
```typescript
41+
import { AvatarSession } from '@livekit/agents-plugin-hedra';
42+
import fs from 'node:fs';
43+
44+
// Read an image file
45+
const imageBuffer = fs.readFileSync('path/to/avatar.jpg');
46+
47+
// Create an avatar session with a custom image
48+
const avatar = new AvatarSession({
49+
avatarImage: {
50+
data: imageBuffer,
51+
mimeType: 'image/jpeg',
52+
filename: 'avatar.jpg',
53+
},
54+
apiKey: 'your-hedra-api-key',
55+
});
56+
57+
// Start the avatar session
58+
await avatar.start(agentSession, room);
59+
```
60+
61+
### Full Example
62+
63+
```typescript
64+
import {
65+
type JobContext,
66+
type JobProcess,
67+
WorkerOptions,
68+
cli,
69+
defineAgent,
70+
voice,
71+
} from '@livekit/agents';
72+
import * as deepgram from '@livekit/agents-plugin-deepgram';
73+
import * as hedra from '@livekit/agents-plugin-hedra';
74+
import * as openai from '@livekit/agents-plugin-openai';
75+
import * as silero from '@livekit/agents-plugin-silero';
76+
77+
export default defineAgent({
78+
prewarm: async (proc: JobProcess) => {
79+
proc.userData.vad = await silero.VAD.load();
80+
},
81+
entry: async (ctx: JobContext) => {
82+
const vad = ctx.proc.userData.vad as silero.VAD;
83+
84+
const assistant = new voice.Agent({
85+
instructions: 'You are a helpful voice AI assistant.',
86+
});
87+
88+
const session = new voice.AgentSession({
89+
vad,
90+
stt: new deepgram.STT(),
91+
llm: new openai.LLM(),
92+
tts: new openai.TTS(),
93+
});
94+
95+
// Create and start the Hedra avatar
96+
const avatar = new hedra.AvatarSession({
97+
avatarId: 'your-avatar-id',
98+
});
99+
100+
await session.start({
101+
agent: assistant,
102+
room: ctx.room,
103+
});
104+
105+
// Start the avatar session after connecting
106+
await avatar.start(session, ctx.room);
107+
108+
session.generateReply({
109+
instructions: 'Greet the user and offer your assistance.',
110+
});
111+
},
112+
});
113+
```
114+
115+
## Configuration Options
116+
117+
### AvatarSessionOptions
118+
119+
| Option | Type | Description |
120+
|--------|------|-------------|
121+
| `avatarId` | `string` | The Hedra avatar ID to use. Either `avatarId` or `avatarImage` must be provided. |
122+
| `avatarImage` | `AvatarImage` | A custom avatar image. Either `avatarId` or `avatarImage` must be provided. |
123+
| `apiUrl` | `string` | The Hedra API URL. Defaults to `HEDRA_API_URL` env var or the default Hedra API endpoint. |
124+
| `apiKey` | `string` | The Hedra API key. Defaults to `HEDRA_API_KEY` env var. |
125+
| `avatarParticipantIdentity` | `string` | The identity of the avatar participant in the room. Defaults to `'hedra-avatar-agent'`. |
126+
| `avatarParticipantName` | `string` | The name of the avatar participant in the room. Defaults to `'hedra-avatar-agent'`. |
127+
| `connOptions` | `APIConnectOptions` | Connection options for API requests (retry count, timeout, etc.). |
128+
129+
### AvatarImage
130+
131+
| Property | Type | Description |
132+
|----------|------|-------------|
133+
| `data` | `Buffer` | The raw image data. |
134+
| `mimeType` | `string` | The MIME type of the image (e.g., `'image/jpeg'`, `'image/png'`). |
135+
| `filename` | `string` | Optional filename for the image. |
136+
137+
## Environment Variables
138+
139+
| Variable | Description |
140+
|----------|-------------|
141+
| `HEDRA_API_KEY` | Your Hedra API key |
142+
| `HEDRA_API_URL` | Custom Hedra API URL (optional) |
143+
| `LIVEKIT_URL` | Your LiveKit server URL |
144+
| `LIVEKIT_API_KEY` | Your LiveKit API key |
145+
| `LIVEKIT_API_SECRET` | Your LiveKit API secret |

‎plugins/hedra/api-extractor.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* Config file for API Extractor. For more info, please visit: https://api-extractor.com
3+
*/
4+
{
5+
"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
6+
7+
/**
8+
* Optionally specifies another JSON config file that this file extends from. This provides a way for
9+
* standard settings to be shared across multiple projects.
10+
*
11+
* If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
12+
* the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be
13+
* resolved using NodeJS require().
14+
*
15+
* SUPPORTED TOKENS: none
16+
* DEFAULT VALUE: ""
17+
*/
18+
"extends": "../../api-extractor-shared.json",
19+
"mainEntryPointFilePath": "./dist/index.d.ts"
20+
}

‎plugins/hedra/package.json‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{
2+
"name": "@livekit/agents-plugin-hedra",
3+
"version": "1.0.0",
4+
"description": "Hedra avatar plugin for LiveKit Node Agents",
5+
"main": "dist/index.js",
6+
"require": "dist/index.cjs",
7+
"types": "dist/index.d.ts",
8+
"exports": {
9+
"import": {
10+
"types": "./dist/index.d.ts",
11+
"default": "./dist/index.js"
12+
},
13+
"require": {
14+
"types": "./dist/index.d.cts",
15+
"default": "./dist/index.cjs"
16+
}
17+
},
18+
"author": "LiveKit",
19+
"type": "module",
20+
"repository": "git@github.com:livekit/agents-js.git",
21+
"license": "Apache-2.0",
22+
"files": [
23+
"dist",
24+
"src",
25+
"README.md"
26+
],
27+
"scripts": {
28+
"build": "tsup --onSuccess \"pnpm build:types\"",
29+
"build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
30+
"clean": "rm -rf dist",
31+
"clean:build": "pnpm clean && pnpm build",
32+
"lint": "eslint -f unix \"src/**/*.{ts,js}\"",
33+
"api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
34+
"api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
35+
},
36+
"devDependencies": {
37+
"@livekit/agents": "workspace:*",
38+
"@livekit/rtc-node": "^0.13.22",
39+
"@microsoft/api-extractor": "^7.35.0",
40+
"pino": "^8.19.0",
41+
"tsup": "^8.3.5",
42+
"typescript": "^5.0.0"
43+
},
44+
"dependencies": {
45+
"livekit-server-sdk": "^2.13.3"
46+
},
47+
"peerDependencies": {
48+
"@livekit/agents": "workspace:*",
49+
"@livekit/rtc-node": "^0.13.22"
50+
}
51+
}

0 commit comments

Comments
 (0)