Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions deep-sea-stories/packages/backend/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ export const AGENT_INSTRUCTIONS_TEMPLATE = fs.readFileSync(
'utf8',
);

export const AGENT_CLIENT_TOOL_INSTRUCTIONS = fs.readFileSync(
join(__dirname, 'prompts', 'client-tool-instructions.md'),
'utf8',
);

export const FISHJAM_AGENT_OPTIONS: PeerOptions = {
output: {
audioSampleRate: 16_000,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
After the user says "I'm guessing now..." and provides a solution to the riddle:

If the answer is correct, first congratulate the user enthusiastically, and then send the client tool call.

If the answer is incorrect, do not send the tool call; instead, continue the game as usual.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
} from './elevenlabs-conversation.js';
import { roomService } from './room.js';
import { getInstructionsForStory } from '../utils.js';
import { CONFIG } from '../config.js';
import { AGENT_CLIENT_TOOL_INSTRUCTIONS, CONFIG } from '../config.js';
import type { VoiceAgentSessionManager } from '../types.js';
import {
GameSessionNotFoundError,
Expand All @@ -18,6 +18,8 @@ export class ElevenLabsSessionManager implements VoiceAgentSessionManager {
PeerId,
Promise<ElevenLabsConversation>
>();
private endingRooms = new Set<RoomId>();
private gameEndingToolId: string | undefined;

private async resolveStory(roomId: RoomId) {
const gameSession = roomService.getGameSession(roomId);
Expand Down Expand Up @@ -57,6 +59,8 @@ export class ElevenLabsSessionManager implements VoiceAgentSessionManager {
): Promise<ElevenLabsConversation> {
await this.deleteSession(peerId);

const toolId = await this.ensureGameEndingTool();

const story = await this.resolveStory(roomId);
const instructions = getInstructionsForStory(story);

Expand All @@ -65,9 +69,14 @@ export class ElevenLabsSessionManager implements VoiceAgentSessionManager {
agent: {
firstMessage: 'Welcome to Deepsea stories',
language: 'en',
prompt: {
prompt: instructions,
},
prompt: toolId
? {
prompt: instructions,
toolIds: [toolId],
}
: {
prompt: instructions,
},
},
},
});
Expand All @@ -77,11 +86,93 @@ export class ElevenLabsSessionManager implements VoiceAgentSessionManager {
CONFIG.ELEVENLABS_API_KEY,
);
await session.connect();
this.registerClientToolHandler(session, peerId, roomId);

this.sessions.set(peerId, session);
return session;
}

private async ensureGameEndingTool(): Promise<string | undefined> {
if (this.gameEndingToolId) {
return this.gameEndingToolId;
}

const tools = await elevenLabs.conversationalAi.tools.list();
const existingTool = (tools.tools ?? []).find(
(tool) =>
tool.toolConfig.type === 'client' &&
tool.toolConfig.name === 'game-ending',
);

if (existingTool?.id) {
this.gameEndingToolId = existingTool.id;
return this.gameEndingToolId;
}

const createdTool = await elevenLabs.conversationalAi.tools.create({
toolConfig: {
type: 'client',
name: 'game-ending',
description: AGENT_CLIENT_TOOL_INSTRUCTIONS,
},
});

this.gameEndingToolId = createdTool.id;
return this.gameEndingToolId;
}

private registerClientToolHandler(
session: ElevenLabsConversation,
peerId: PeerId,
roomId: RoomId,
): void {
session.on('clientToolCall', async (clientToolCall: unknown) => {
const call =
clientToolCall && typeof clientToolCall === 'object'
? (clientToolCall as Record<string, unknown>)
: undefined;
const toolName =
typeof call?.tool_name === 'string' ? call.tool_name : undefined;
if (!toolName || toolName !== 'game-ending') {
return;
}

if (this.endingRooms.has(roomId)) {
return;
}

this.endingRooms.add(roomId);
const gameSession = roomService.getGameSession(roomId);

if (!gameSession) {
console.warn(
`Received game-ending tool call for room ${roomId} without active game session`,
);
this.endingRooms.delete(roomId);
return;
}

if (!roomService.isGameActive(roomId)) {
this.endingRooms.delete(roomId);
return;
}

try {
await gameSession.stopGame();
console.log(
`Game session for room ${roomId} ended after game-ending tool call from peer ${peerId}`,
);
} catch (error) {
console.error(
`Failed to stop game for room ${roomId} after game-ending tool call from peer ${peerId}:`,
error,
);
} finally {
this.endingRooms.delete(roomId);
}
});
}

async deleteSession(peerId: PeerId): Promise<void> {
const session = this.sessions.get(peerId);
if (session) {
Expand Down
8 changes: 4 additions & 4 deletions deep-sea-stories/packages/backend/src/service/game-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,16 +166,16 @@ export class GameSession {
orchestrator.setupAudioPipelines();
}

async stopGame(roomId: RoomId): Promise<void> {
async stopGame(): Promise<void> {
this.voiceSessionManager?.cleanup();
this.setStory(undefined);
console.log(`Stopped game for room ${roomId}`);
console.log(`Stopped game for room ${this.roomId}`);
}

async removePeerFromGame(roomId: RoomId, peerId: PeerId): Promise<void> {
async removePeerFromGame(peerId: PeerId): Promise<void> {
if (this.voiceSessionManager) {
await this.voiceSessionManager.deleteSession(peerId);
console.log(`Removed peer ${peerId} from game in room ${roomId}`);
console.log(`Removed peer ${peerId} from game in room ${this.roomId}`);
}
}
}
2 changes: 1 addition & 1 deletion deep-sea-stories/packages/backend/src/service/notifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class NotifierService {
gameSession.removeConnectedPeer(msg.peerId);

if (roomService.isGameActive(msg.roomId)) {
await gameSession.removePeerFromGame(msg.roomId, msg.peerId);
await gameSession.removePeerFromGame(msg.peerId);
}
});
}
Expand Down