iseer-live-api-example/src/live-client.js at main · GodelEsherBach/iseer-live-api-example · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
export class LiveClient {
  constructor(onEvent) {
    this.ws = null;
    this.onEvent = onEvent;
    this.isConnected = false;
  }

  async connect() {
    try {

      const wsUrl = 'wss://genai.api.iseer.co?key=iseer_live_a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6';

      const setupMsg = {
        setup: {
          model: "models/arete-live",
          generationConfig: {
            responseModalities: ["AUDIO"]
          },
          systemInstruction: {
            parts: [{ text: "You are a helpful and expressive Iseer assistant. Respond naturally and conversationally." }]
          }
        }
      };

      this.ws = new WebSocket(wsUrl);

      this.ws.onopen = () => {
          this.ws.send(JSON.stringify(setupMsg));
      };

      this.ws.onmessage = async (event) => {
        try {
          let text = event.data;
          if (text instanceof Blob) {
            text = await text.text();
          }
          const msg = JSON.parse(text);
          this.handleMessage(msg);
        } catch (e) {
          console.error("Failed to parse message", e);
        }
      };

      this.ws.onerror = (error) => {
        console.error('WebSocket Error:', error);
        this.onEvent({ type: 'error', error: 'Connection to Iseer Live API failed.' });
        this.disconnect();
      };

      this.ws.onclose = (event) => {
        this.isConnected = false;
        console.error(`WebSocket Closed: Code ${event.code}, Reason: ${event.reason || 'None'}`);
        this.onEvent({ type: 'disconnected', code: event.code, reason: event.reason });
      };

    } catch (error) {
      console.error('Connection Error:', error);
      this.onEvent({ type: 'error', error: error.message });
    }
  }

  handleMessage(msg) {
    if (msg.setupComplete) {
      this.isConnected = true;
      this.onEvent({ type: 'connected' });
      return;
    }

    if (msg.serverContent) {
      const content = msg.serverContent;

      if (content.interrupted) {
        this.onEvent({ type: 'interrupted' });
      }

      if (content.inputTranscription) {
        this.onEvent({ type: 'user-transcript', text: content.inputTranscription.text });
      }

      if (content.outputTranscription) {
        this.onEvent({ type: 'model-transcript', text: content.outputTranscription.text });
      }

      if (content.modelTurn && content.modelTurn.parts) {
        for (const part of content.modelTurn.parts) {
          if (part.inlineData && part.inlineData.mimeType && part.inlineData.mimeType.includes('audio/pcm')) {
            this.onEvent({ type: 'audio', base64: part.inlineData.data });
          }
        }
      }
    }
  }

  sendAudioChunk(base64PCM) {
    if (!this.isConnected || !this.ws) return;

    this.ws.send(JSON.stringify({
      realtimeInput: {
        audio: {
          mimeType: "audio/pcm;rate=16000",
          data: base64PCM
        }
      }
    }));
  }

  sendAudioEnd() {
    if (!this.isConnected || !this.ws) return;

    this.ws.send(JSON.stringify({
      realtimeInput: {
        audioStreamEnd: true
      }
    }));
  }

  disconnect() {
    if (this.ws) {
      this.ws.close();
      this.ws = null;
    }
    this.isConnected = false;
    this.onEvent({ type: 'disconnected' });
  }
}