Merge pull request #22 from second-state/feat/responses-api

Feat/responses api

Author: L-jasmine

.github/workflows/rust_ci.yml

@@ -0,0 +1,43 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "**/README.md"
+      - ".github/workflows/**"
+      - "setup/**"
+      - "workshops/**"
+      - "*.sh"
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build-x86_64:
+    name: Build (x86_64)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install base dependencies
+        shell: bash
+        run: |
+          set -euo pipefail
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            ca-certificates \
+            pkg-config \
+            libssl-dev \
+            build-essential \
+            git \
+            curl \
+            unzip
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build binary
+        run: cargo build --release

src/ai/gemini/mod.rs

@@ -100,7 +100,7 @@ mod test {
     use super::types;
     use super::*;
 
-    // cargo test --package esp_assistant --bin esp_assistant -- ai::gemini::test::test_live_client --exact --show-output
+    // cargo test --package echokit_server --bin echokit_server -- ai::gemini::test::test_live_client --exact --show-output
     #[tokio::test]
     async fn test_live_client() -> anyhow::Result<()> {
         env_logger::init();
@@ -121,24 +121,96 @@ mod test {
                 )],
             }),
             input_audio_transcription: Some(types::AudioTranscriptionConfig {}),
+            output_audio_transcription: None,
+            realtime_input_config: Some(types::RealtimeInputConfig {
+                automatic_activity_detection: Some(types::AutomaticActivityDetectionConfig {
+                    disabled: true,
+                }),
+            }),
         };
         client.setup(setup).await?;
         log::info!("Setup completed");
 
         // let submit_data = std::fs::read("sample.pcm").unwrap();
-        let data = std::fs::read("asr.fc012ccfcd71.wav").unwrap();
+        let data = std::fs::read("tmp.wav").unwrap();
         let mut reader = wav_io::reader::Reader::from_vec(data).unwrap();
         let header = reader.read_header().unwrap();
         log::info!("WAV Header: {:?}", header);
-        let x = reader.get_samples_f32().unwrap();
+        let x = crate::util::get_samples_f32(&mut reader).unwrap();
         let x = wav_io::resample::linear(x, 1, header.sample_rate, 16000);
-        let data = wav_io::convert_samples_f32_to_i16(&x);
+        let submit_data = crate::util::convert_samples_f32_to_i16_bytes(&x);
+
+        // let input = types::RealtimeInput {
+        //     audio: None,
+        //     text: Some("你是谁".to_string()),
+        // };
+        client
+            .send_realtime_input(types::RealtimeInput::ActivityStart {})
+            .await?;
 
-        let mut submit_data = Vec::with_capacity(data.len() * 2);
-        for sample in data {
-            submit_data.extend_from_slice(&sample.to_le_bytes());
+        let input = types::RealtimeInput::Audio(types::RealtimeAudio {
+            data: types::Blob::new(submit_data),
+            mime_type: "audio/pcm;rate=16000".to_string(),
+        });
+        log::info!("Sending realtime input");
+        client.send_realtime_input(input).await?;
+        log::info!("Sent realtime input");
+        // client
+        //     .send_realtime_input(types::RealtimeInput::AudioStreamEnd(true))
+        //     .await?;
+        client
+            .send_realtime_input(types::RealtimeInput::ActivityEnd {})
+            .await?;
+
+        log::info!("Sent realtime input");
+        loop {
+            let content = client.receive().await?;
+            log::info!("Received content: {:?}", content);
+            if let types::ServerContent::TurnComplete(true) = content {
+                log::info!("Generation complete");
+                break;
+            }
         }
 
+        Ok(())
+    }
+
+    // cargo test --package echokit_server --bin echokit_server -- ai::gemini::test::test_live_client_audio --exact --show-output
+    #[tokio::test]
+    async fn test_live_client_audio() -> anyhow::Result<()> {
+        env_logger::init();
+        let api_key = std::env::var("GEMINI_API_KEY").unwrap();
+        log::info!("api_key={api_key}");
+        let mut client = LiveClient::connect(&api_key).await?;
+        log::info!("Connected to Gemini Live Client");
+
+        let mut cfg = types::GenerationConfig::default();
+        cfg.response_modalities = Some(vec![types::Modality::AUDIO]);
+
+        let setup = types::Setup {
+            model: "models/gemini-2.0-flash-exp".to_string(),
+            generation_config: Some(cfg),
+            system_instruction: Some(types::Content {
+                parts: vec![types::Parts::Text(
+                    "You are a helpful assistant and answer in a friendly tone.".to_string(),
+                )],
+            }),
+            input_audio_transcription: Some(types::AudioTranscriptionConfig {}),
+            output_audio_transcription: Some(types::AudioTranscriptionConfig {}),
+            realtime_input_config: None,
+        };
+        client.setup(setup).await?;
+        log::info!("Setup completed");
+
+        // let submit_data = std::fs::read("sample.pcm").unwrap();
+        let data = std::fs::read("tmp.wav").unwrap();
+        let mut reader = wav_io::reader::Reader::from_vec(data).unwrap();
+        let header = reader.read_header().unwrap();
+        log::info!("WAV Header: {:?}", header);
+        let x = crate::util::get_samples_f32(&mut reader).unwrap();
+        let x = wav_io::resample::linear(x, 1, header.sample_rate, 16000);
+        let submit_data = crate::util::convert_samples_f32_to_i16_bytes(&x);
+
         // let input = types::RealtimeInput {
         //     audio: None,
         //     text: Some("你是谁".to_string()),
@@ -147,12 +219,14 @@ mod test {
             data: types::Blob::new(submit_data),
             mime_type: "audio/pcm;rate=16000".to_string(),
         });
+        log::info!("Sending realtime input");
         client.send_realtime_input(input).await?;
+        log::info!("Sent realtime input");
         client
             .send_realtime_input(types::RealtimeInput::AudioStreamEnd(true))
             .await?;
 
-        log::info!("Sent realtime input");
+        log::info!("Sent realtime AudioStreamEnd");
         loop {
             let content = client.receive().await?;
             log::info!("Received content: {:?}", content);

src/ai/gemini/types.rs

@@ -23,6 +23,19 @@ pub enum Parts {
 #[serde(rename_all = "camelCase")]
 pub struct AudioTranscriptionConfig {}
 
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AutomaticActivityDetectionConfig {
+    pub disabled: bool,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RealtimeInputConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub automatic_activity_detection: Option<AutomaticActivityDetectionConfig>,
+}
+
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Setup {
@@ -33,6 +46,10 @@ pub struct Setup {
     pub system_instruction: Option<Content>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_audio_transcription: Option<AudioTranscriptionConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_audio_transcription: Option<AudioTranscriptionConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub realtime_input_config: Option<RealtimeInputConfig>,
 }
 
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
@@ -70,7 +87,7 @@ impl Default for Modality {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct Blob(Vec<u8>);
 impl Blob {
     pub fn new(data: Vec<u8>) -> Self {
@@ -81,6 +98,13 @@ impl Blob {
         self.0
     }
 }
+
+impl std::fmt::Debug for Blob {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Blob(len={})", self.0.len())
+    }
+}
+
 impl serde::Serialize for Blob {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -111,6 +135,10 @@ pub enum RealtimeInput {
     Text(String),
     #[serde(rename = "audioStreamEnd")]
     AudioStreamEnd(bool),
+    #[serde(rename = "activityEnd")]
+    ActivityEnd {},
+    #[serde(rename = "activityStart")]
+    ActivityStart {},
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -124,6 +152,8 @@ pub struct RealtimeAudio {
 pub enum ServerContent {
     #[serde(rename = "inputTranscription")]
     InputTranscription { text: String },
+    #[serde(rename = "outputTranscription")]
+    OutputTranscription { text: String },
     #[serde(rename = "modelTurn")]
     ModelTurn(Content),
     #[serde(rename = "generationComplete")]
@@ -178,6 +208,8 @@ mod test {
                 )],
             }),
             input_audio_transcription: None,
+            output_audio_transcription: None,
+            realtime_input_config: None,
         };
         let serialized = serde_json::to_string(&setup).unwrap();
         assert!(serialized.contains("gemini-2.0-flash-live-001"));