Merge branch 'feature/openai_support_voice_change' into 'main'

Add voice change support for OpenAI Demo

See merge request adf/esp-webrtc-solution!17

Author: TempoTian

solutions/openai_demo/main/common.h

@@ -24,6 +24,16 @@ extern "C" {
  */
 void init_board(void);
 
+/**
+ * @brief  OpenAI signaling configuration
+ *
+ * @note   Details see: https://platform.openai.com/docs/api-reference/realtime-sessions/create#realtime-sessions-create-voice
+ */
+typedef struct {
+   char *token; /*!< OpenAI token */
+   char *voice; /*!< Voice to select */
+} openai_signaling_cfg_t;
+
 /**
  * @brief  Get OpenAI signaling implementation
  *

solutions/openai_demo/main/main.c

@@ -177,6 +177,9 @@ static void thread_scheduler(const char *thread_name, media_lib_thread_cfg_t *th
         thread_cfg->priority = 18;
         thread_cfg->core_id = 1;
     }
+    if (strcmp(thread_name, "start") == 0) {
+        thread_cfg->stack_size = 6 * 1024;
+    }
     if (strcmp(thread_name, "pc_send") == 0) {
         thread_cfg->stack_size = 4 * 1024;
         thread_cfg->priority = 15;

solutions/openai_demo/main/openai_signaling.c

@@ -13,29 +13,99 @@
 #include "https_client.h"
 #include "common.h"
 #include "esp_log.h"
+#include <cJSON.h>
 
-#define TAG "OPENAI_SIGNALING"
+#define TAG                   "OPENAI_SIGNALING"
 
-#define OPENAI_REALTIME_URL "https://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview-2024-12-17"
+// Prefer to use mini model currently
+#define OPENAI_REALTIME_MODEL "gpt-4o-mini-realtime-preview-2024-12-17"
+#define OPENAI_REALTIME_URL   "https://api.openai.com/v1/realtime?model=" OPENAI_REALTIME_MODEL
 
 #define SAFE_FREE(p) if (p) {   \
     free(p);                    \
     p = NULL;                   \
 }
 
+#define GET_KEY_END(str, key) get_key_end(str, key, sizeof(key) - 1)
+
 typedef struct {
     esp_peer_signaling_cfg_t cfg;
     uint8_t                 *remote_sdp;
     int                      remote_sdp_size;
+    char                    *ephemeral_token;
 } openai_signaling_t;
 
+static char *get_key_end(char *str, char *key, int len)
+{
+    char *p = strstr(str, key);
+    if (p == NULL) {
+        return NULL;
+    }
+    return p + len;
+}
+
+static void session_answer(http_resp_t *resp, void *ctx)
+{
+    openai_signaling_t *sig = (openai_signaling_t *)ctx;
+    char *token = GET_KEY_END((char *)resp->data, "\"client_secret\"");
+    if (token == NULL) {
+        return;
+    }
+    char *secret = GET_KEY_END(token, "\"value\"");
+    if (secret == NULL) {
+        return;
+    }
+    char *s = strchr(secret, '"');
+    if (s == NULL) {
+        return;
+    }
+    s++;
+    char *e = strchr(s, '"');
+    *e = 0;
+    sig->ephemeral_token = strdup(s);
+    *e = '"';
+}
+
+static void get_ephemeral_token(openai_signaling_t *sig, char *token, char *voice)
+{
+    char content_type[32] = "Content-Type: application/json";
+    int len = strlen("Authorization: Bearer ") + strlen(token) + 1;
+    char auth[len];
+    snprintf(auth, len, "Authorization: Bearer %s", token);
+    char *header[] = {
+        content_type,
+        auth,
+        NULL,
+    };
+    cJSON *root = cJSON_CreateObject();
+    cJSON_AddStringToObject(root, "model", OPENAI_REALTIME_MODEL);
+    cJSON *modalities = cJSON_CreateArray();
+    cJSON_AddItemToArray(modalities, cJSON_CreateString("text"));
+    cJSON_AddItemToArray(modalities, cJSON_CreateString("audio"));
+    cJSON_AddItemToObject(root, "modalities", modalities);
+    cJSON_AddStringToObject(root, "voice", voice);
+    char *json_string = cJSON_Print(root);
+    if (json_string) {
+        https_post("https://api.openai.com/v1/realtime/sessions", header, json_string, NULL, session_answer, sig);
+        free(json_string);
+    }
+    cJSON_Delete(root);
+}
+
 static int openai_signaling_start(esp_peer_signaling_cfg_t *cfg, esp_peer_signaling_handle_t *h)
 {
     openai_signaling_t *sig = (openai_signaling_t *)calloc(1, sizeof(openai_signaling_t));
     if (sig == NULL) {
         return ESP_PEER_ERR_NO_MEM;
     }
+    openai_signaling_cfg_t *openai_cfg = (openai_signaling_cfg_t *)cfg->extra_cfg;
     sig->cfg = *cfg;
+    // alloy, ash, ballad, coral, echo sage, shimmer and verse
+    get_ephemeral_token(sig, openai_cfg->token, openai_cfg->voice ? openai_cfg->voice : "alloy");
+    if (sig->ephemeral_token == NULL) {
+        free(sig);
+        return ESP_PEER_ERR_NOT_SUPPORT;
+    }
     *h = sig;
     esp_peer_signaling_ice_info_t ice_info = {
         .is_initiator = true,
@@ -67,9 +137,10 @@ static int openai_signaling_send_msg(esp_peer_signaling_handle_t h, esp_peer_sig
     } else if (msg->type == ESP_PEER_SIGNALING_MSG_SDP) {
         printf("Receive local SDP\n");
         char content_type[32] = "Content-Type: application/sdp";
-        int len = strlen("Authorization: Bearer ") + strlen((char *)sig->cfg.extra_cfg) + 1;
+        char *token = sig->ephemeral_token;
+        int len = strlen("Authorization: Bearer ") + strlen(token) + 1;
         char auth[len];
-        snprintf(auth, len, "Authorization: Bearer %s", (char *)sig->cfg.extra_cfg);
+        snprintf(auth, len, "Authorization: Bearer %s", token);
         char *header[] = {
             content_type,
             auth,
@@ -95,6 +166,7 @@ static int openai_signaling_stop(esp_peer_signaling_handle_t h)
     openai_signaling_t *sig = (openai_signaling_t *)h;
     sig->cfg.on_close(sig->cfg.ctx);
     SAFE_FREE(sig->remote_sdp);
+    SAFE_FREE(sig->ephemeral_token);
     SAFE_FREE(sig);
     return 0;
 }

solutions/openai_demo/main/webrtc.c

@@ -519,6 +519,9 @@ int start_webrtc(void)
     esp_peer_default_cfg_t peer_cfg = {
         .agent_recv_timeout = 500,
     };
+    openai_signaling_cfg_t openai_cfg = {
+        .token = OPENAI_API_KEY,
+    };
     esp_webrtc_cfg_t cfg = {
         .peer_cfg = {
             .audio_info = {
@@ -536,7 +539,7 @@ int start_webrtc(void)
             .extra_cfg = &peer_cfg,
             .extra_size = sizeof(peer_cfg),
         },
-        .signaling_cfg.extra_cfg = OPENAI_API_KEY,
+        .signaling_cfg.extra_cfg = &openai_cfg,
         .peer_impl = esp_peer_get_default_impl(),
         .signaling_impl = esp_signaling_get_openai_signaling(),
     };