Add support for dynamic transition functions + fix tts-say

Modified-by: Ovi Stoica <ovidiu.stoica1094@gmail.com>

Author: ovistoica

src/voice_fn/processors/deepgram.clj

@@ -169,7 +169,7 @@ https://developers.deepgram.com/docs/understanding-end-of-speech-detection#using
                    keep-alive #(loop []
                                  (when @alive?
                                    (a/<!! (a/timeout 3000))
-                                   (t/log! {:level :debug :id :deepgram} "Sending keep-alive message")
+                                   (t/log! {:level :trace :id :deepgram} "Sending keep-alive message")
                                    (ws/send! ws-conn keep-alive-payload)
                                    (recur)))]
                ((flow/futurize write-to-ws :exec :io))

src/voice_fn/processors/elevenlabs.clj

@@ -157,7 +157,7 @@
                    keep-alive #(loop []
                                  (when @alive?
                                    (a/<!! (a/timeout 3000))
-                                   (t/log! {:level :debug :id :elevenlabs} "Sending keep-alive message")
+                                   (t/log! {:level :trace :id :elevenlabs} "Sending keep-alive message")
                                    (ws/send! ws-conn keep-alive-message)
                                    (recur)))]
                ((flow/futurize write-to-ws :exec :io))
@@ -192,5 +192,7 @@
                         [(assoc state :audio/acc attempt)]))
                     (cond
                       (frame/speak-frame? msg)
-                      [state {:ws-write [msg]}]
+                      (do
+                        (t/log! {:id :elevenlabs :level :debug} ["SPEAK" (:frame/data msg)])
+                        [state {:ws-write [msg]}])
                       :else [state])))}))

src/voice_fn/processors/llm_context_aggregator.clj

@@ -234,7 +234,7 @@ S: Start, E: End, T: Transcription, I: Interim, X: Text
            ;; function, to wait for the new context
            ;; messages from the new scenario node
            :properties {:run-llm? (nil? transition-cb)
-                        :on-update transition-cb}}))
+                        :on-update #(transition-cb args)}}))
       (frame/llm-tool-call-result
         {:request tool-call-msg
          :result {:role :tool

src/voice_fn/processors/openai.clj

@@ -14,7 +14,8 @@
 (def openai-completions-url "https://api.openai.com/v1/chat/completions")
 
 (defn stream-openai-chat-completion
-  [{:keys [api-key messages tools model]}]
+  [{:keys [api-key messages tools model]
+    :or {model "gpt-4o-mini"}}]
   (:body (request/sse-request {:request {:url openai-completions-url
                                          :headers {"Authorization" (str "Bearer " api-key)
                                                    "Content-Type" "application/json"}
@@ -23,8 +24,9 @@
                                          :body (u/json-str (cond-> {:messages messages
                                                                     :stream true
                                                                     :model model}
-                                                             tools (assoc :tools tools)))}
+                                                             (pos? (count tools)) (assoc :tools tools)))}
                                :params {:stream/close? true}})))
+
 (defn normal-chat-completion
   [{:keys [api-key messages tools model]}]
   (http/request {:url openai-completions-url
@@ -36,7 +38,7 @@
                  :body (u/json-str (cond-> {:messages messages
                                             :stream true
                                             :model model}
-                                     tools (assoc :tools tools)))}))
+                                     (pos? (count tools)) (assoc :tools tools)))}))
 
 (comment
   (require '[voice-fn.secrets :refer [secret]])

src/voice_fn/scenario_manager.clj

@@ -17,7 +17,7 @@
   [:or
    [:map
     [:type (schema/flex-enum ["tts-say"])]
-    [:text :string]]
+    [:text [:or :string [:vector :string]]]]
    [:map
     [:type (schema/flex-enum ["end-conversation"])]]
    [:map
@@ -50,7 +50,8 @@
                                             vals
                                             (mapcat :functions)
                                             (keep (fn [f] (get-in f [:function :transition-to])))
-                                            (remove nil?))
+                                            (remove #(or (nil? %)
+                                                         (fn? %))))
                            invalid-transition (first (remove nodes transitions))]
                        (when invalid-transition
                          (format "Unreachable node: %s" invalid-transition))))}
@@ -60,7 +61,8 @@
                              vals
                              (mapcat :functions)
                              (keep (fn [f] (get-in f [:function :transition-to])))
-                             (remove nil?))]
+                             (remove #(or (nil? %)
+                                        (fn? %))))]
         (every? defined-nodes transitions)))]])
 
 (defprotocol Scenario
@@ -79,8 +81,13 @@
   [scenario tool]
   (let [fndef (:function tool)
         handler (:handler fndef)
-        next-node (:transition-to fndef)
-        cb #(set-node scenario next-node)]
+
+        next-node-or-fn (:transition-to fndef)
+        cb (if (fn? next-node-or-fn)
+             (fn [args]
+               (let [next-node (next-node-or-fn args)]
+                 (set-node scenario next-node)))
+             (fn [_] (set-node scenario next-node-or-fn)))]
     (cond-> tool
       true (update-in [:function] dissoc :transition-to)
       true (assoc-in [:function :transition-cb] cb)
@@ -96,10 +103,14 @@
         initialized? (atom false)
         tts-action? #(contains? #{:tts-say "tts-say"} (:type %))
         end-action? #(contains? #{:end-conversation "end-conversation"} (:type %))
-        handle-action #(cond
-                         (tts-action? %) (flow/inject flow flow-in-coord [(frame/speak-frame (:text %))])
-                         (end-action? %) (flow/stop flow)
-                         :else ((:handler %)))]
+        handle-action (fn [a]
+                        (cond
+                          (tts-action? a)
+                          (flow/inject flow flow-in-coord (if (coll? (:text a))
+                                                            (mapv #(frame/speak-frame %) (:text a))
+                                                            [(frame/speak-frame (:text a))]))
+                          (end-action? a) (flow/stop flow)
+                          :else ((:handler a))))]
     (reify Scenario
       (current-node [_] @current-node)
       (set-node [this node-id]
@@ -113,11 +124,10 @@
             (doseq [a (:post-actions node)] (handle-action a)))
           (reset! current-node node-id)
           (try
-            (t/log! "Sending new scenario")
-            (doseq [a (:pre-actions node)] (handle-action a))
             (flow/inject flow flow-in-coord [(frame/scenario-context-update {:messages append-context
                                                                              :tools tools
                                                                              :properties {:run-llm? (if (boolean? (:run-llm? node)) (:run-llm? node) true)}})])
+            (doseq [a (:pre-actions node)] (handle-action a))
 
             (catch Exception e
               (t/log! :error e)))))

src/voice_fn/schema.clj

@@ -509,13 +509,25 @@
        [:handler [:=> [:cat :map] :any]]
        [:transition-cb {:optional true} [:=> :cat :nil]]]]]))
 
+(def TransitionTo
+  [:or
+   :keyword
+   [:and
+    [:=> [:cat :map] :any]
+    [:fn {:error/message "Transition function must return a valid node keyword"}
+     (fn [f]
+       (try
+         (let [result (f {})]
+           (keyword? result))
+         (catch Exception _ false)))]]])
+
 (def LLMTransitionToolDefinition
   (mu/merge LLMFunctionToolDefinitionWithHandling
             [:map {:closed true}
              [:function
               [:map
                [:handler {:optional true} [:=> [:cat :map] :any]]
-               [:transition-to :keyword]]]]))
+               [:transition-to TransitionTo]]]]))
 
 (def RegisteredFunctions [:map-of :string [:map
                                            [:tool [:=> [:cat :map] :any]]

src/voice_fn/transport.clj

@@ -4,7 +4,7 @@
    [clojure.core.async.flow :as flow]
    [taoensso.telemere :as t]
    [uncomplicate.clojure-sound.core :refer [open! read! start! stop! write!]]
-   [uncomplicate.clojure-sound.sampled :refer [audio-format line line-info]]
+   [uncomplicate.clojure-sound.sampled :refer [audio-format flush! line line-info]]
    [uncomplicate.commons.core :refer [close!]]
    [voice-fn.frame :as frame]
    [voice-fn.schema :as schema]
@@ -14,7 +14,7 @@
    [voice-fn.utils.core :as u])
   (:import
    (java.util Arrays)
-   (javax.sound.sampled AudioFormat AudioSystem DataLine$Info TargetDataLine)))
+   (javax.sound.sampled AudioFormat AudioSystem DataLine$Info)))
 
 (def AsyncOutputProcessorSchema
   [:map
@@ -233,6 +233,7 @@
                    (when (= transition ::flow/stop)
                      (when-let [line (::speaker-line state)]
                        (stop! line)
+                       (flush! line)
                        (close! line))
                      (doseq [port (concat (vals in-ports) (vals out-ports))]
                        (a/close! port))))