ref

Author: Guikingone

docs/components/platform.rst

@@ -501,8 +501,58 @@ This allows fast and isolated testing of AI-powered features without relying on
 
     This requires `cURL` and the `ext-curl` extension to be installed.
 
-Adding Voice
-~~~~~~~~~~~~
+Speech support
+~~~~~~~~~~~~~~
+
+Using speech to send messages / receive answers as audio is a common use case when integrating agents and/or chats.
+
+Speech support can be enable using ``Symfony\AI\Platform\Speech\SpeechProviderListener``::
+
+    use Symfony\AI\Agent\Agent;
+    use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
+    use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
+    use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
+    use Symfony\AI\Platform\Message\Message;
+    use Symfony\AI\Platform\Message\MessageBag;
+    use Symfony\AI\Platform\Speech\SpeechConfiguration;
+    use Symfony\AI\Platform\Speech\SpeechProviderListener;
+    use Symfony\Component\EventDispatcher\EventDispatcher;
+
+    $eventDispatcher = new EventDispatcher();
+    $eventDispatcher->addSubscriber(new SpeechProviderListener([
+        new ElevenLabsSpeechProvider(PlatformFactory::create(
+            apiKey: $elevenLabsApiKey,
+            httpClient: http_client(),
+            speechConfiguration: new SpeechConfiguration(
+                ttsModel: 'eleven_multilingual_v2',
+                ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
+                sttModel: 'eleven_multilingual_v2'
+            )),
+        ),
+    ], []));
+
+    $platform = OpenAiPlatformFactory::create($openAiApiKey, httpClient: HttpClient::create(), eventDispatcher: $eventDispatcher);
+
+    $agent = new Agent($platform, 'gpt-4o');
+    $answer = $agent->call(new MessageBag(
+        Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
+    ));
+
+    echo $answer->getSpeech('eleven_labs')->asBinary();
+
+When using the bundle, the configuration allows to configure models and voices::
+
+    ai:
+        platform:
+            eleven_labs:
+                api_key: '%env(ELEVEN_LABS_API_KEY)%'
+
+        speech:
+            eleven_labs:
+                tts_model: 'eleven_multilingual_v2'
+                tts_voice: '%env(ELEVEN_LABS_VOICE_IDENTIFIER)%'
+                tts_extra_options:
+                    foo: bar
 
 Code Examples
 ~~~~~~~~~~~~~

examples/speech/agent-eleven-labs-speech-tts.php

@@ -41,4 +41,4 @@
     Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
 ));
 
-echo $answer->getSpeech()->asBinary();
+echo $answer->getSpeech('eleven_labs')?->asBinary();

src/ai-bundle/config/services.php

@@ -63,6 +63,7 @@
 use Symfony\AI\Platform\Contract\JsonSchema\DescriptionParser;
 use Symfony\AI\Platform\Contract\JsonSchema\Factory as SchemaFactory;
 use Symfony\AI\Platform\Serializer\StructuredOutputSerializer;
+use Symfony\AI\Platform\Speech\SpeechProviderListener;
 use Symfony\AI\Platform\StructuredOutput\PlatformSubscriber;
 use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactory;
 use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactoryInterface;
@@ -235,5 +236,13 @@
                 tagged_locator('ai.message_store', 'name'),
             ])
             ->tag('console.command')
+
+        // listeners
+        ->set('ai.speech_provider.listener', SpeechProviderListener::class)
+            ->args([
+                tagged_locator('ai.speech_provider', 'name'),
+                tagged_locator('ai.speech_listener', 'name'),
+            ])
+            ->tag('kernel.event_subscriber')
     ;
 };

src/platform/src/Bridge/ElevenLabs/ElevenLabsSpeechProvider.php

@@ -39,7 +39,7 @@ public function generate(DeferredResult $result, array $options): Speech
             ...$options,
         ]);
 
-        return new Speech($payload, $speechResult);
+        return new Speech($payload, $speechResult, 'eleven_labs');
     }
 
     public function support(DeferredResult $result, array $options): bool

src/platform/src/CachedPlatform.php

@@ -13,6 +13,7 @@
 
 use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface;
 use Symfony\AI\Platform\Result\DeferredResult;
+use Symfony\AI\Platform\Speech\SpeechConfiguration;
 use Symfony\Component\Cache\Adapter\TagAwareAdapterInterface;
 use Symfony\Contracts\Cache\CacheInterface;
 use Symfony\Contracts\Cache\ItemInterface;
@@ -66,4 +67,9 @@ public function getModelCatalog(): ModelCatalogInterface
     {
         return $this->platform->getModelCatalog();
     }
+
+    public function getSpeechConfiguration(): ?SpeechConfiguration
+    {
+        return $this->platform->getSpeechConfiguration();
+    }
 }

src/platform/src/Message/UserMessage.php

@@ -72,7 +72,7 @@ public function hasAudioContent(): bool
         return false;
     }
 
-    public function getAudioContent(): ?Audio
+    public function getAudioContent(): Audio
     {
         foreach ($this->content as $content) {
             if (!$content instanceof Audio) {

src/platform/src/Result/BaseResult.php

@@ -12,7 +12,7 @@
 namespace Symfony\AI\Platform\Result;
 
 use Symfony\AI\Platform\Metadata\MetadataAwareTrait;
-use Symfony\AI\Platform\Speech\SpeechAwareTrait;
+use Symfony\AI\Platform\Speech\SpeechBagAwareTrait;
 
 /**
  * Base result of converted result classes.
@@ -23,5 +23,5 @@ abstract class BaseResult implements ResultInterface
 {
     use MetadataAwareTrait;
     use RawResultAwareTrait;
-    use SpeechAwareTrait;
+    use SpeechBagAwareTrait;
 }

src/platform/src/Result/DeferredResult.php

@@ -15,7 +15,7 @@
 use Symfony\AI\Platform\Exception\UnexpectedResultTypeException;
 use Symfony\AI\Platform\Metadata\MetadataAwareTrait;
 use Symfony\AI\Platform\ResultConverterInterface;
-use Symfony\AI\Platform\Speech\SpeechAwareTrait;
+use Symfony\AI\Platform\Speech\SpeechBagAwareTrait;
 use Symfony\AI\Platform\Vector\Vector;
 
 /**
@@ -24,7 +24,7 @@
 final class DeferredResult
 {
     use MetadataAwareTrait;
-    use SpeechAwareTrait;
+    use SpeechBagAwareTrait;
 
     private bool $isConverted = false;
     private ResultInterface $convertedResult;
@@ -56,7 +56,10 @@ public function getResult(): ResultInterface
         }
 
         $this->convertedResult->getMetadata()->set($this->getMetadata()->all());
-        $this->convertedResult->setSpeech($this->getSpeech());
+
+        foreach ($this->speechBag as $speech) {
+            $this->convertedResult->addSpeech($speech);
+        }
 
         return $this->convertedResult;
     }

src/platform/src/Result/ResultInterface.php

@@ -13,6 +13,7 @@
 
 use Symfony\AI\Platform\Metadata\Metadata;
 use Symfony\AI\Platform\Result\Exception\RawResultAlreadySetException;
+use Symfony\AI\Platform\Speech\Speech;
 
 /**
  * @author Christopher Hertel <[email protected]>
@@ -33,4 +34,8 @@ public function getRawResult(): ?RawResultInterface;
      * @throws RawResultAlreadySetException if the result is tried to be set more than once
      */
     public function setRawResult(RawResultInterface $rawResult): void;
+
+    public function addSpeech(Speech $speech): void;
+
+    public function getSpeech(string $identifier): ?Speech;
 }

src/platform/src/Speech/Speech.php

@@ -18,12 +18,19 @@
  */
 final class Speech
 {
+    /**
+     * @param string|array<mixed, mixed> $payload
+     */
     public function __construct(
         private readonly string|array $payload,
         private readonly DeferredResult $result,
+        private readonly string $identifier,
     ) {
     }
 
+    /**
+     * @return string|array<mixed, mixed>
+     */
     public function getPayload(): string|array
     {
         return $this->payload;
@@ -33,4 +40,9 @@ public function asBinary(): string
     {
         return $this->result->asBinary();
     }
+
+    public function getIdentifier(): string
+    {
+        return $this->identifier;
+    }
 }