feat: add audio & document input support for Gemini (#339)

valtzu · chr-hertel · commit cc38ddd9019f · 2025-06-22T21:50:49.000+02:00
diff --git a/examples/google/audio-input.php b/examples/google/audio-input.php
@@ -0,0 +1,40 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Platform\Bridge\Google\Gemini;
+use Symfony\AI\Platform\Bridge\Google\PlatformFactory;
+use Symfony\AI\Platform\Message\Content\Audio;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\Component\Dotenv\Dotenv;
+
+require_once dirname(__DIR__).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__).'/.env');
+
+if (empty($_ENV['GOOGLE_API_KEY'])) {
+    echo 'Please set the GOOGLE_API_KEY environment variable.'.\PHP_EOL;
+    exit(1);
+}
+
+$platform = PlatformFactory::create($_ENV['GOOGLE_API_KEY']);
+$model = new Gemini(Gemini::GEMINI_1_5_FLASH);
+
+$agent = new Agent($platform, $model);
+$messages = new MessageBag(
+    Message::ofUser(
+        'What is this recording about?',
+        Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'),
+    ),
+);
+$response = $agent->call($messages);
+
+echo $response->getContent().\PHP_EOL;
diff --git a/examples/google/pdf-input-binary.php b/examples/google/pdf-input-binary.php
@@ -0,0 +1,40 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Platform\Bridge\Google\Gemini;
+use Symfony\AI\Platform\Bridge\Google\PlatformFactory;
+use Symfony\AI\Platform\Message\Content\Document;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\Component\Dotenv\Dotenv;
+
+require_once dirname(__DIR__).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__).'/.env');
+
+if (empty($_ENV['GOOGLE_API_KEY'])) {
+    echo 'Please set the GOOGLE_API_KEY environment variable.'.\PHP_EOL;
+    exit(1);
+}
+
+$platform = PlatformFactory::create($_ENV['GOOGLE_API_KEY']);
+$model = new Gemini(Gemini::GEMINI_1_5_FLASH);
+
+$agent = new Agent($platform, $model);
+$messages = new MessageBag(
+    Message::ofUser(
+        Document::fromFile(dirname(__DIR__, 2).'/fixtures/document.pdf'),
+        'What is this document about?',
+    ),
+);
+$response = $agent->call($messages);
+
+echo $response->getContent().\PHP_EOL;
diff --git a/examples/google/toolcall.php b/examples/google/toolcall.php
@@ -19,8 +19,8 @@
 use Symfony\AI\Platform\Message\MessageBag;
 use Symfony\Component\Dotenv\Dotenv;
 
-require_once dirname(__DIR__, 2).'/vendor/autoload.php';
-(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
+require_once dirname(__DIR__).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__).'/.env');
 
 if (empty($_ENV['GOOGLE_API_KEY'])) {
     echo 'Please set the GOOGLE_API_KEY environment variable.'.\PHP_EOL;
diff --git a/src/platform/src/Bridge/Google/Contract/UserMessageNormalizer.php b/src/platform/src/Bridge/Google/Contract/UserMessageNormalizer.php
@@ -13,7 +13,7 @@
 
 use Symfony\AI\Platform\Bridge\Google\Gemini;
 use Symfony\AI\Platform\Contract\Normalizer\ModelContractNormalizer;
-use Symfony\AI\Platform\Message\Content\Image;
+use Symfony\AI\Platform\Message\Content\File;
 use Symfony\AI\Platform\Message\Content\Text;
 use Symfony\AI\Platform\Message\UserMessage;
 use Symfony\AI\Platform\Model;
@@ -45,7 +45,7 @@ public function normalize(mixed $data, ?string $format = null, array $context =
             if ($content instanceof Text) {
                 $parts[] = ['text' => $content->text];
             }
-            if ($content instanceof Image) {
+            if ($content instanceof File) {
                 $parts[] = ['inline_data' => [
                     'mime_type' => $content->getFormat(),
                     'data' => $content->asBase64(),
diff --git a/src/platform/src/Bridge/Google/Gemini.php b/src/platform/src/Bridge/Google/Gemini.php
@@ -33,6 +33,8 @@ public function __construct(string $name = self::GEMINI_2_PRO, array $options =
         $capabilities = [
             Capability::INPUT_MESSAGES,
             Capability::INPUT_IMAGE,
+            Capability::INPUT_AUDIO,
+            Capability::INPUT_PDF,
             Capability::OUTPUT_STREAMING,
             Capability::TOOL_CALLING,
         ];
diff --git a/src/platform/src/Bridge/Google/ModelHandler.php b/src/platform/src/Bridge/Google/ModelHandler.php
@@ -197,13 +197,13 @@ private function convertChoice(array $choice): Choice
 
     /**
      * @param array{
-     *     id: string,
+     *     id?: string,
      *     name: string,
      *     args: mixed[]
      * } $toolCall
      */
     private function convertToolCall(array $toolCall): ToolCall
     {
-        return new ToolCall($toolCall['id'], $toolCall['name'], $toolCall['args']);
+        return new ToolCall($toolCall['id'] ?? '', $toolCall['name'], $toolCall['args']);
     }
 }
diff --git a/src/platform/tests/Bridge/Google/Contract/UserMessageNormalizerTest.php b/src/platform/tests/Bridge/Google/Contract/UserMessageNormalizerTest.php
@@ -12,13 +12,16 @@
 namespace Symfony\AI\Platform\Tests\Bridge\Google\Contract;
 
 use PHPUnit\Framework\Attributes\CoversClass;
+use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Small;
 use PHPUnit\Framework\Attributes\Test;
 use PHPUnit\Framework\Attributes\UsesClass;
 use PHPUnit\Framework\TestCase;
 use Symfony\AI\Platform\Bridge\Google\Contract\UserMessageNormalizer;
 use Symfony\AI\Platform\Bridge\Google\Gemini;
 use Symfony\AI\Platform\Contract;
+use Symfony\AI\Platform\Message\Content\Audio;
+use Symfony\AI\Platform\Message\Content\Document;
 use Symfony\AI\Platform\Message\Content\File;
 use Symfony\AI\Platform\Message\Content\Image;
 use Symfony\AI\Platform\Message\Content\Text;
@@ -30,6 +33,9 @@
 #[UsesClass(UserMessage::class)]
 #[UsesClass(Text::class)]
 #[UsesClass(File::class)]
+#[UsesClass(Image::class)]
+#[UsesClass(Document::class)]
+#[UsesClass(Audio::class)]
 final class UserMessageNormalizerTest extends TestCase
 {
     #[Test]
@@ -62,22 +68,32 @@ public function normalizeTextContent(): void
         self::assertSame([['text' => 'Write a story about a magic backpack.']], $normalized);
     }
 
+    #[DataProvider('binaryContentProvider')]
     #[Test]
-    public function normalizeImageContent(): void
+    public function normalizeBinaryContent(File $content, string $expectedMimeType, string $expectedPrefix): void
     {
         $normalizer = new UserMessageNormalizer();
-        $imageContent = Image::fromFile(\dirname(__DIR__, 6).'/fixtures/image.jpg');
-        $message = new UserMessage(new Text('Tell me about this instrument'), $imageContent);
+        $message = new UserMessage(new Text('Tell me about this instrument'), $content);
 
         $normalized = $normalizer->normalize($message);
 
         self::assertCount(2, $normalized);
         self::assertSame(['text' => 'Tell me about this instrument'], $normalized[0]);
         self::assertArrayHasKey('inline_data', $normalized[1]);
-        self::assertSame('image/jpeg', $normalized[1]['inline_data']['mime_type']);
+        self::assertSame($expectedMimeType, $normalized[1]['inline_data']['mime_type']);
         self::assertNotEmpty($normalized[1]['inline_data']['data']);
 
-        // Verify that the base64 data string starts correctly for a JPEG
-        self::assertStringStartsWith('/9j/', $normalized[1]['inline_data']['data']);
+        // Verify that the base64 data string starts correctly
+        self::assertStringStartsWith($expectedPrefix, $normalized[1]['inline_data']['data']);
+    }
+
+    /**
+     * @return iterable<string, array{0: File, 1: string, 2: string}>
+     */
+    public static function binaryContentProvider(): iterable
+    {
+        yield 'image' => [Image::fromFile(\dirname(__DIR__, 6).'/fixtures/image.jpg'), 'image/jpeg', '/9j/'];
+        yield 'document' => [Document::fromFile(\dirname(__DIR__, 6).'/fixtures/document.pdf'), 'application/pdf', 'JVBE'];
+        yield 'audio' => [Audio::fromFile(\dirname(__DIR__, 6).'/fixtures/audio.mp3'), 'audio/mpeg', 'SUQz'];
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -197,13 +197,13 @@ private function convertChoice(array $choice): Choice`
`197`	`197`
`198`	`198`	`/**`
`199`	`199`	`* @param array{`
`200`		`- * id: string,`
	`200`	`+ * id?: string,`
`201`	`201`	`* name: string,`
`202`	`202`	`* args: mixed[]`
`203`	`203`	`* } $toolCall`
`204`	`204`	`*/`
`205`	`205`	`private function convertToolCall(array $toolCall): ToolCall`
`206`	`206`	`{`
`207`		`- return new ToolCall($toolCall['id'], $toolCall['name'], $toolCall['args']);`
	`207`	`+ return new ToolCall($toolCall['id'] ?? '', $toolCall['name'], $toolCall['args']);`
`208`	`208`	`}`
`209`	`209`	`}`