Skip to content

Commit 4674607

Browse files
OskarStarkOskarStark
committed
bug #735 [Platform][Gemini][VertexAI] Handle inlineData (lochmueller)
This PR was merged into the main branch. Discussion ---------- [Platform][Gemini][VertexAI] Handle `inlineData` | Q | A | ------------- | --- | Bug fix? | yes | New feature? | no | Docs? | no | Issues | Fix #730 | License | MIT Fix handling of inlineData as Binary response in ResultConverter of Google Gemini. Commits ------- 2175062 [Platform][Gemini][VertexAI] Handle `inlineData`
2 parents 0d880f6 + 2175062 commit 4674607

File tree

4 files changed

+138
-3
lines changed

4 files changed

+138
-3
lines changed

‎src/platform/src/Bridge/Gemini/Gemini/ResultConverter.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use Symfony\AI\Platform\Exception\RateLimitExceededException;
1616
use Symfony\AI\Platform\Exception\RuntimeException;
1717
use Symfony\AI\Platform\Model;
18+
use Symfony\AI\Platform\Result\BinaryResult;
1819
use Symfony\AI\Platform\Result\ChoiceResult;
1920
use Symfony\AI\Platform\Result\RawHttpResult;
2021
use Symfony\AI\Platform\Result\RawResultInterface;
@@ -138,7 +139,7 @@ private function convertStream(HttpResponse $result): \Generator
138139
* }
139140
* } $choice
140141
*/
141-
private function convertChoice(array $choice): ToolCallResult|TextResult
142+
private function convertChoice(array $choice): ToolCallResult|TextResult|BinaryResult
142143
{
143144
$contentParts = $choice['content']['parts'];
144145

@@ -156,6 +157,10 @@ private function convertChoice(array $choice): ToolCallResult|TextResult
156157
return new TextResult($contentPart['text']);
157158
}
158159

160+
if (isset($contentPart['inlineData'])) {
161+
return new BinaryResult($contentPart['inlineData']['data'], $contentPart['inlineData']['mimeType'] ?? null);
162+
}
163+
159164
throw new RuntimeException(\sprintf('Unsupported finish reason "%s".', $choice['finishReason']));
160165
}
161166

‎src/platform/src/Bridge/VertexAi/Gemini/ResultConverter.php‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111

1212
namespace Symfony\AI\Platform\Bridge\VertexAi\Gemini;
1313

14+
use Symfony\AI\Platform\Exception\RateLimitExceededException;
1415
use Symfony\AI\Platform\Exception\RuntimeException;
1516
use Symfony\AI\Platform\Model as BaseModel;
17+
use Symfony\AI\Platform\Result\BinaryResult;
1618
use Symfony\AI\Platform\Result\ChoiceResult;
1719
use Symfony\AI\Platform\Result\RawHttpResult;
1820
use Symfony\AI\Platform\Result\RawResultInterface;
@@ -42,8 +44,14 @@ public function supports(BaseModel $model): bool
4244

4345
public function convert(RawResultInterface|RawHttpResult $result, array $options = []): ResultInterface
4446
{
47+
$response = $result->getObject();
48+
49+
if (429 === $response->getStatusCode()) {
50+
throw new RateLimitExceededException();
51+
}
52+
4553
if ($options['stream'] ?? false) {
46-
return new StreamResult($this->convertStream($result->getObject()));
54+
return new StreamResult($this->convertStream($response));
4755
}
4856

4957
$data = $result->getData();
@@ -125,7 +133,7 @@ private function convertStream(HttpResponse $result): \Generator
125133
* }
126134
* } $choice
127135
*/
128-
private function convertChoice(array $choice): ToolCallResult|TextResult
136+
private function convertChoice(array $choice): ToolCallResult|TextResult|BinaryResult
129137
{
130138
$contentParts = $choice['content']['parts'];
131139

@@ -143,6 +151,10 @@ private function convertChoice(array $choice): ToolCallResult|TextResult
143151
return new TextResult($contentPart['text']);
144152
}
145153

154+
if (isset($contentPart['inlineData'])) {
155+
return new BinaryResult($contentPart['inlineData']['data'], $contentPart['inlineData']['mimeType'] ?? null);
156+
}
157+
146158
throw new RuntimeException(\sprintf('Unsupported finish reason "%s".', $choice['finishReason']));
147159
}
148160

‎src/platform/tests/Bridge/Gemini/Gemini/ResultConverterTest.php‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
use PHPUnit\Framework\TestCase;
1515
use Symfony\AI\Platform\Bridge\Gemini\Gemini\ResultConverter;
1616
use Symfony\AI\Platform\Exception\RuntimeException;
17+
use Symfony\AI\Platform\Result\BinaryResult;
1718
use Symfony\AI\Platform\Result\RawHttpResult;
1819
use Symfony\AI\Platform\Result\ToolCall;
1920
use Symfony\AI\Platform\Result\ToolCallResult;
@@ -76,4 +77,59 @@ public function testReturnsToolCallEvenIfMultipleContentPartsAreGiven()
7677
$this->assertInstanceOf(ToolCall::class, $toolCall);
7778
$this->assertSame('1234', $toolCall->id);
7879
}
80+
81+
public function testConvertsInlineDataToBinaryResult()
82+
{
83+
$converter = new ResultConverter();
84+
$httpResponse = self::createMock(ResponseInterface::class);
85+
$httpResponse->method('getStatusCode')->willReturn(200);
86+
$httpResponse->method('toArray')->willReturn([
87+
'candidates' => [
88+
[
89+
'content' => [
90+
'parts' => [
91+
[
92+
'inlineData' => [
93+
'mimeType' => 'image/png',
94+
'data' => 'base64EncodedImageData',
95+
],
96+
],
97+
],
98+
],
99+
],
100+
],
101+
]);
102+
103+
$result = $converter->convert(new RawHttpResult($httpResponse));
104+
$this->assertInstanceOf(BinaryResult::class, $result);
105+
$this->assertSame('base64EncodedImageData', $result->getContent());
106+
$this->assertSame('image/png', $result->mimeType);
107+
}
108+
109+
public function testConvertsInlineDataWithoutMimeTypeToBinaryResult()
110+
{
111+
$converter = new ResultConverter();
112+
$httpResponse = self::createMock(ResponseInterface::class);
113+
$httpResponse->method('getStatusCode')->willReturn(200);
114+
$httpResponse->method('toArray')->willReturn([
115+
'candidates' => [
116+
[
117+
'content' => [
118+
'parts' => [
119+
[
120+
'inlineData' => [
121+
'data' => 'base64EncodedData',
122+
],
123+
],
124+
],
125+
],
126+
],
127+
],
128+
]);
129+
130+
$result = $converter->convert(new RawHttpResult($httpResponse));
131+
$this->assertInstanceOf(BinaryResult::class, $result);
132+
$this->assertSame('base64EncodedData', $result->getContent());
133+
$this->assertNull($result->mimeType);
134+
}
79135
}

‎src/platform/tests/Bridge/VertexAi/Gemini/ResultConverterTest.php‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
use PHPUnit\Framework\TestCase;
1515
use Symfony\AI\Platform\Bridge\VertexAi\Gemini\ResultConverter;
16+
use Symfony\AI\Platform\Result\BinaryResult;
1617
use Symfony\AI\Platform\Result\RawHttpResult;
1718
use Symfony\AI\Platform\Result\TextResult;
1819
use Symfony\AI\Platform\Result\ToolCall;
@@ -124,4 +125,65 @@ public function testItThrowsExceptionOnTimeout()
124125
$this->expectException(\RuntimeException::class);
125126
$converter->convert(new RawHttpResult($response));
126127
}
128+
129+
public function testConvertsInlineDataToBinaryResult()
130+
{
131+
$response = $this->createStub(ResponseInterface::class);
132+
$response
133+
->method('toArray')
134+
->willReturn([
135+
'candidates' => [
136+
[
137+
'content' => [
138+
'parts' => [
139+
[
140+
'inlineData' => [
141+
'mimeType' => 'image/png',
142+
'data' => 'base64EncodedImageData',
143+
],
144+
],
145+
],
146+
],
147+
],
148+
],
149+
]);
150+
151+
$resultConverter = new ResultConverter();
152+
153+
$result = $resultConverter->convert(new RawHttpResult($response));
154+
155+
$this->assertInstanceOf(BinaryResult::class, $result);
156+
$this->assertSame('base64EncodedImageData', $result->getContent());
157+
$this->assertSame('image/png', $result->mimeType);
158+
}
159+
160+
public function testConvertsInlineDataWithoutMimeTypeToBinaryResult()
161+
{
162+
$response = $this->createStub(ResponseInterface::class);
163+
$response
164+
->method('toArray')
165+
->willReturn([
166+
'candidates' => [
167+
[
168+
'content' => [
169+
'parts' => [
170+
[
171+
'inlineData' => [
172+
'data' => 'base64EncodedData',
173+
],
174+
],
175+
],
176+
],
177+
],
178+
],
179+
]);
180+
181+
$resultConverter = new ResultConverter();
182+
183+
$result = $resultConverter->convert(new RawHttpResult($response));
184+
185+
$this->assertInstanceOf(BinaryResult::class, $result);
186+
$this->assertSame('base64EncodedData', $result->getContent());
187+
$this->assertNull($result->mimeType);
188+
}
127189
}

0 commit comments

Comments
 (0)