Skip to content

Commit da90721

Browse files
julien-ncjulien-nc
authored
Merge pull request #326 from nextcloud/feat/noid/structured-translation
feat(Translate): use structured output to limit extraneous text
2 parents 8f513d7 + 8f86401 commit da90721

File tree

2 files changed

+102
-40
lines changed

2 files changed

+102
-40
lines changed

‎lib/TaskProcessing/TranslateProvider.php‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,39 @@
1919
use OCP\IL10N;
2020
use OCP\L10N\IFactory;
2121
use OCP\TaskProcessing\EShapeType;
22+
use OCP\TaskProcessing\Exception\ProcessingException;
23+
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
2224
use OCP\TaskProcessing\ISynchronousProvider;
2325
use OCP\TaskProcessing\ShapeDescriptor;
2426
use OCP\TaskProcessing\ShapeEnumValue;
2527
use OCP\TaskProcessing\TaskTypes\TextToTextTranslate;
2628
use Psr\Log\LoggerInterface;
27-
use RuntimeException;
2829

2930
class TranslateProvider implements ISynchronousProvider {
3031

32+
public const SYSTEM_PROMPT = 'You are a translations expert that ONLY outputs a valid JSON with the translated text in the following format: { "translation": "<translated text>" } .';
33+
public const JSON_RESPONSE_FORMAT = [
34+
'response_format' => [
35+
'type' => 'json_schema',
36+
'json_schema' => [
37+
'name' => 'TranslationResponse',
38+
'description' => 'A JSON object containing the translated text',
39+
'strict' => true,
40+
'schema' => [
41+
'type' => 'object',
42+
'properties' => [
43+
'translation' => [
44+
'type' => 'string',
45+
'description' => 'The translated text',
46+
],
47+
],
48+
'required' => [ 'translation' ],
49+
'additionalProperties' => false,
50+
],
51+
],
52+
],
53+
];
54+
3155
public function __construct(
3256
private OpenAiAPIService $openAiAPIService,
3357
private IAppConfig $appConfig,
@@ -144,7 +168,10 @@ public function process(?string $userId, array $input, callable $reportProgress)
144168
}
145169

146170
if (!isset($input['input']) || !is_string($input['input'])) {
147-
throw new RuntimeException('Invalid input text');
171+
throw new ProcessingException('Invalid input text');
172+
}
173+
if (empty($input['input'])) {
174+
throw new UserFacingProcessingException($this->l->t('Input text cannot be empty'));
148175
}
149176
$inputText = $input['input'];
150177

@@ -160,13 +187,14 @@ public function process(?string $userId, array $input, callable $reportProgress)
160187
try {
161188
$coreLanguages = $this->getCoreLanguagesByCode();
162189

190+
$fromLanguage = $input['origin_language'];
163191
$toLanguage = $coreLanguages[$input['target_language']] ?? $input['target_language'];
164192

165193
if ($input['origin_language'] !== 'detect_language') {
166194
$fromLanguage = $coreLanguages[$input['origin_language']] ?? $input['origin_language'];
167-
$promptStart = 'Translate from ' . $fromLanguage . ' to ' . $toLanguage . ': ';
195+
$promptStart = 'Translate the following text from ' . $fromLanguage . ' to ' . $toLanguage . ': ';
168196
} else {
169-
$promptStart = 'Translate to ' . $toLanguage . ': ';
197+
$promptStart = 'Translate the following text to ' . $toLanguage . ': ';
170198
}
171199

172200
foreach ($chunks as $chunk) {
@@ -180,33 +208,55 @@ public function process(?string $userId, array $input, callable $reportProgress)
180208
$reportProgress($progress);
181209
continue;
182210
}
183-
$prompt = $promptStart . $chunk;
211+
$prompt = $promptStart . PHP_EOL . PHP_EOL . $chunk;
184212

185213
if ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()) {
186-
$completion = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, null, null, 1, $maxTokens);
187-
$completion = $completion['messages'];
214+
$completionsObj = $this->openAiAPIService->createChatCompletion(
215+
$userId, $model, $prompt, self::SYSTEM_PROMPT, null, 1, $maxTokens, self::JSON_RESPONSE_FORMAT
216+
);
217+
$completions = $completionsObj['messages'];
188218
} else {
189-
$completion = $this->openAiAPIService->createCompletion($userId, $prompt, 1, $model, $maxTokens);
219+
$completions = $this->openAiAPIService->createCompletion(
220+
$userId, $prompt . PHP_EOL . self::SYSTEM_PROMPT . PHP_EOL . PHP_EOL, 1, $model, $maxTokens
221+
);
190222
}
191223

192224
$reportProgress($progress);
193225

194-
if (count($completion) > 0) {
195-
$completion = array_pop($completion);
196-
$result .= $completion;
197-
$cache->set($cacheKey, $completion);
226+
if (count($completions) === 0) {
227+
$this->logger->error('Empty translation response received for chunk');
198228
continue;
199229
}
200230

201-
throw new RuntimeException("Failed translate from {$fromLanguage} to {$toLanguage} for chunk");
231+
$completion = array_pop($completions);
232+
$decodedCompletion = json_decode($completion, true);
233+
if (
234+
!isset($decodedCompletion['translation'])
235+
|| !is_string($decodedCompletion['translation'])
236+
|| empty($decodedCompletion['translation'])
237+
) {
238+
$this->logger->error('Invalid translation response received for chunk', ['response' => $completion]);
239+
continue;
240+
}
241+
$result .= $decodedCompletion['translation'];
242+
$cache->set($cacheKey, $decodedCompletion['translation']);
243+
continue;
202244
}
203245

204246
$endTime = time();
205247
$this->openAiAPIService->updateExpTextProcessingTime($endTime - $startTime);
206-
return ['output' => $result];
248+
249+
if (empty(trim($result))) {
250+
throw new ProcessingException("Empty translation result from {$fromLanguage} to {$toLanguage}");
251+
}
252+
return ['output' => trim($result)];
207253

208254
} catch (Exception $e) {
209-
throw new RuntimeException("Failed translate from {$fromLanguage} to {$toLanguage}", 0, $e);
255+
throw new ProcessingException(
256+
"Failed to translate from {$fromLanguage} to {$toLanguage}: {$e->getMessage()}",
257+
$e->getCode(),
258+
$e,
259+
);
210260
}
211261
}
212262
}

‎tests/unit/Providers/OpenAiProviderTest.php‎

Lines changed: 37 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -521,53 +521,65 @@ public function testTranslationProvider(): void {
521521
$inputText = 'This is a test prompt';
522522
$n = 1;
523523
$fromLang = 'Swedish';
524-
$toLang = 'en';
524+
$toLang = 'English';
525+
$aiContent = ['translation' => 'This is a test response.'];
525526

526527
$response = '{
527-
"id": "chatcmpl-123",
528-
"object": "chat.completion",
529-
"created": 1677652288,
530-
"model": "gpt-3.5-turbo-0613",
531-
"system_fingerprint": "fp_44709d6fcb",
532-
"choices": [
533-
{
534-
"index": 0,
535-
"message": {
536-
"role": "assistant",
537-
"content": "This is a test response."
538-
},
539-
"finish_reason": "stop"
540-
}
541-
],
542-
"usage": {
543-
"prompt_tokens": 9,
544-
"completion_tokens": 12,
545-
"total_tokens": 21
546-
}
547-
}';
528+
"id": "chatcmpl-123",
529+
"object": "chat.completion",
530+
"created": 1677652288,
531+
"model": "gpt-4.1-mini",
532+
"system_fingerprint": "fp_44709d6fcb",
533+
"choices": [
534+
{
535+
"index": 0,
536+
"message": {
537+
"role": "assistant",
538+
"content": ' . json_encode(json_encode($aiContent)) . '
539+
},
540+
"finish_reason": "stop"
541+
}
542+
],
543+
"usage": {
544+
"prompt_tokens": 9,
545+
"completion_tokens": 12,
546+
"total_tokens": 21
547+
}
548+
}';
548549

549550
$url = self::OPENAI_API_BASE . 'chat/completions';
551+
$prompt = 'Translate the following text from ' . $fromLang . ' to ' . $toLang . ': ' . PHP_EOL . PHP_EOL . $inputText;
550552

551553
$options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']];
552554
$options['body'] = json_encode([
553555
'model' => Application::DEFAULT_COMPLETION_MODEL_ID,
554556
'messages' => [
555-
['role' => 'user', 'content' => 'Translate from ' . $fromLang . ' to English (US): ' . $inputText],
557+
['role' => 'system', 'content' => $translationProvider::SYSTEM_PROMPT],
558+
['role' => 'user', 'content' => $prompt],
556559
],
557560
'n' => $n,
558561
'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS,
559562
'user' => self::TEST_USER1,
563+
...$translationProvider::JSON_RESPONSE_FORMAT,
560564
]);
561565

562566
$iResponse = $this->createMock(\OCP\Http\Client\IResponse::class);
563567
$iResponse->method('getBody')->willReturn($response);
564568
$iResponse->method('getStatusCode')->willReturn(200);
565569
$iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json');
566570

567-
$this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse);
571+
$this->iClient->expects($this->once())->method('post')->with(
572+
$this->equalTo($url),
573+
$this->callback(function ($revdOptions) use ($options) {
574+
$body = json_decode($revdOptions['body'], true);
575+
$expectedBody = json_decode($options['body'], true);
576+
$this->assertEquals($expectedBody, $body);
577+
return true;
578+
}),
579+
)->willReturn($iResponse);
568580

569581
$result = $translationProvider->process(self::TEST_USER1, ['input' => $inputText, 'origin_language' => $fromLang, 'target_language' => $toLang], fn () => null);
570-
$this->assertEquals(['output' => 'This is a test response.'], $result);
582+
$this->assertEquals(['output' => $aiContent['translation']], $result);
571583

572584
// Check that token usage is logged properly
573585
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT);

0 commit comments

Comments
 (0)