Merge pull request #340 from lingarr-translate/add_retry_mechanism

added retry mechanism to libretranslate and public translate services

Author: rowanfuchs

Lingarr.Client/src/components/common/TranslationAction.vue

@@ -5,7 +5,7 @@
             @click="router.push({ name: 'translation-detail', params: { id: item.id } })">
             <EyeOnIcon class="h-5 w-5" />
         </button>
-        <div class="flex w-[3rem] items-center gap-2">
+        <div class="flex w-12 items-center gap-2">
             <LoaderCircleIcon v-if="loading" class="h-5 w-5 animate-spin" />
             <button
                 v-else-if="inProgress"

Lingarr.Client/src/components/common/TranslationStatus.vue

@@ -1,13 +1,23 @@
 <template>
     <span :title="translationStatus.toString()">
-        {{ translationStatus }}
+        {{ displayStatus }}
     </span>
 </template>
 
 <script setup lang="ts">
-import { TranslationStatus } from '@/ts'
+import { computed } from 'vue'
+import { TranslationStatus, TRANSLATION_STATUS } from '@/ts'
 
-defineProps<{
+const props = defineProps<{
     translationStatus: TranslationStatus
 }>()
+
+const displayStatus = computed(() => {
+    switch (props.translationStatus) {
+        case TRANSLATION_STATUS.INPROGRESS:
+            return 'In Progress'
+        default:
+            return props.translationStatus
+    }
+})
 </script>

Lingarr.Client/src/components/features/settings/IntegrationSettings.vue

@@ -65,7 +65,7 @@ const saveNotification = ref<InstanceType<typeof SaveNotification> | null>(null)
 const settingsStore = useSettingStore()
 
 const radarrApiKey = computed({
-    get: (): string => settingsStore.getEncryptedSetting(ENCRYPTED_SETTINGS.RADARR_API_KEY) as string,
+    get: (): string | null => settingsStore.getEncryptedSetting(ENCRYPTED_SETTINGS.RADARR_API_KEY) ?? null,
     set: (newValue: string): void => {
         settingsStore.updateEncryptedSetting(ENCRYPTED_SETTINGS.RADARR_API_KEY, newValue, isValid.radarrApiKey)
         if (isValid.radarrApiKey) {
@@ -74,7 +74,7 @@ const radarrApiKey = computed({
     }
 })
 const sonarrApiKey = computed({
-    get: (): string => settingsStore.getEncryptedSetting(ENCRYPTED_SETTINGS.SONARR_API_KEY) as string,
+    get: (): string | null => settingsStore.getEncryptedSetting(ENCRYPTED_SETTINGS.SONARR_API_KEY) ?? null,
     set: (newValue: string): void => {
         settingsStore.updateEncryptedSetting(ENCRYPTED_SETTINGS.SONARR_API_KEY, newValue, isValid.sonarrApiKey)
         if (isValid.sonarrApiKey) {
@@ -83,7 +83,7 @@ const sonarrApiKey = computed({
     }
 })
 const radarrUrl = computed({
-    get: (): string => settingsStore.getSetting(SETTINGS.RADARR_URL) as string,
+    get: (): string | null => (settingsStore.getSetting(SETTINGS.RADARR_URL) as string) ?? null,
     set: (newValue: string): void => {
         settingsStore.updateSetting(SETTINGS.RADARR_URL, newValue, isValid.radarrUrl)
         if (isValid.radarrUrl) {
@@ -92,7 +92,7 @@ const radarrUrl = computed({
     }
 })
 const sonarrUrl = computed({
-    get: (): string => settingsStore.getSetting(SETTINGS.SONARR_URL) as string,
+    get: (): string | null => (settingsStore.getSetting(SETTINGS.SONARR_URL) as string) ?? null,
     set: (newValue: string): void => {
         settingsStore.updateSetting(SETTINGS.SONARR_URL, newValue, isValid.sonarrUrl)
         if (isValid.sonarrUrl) {

Lingarr.Client/src/components/features/settings/TranslationSettings.vue

@@ -1,22 +1,35 @@
 <template>
     <CardComponent title="Translation Request">
         <template #description>
-            Modify translation request settings by changing batch size or retry options.
+            Modify translation request settings by changing retry options or batch size if available in the service.
         </template>
         <template #content>
             <SaveNotification ref="saveNotification" />
 
-            <div class="flex flex-col space-x-2">
-                <span class="font-semibold">Use batch translation</span>
-                Process multiple subtitle lines together in batches to improve translation
-                efficiency and context awareness. Note that single-line translations with context
-                are still more reliable and of higher quality.
-            </div>
-            <ToggleButton v-model="useBatchTranslation">
-                <span class="text-sm font-medium text-primary-content">
-                    {{ useBatchTranslation == 'true' ? 'Enabled' : 'Disabled' }}
-                </span>
-            </ToggleButton>
+            <template
+                v-if="
+                [
+                    SERVICE_TYPE.ANTHROPIC,
+                    SERVICE_TYPE.DEEPSEEK,
+                    SERVICE_TYPE.GEMINI,
+                    SERVICE_TYPE.LOCALAI,
+                    SERVICE_TYPE.OPENAI
+                ].includes(
+                    serviceType as 'openai' | 'anthropic' | 'localai' | 'gemini' | 'deepseek'
+                )
+            ">
+                <div class="flex flex-col space-x-2">
+                    <span class="font-semibold">Use batch translation</span>
+                    Process multiple subtitle lines together in batches to improve translation
+                    efficiency and context awareness. Note that single-line translations with context
+                    are still more reliable and of higher quality.
+                </div>
+                <ToggleButton v-model="useBatchTranslation">
+                    <span class="text-sm font-medium text-primary-content">
+                        {{ useBatchTranslation == 'true' ? 'Enabled' : 'Disabled' }}
+                    </span>
+                </ToggleButton>
+            </template>
             <InputComponent
                 v-if="useBatchTranslation == 'true'"
                 v-model="maxBatchSize"
@@ -56,7 +69,7 @@
 <script setup lang="ts">
 import { computed, ref, reactive } from 'vue'
 import { useSettingStore } from '@/store/setting'
-import { INPUT_VALIDATION_TYPE, SETTINGS } from '@/ts'
+import { INPUT_VALIDATION_TYPE, SERVICE_TYPE, SETTINGS } from '@/ts'
 import CardComponent from '@/components/common/CardComponent.vue'
 import SaveNotification from '@/components/common/SaveNotification.vue'
 import InputComponent from '@/components/common/InputComponent.vue'
@@ -70,6 +83,7 @@ const isValid = reactive({
     retryDelay: true,
     retryDelayMultiplier: true
 })
+const serviceType = computed(() => settingsStore.getSetting(SETTINGS.SERVICE_TYPE))
 
 const useBatchTranslation = computed({
     get: (): string => settingsStore.getSetting(SETTINGS.USE_BATCH_TRANSLATION) as string,

Lingarr.Client/src/components/features/settings/WebhookInstructions.vue

@@ -12,11 +12,11 @@
                     and use this URL:
                 </span>
                 <span class="font-semibold">Radarr</span>
-                <code class="bg-accent/20 mt-1 block rounded p-2 text-sm">
+                <code class="bg-accent/20 mt-1 block rounded p-2 text-sm overflow-x-auto">
                     {{ webhookUrl }}/api/webhook/radarr
                 </code>
                 <span class="font-semibold">Sonarr</span>
-                <code class="bg-accent/20 mt-1 block rounded p-2 text-sm">
+                <code class="bg-accent/20 mt-1 block rounded p-2 text-sm overflow-x-auto">
                     {{ webhookUrl }}/api/webhook/sonarr
                 </code>
             </div>

Lingarr.Client/src/pages/TranslationDetailPage.vue

@@ -163,7 +163,9 @@ const latestPosition = ref<number>(0)
 const reversedLines = computed(() => (detail.value ? [...detail.value.lines].reverse() : []))
 
 const showProgress = computed(() => {
-    if (!detail.value) return false
+    if (!detail.value) {
+        return false
+    }
     const status = detail.value.status
     return (
         (status === TRANSLATION_STATUS.INPROGRESS && progress.value > 0) ||

Lingarr.Client/src/pages/settings/ServicesPage.vue

@@ -2,28 +2,11 @@
     <div
         class="grid grid-flow-row auto-rows-max grid-cols-1 gap-4 p-4 md:grid-cols-2 xl:grid-cols-2 2xl:grid-cols-3">
         <ServicesSettings />
-        <TranslationSettings
-            v-if="
-                [
-                    SERVICE_TYPE.ANTHROPIC,
-                    SERVICE_TYPE.DEEPSEEK,
-                    SERVICE_TYPE.GEMINI,
-                    SERVICE_TYPE.LOCALAI,
-                    SERVICE_TYPE.OPENAI
-                ].includes(
-                    serviceType as 'openai' | 'anthropic' | 'localai' | 'gemini' | 'deepseek'
-                )
-            " />
+        <TranslationSettings />
     </div>
 </template>
 
 <script setup lang="ts">
-import { computed } from 'vue'
-import { SETTINGS, SERVICE_TYPE } from '@/ts'
-import { useSettingStore } from '@/store/setting'
 import ServicesSettings from '@/components/features/settings/ServicesSettings.vue'
 import TranslationSettings from '@/components/features/settings/TranslationSettings.vue'
-
-const settingsStore = useSettingStore()
-const serviceType = computed(() => settingsStore.getSetting(SETTINGS.SERVICE_TYPE))
 </script>

Lingarr.Server/Services/SettingService.cs

@@ -89,19 +89,20 @@ public async Task<Dictionary<string, string>> GetSettings(IEnumerable<string> ke
             }
         }
 
-        if (keysToFetch.Any())
+        if (!keysToFetch.Any())
         {
-            var dbSettings = await _dbContext.Settings
-                .Where(s => keysToFetch.Contains(s.Key))
-                .ToListAsync();
+            return result;
+        }
+        var dbSettings = await _dbContext.Settings
+            .Where(s => keysToFetch.Contains(s.Key))
+            .ToListAsync();
 
-            foreach (var setting in dbSettings)
-            {
-                result[setting.Key] = setting.Value;
-                _cache.Set(setting.Key, setting.Value, _cacheOptions);
-            }
+        foreach (var setting in dbSettings)
+        {
+            result[setting.Key] = setting.Value;
+            _cache.Set(setting.Key, setting.Value, _cacheOptions);
         }
-        
+
         return result;
     }
 

Lingarr.Server/Services/Translation/GTranslatorService.cs

@@ -1,5 +1,6 @@
-using System.Net;
+using System.Net;
 using GTranslate.Translators;
+using Lingarr.Core.Configuration;
 using Lingarr.Server.Exceptions;
 using Lingarr.Server.Interfaces.Services;
 using Lingarr.Server.Services.Translation.Base;
@@ -9,6 +10,13 @@ namespace Lingarr.Server.Services.Translation;
 public class GTranslatorService<T> : BaseLanguageService where T : ITranslator
 {
     private readonly T _translator;
+    private bool _initialized;
+    private readonly SemaphoreSlim _initLock = new(1, 1);
+
+    // retry settings
+    private int _maxRetries;
+    private TimeSpan _retryDelay;
+    private int _retryDelayMultiplier;
 
     /// <inheritdoc />
     public override string? ModelName => null;
@@ -22,49 +30,85 @@ public GTranslatorService(
     {
         _translator = translator;
     }
-    
+
+    private async Task InitializeAsync()
+    {
+        if (_initialized) return;
+
+        try
+        {
+            await _initLock.WaitAsync();
+            if (_initialized) return;
+
+            var settings = await _settings.GetSettings([
+                SettingKeys.Translation.MaxRetries,
+                SettingKeys.Translation.RetryDelay,
+                SettingKeys.Translation.RetryDelayMultiplier
+            ]);
+
+            _maxRetries = int.TryParse(settings[SettingKeys.Translation.MaxRetries], out var maxRetries)
+                ? maxRetries
+                : 3;
+
+            var retryDelaySeconds = int.TryParse(settings[SettingKeys.Translation.RetryDelay], out var delaySeconds)
+                ? delaySeconds
+                : 2;
+            _retryDelay = TimeSpan.FromSeconds(retryDelaySeconds);
+
+            _retryDelayMultiplier = int.TryParse(settings[SettingKeys.Translation.RetryDelayMultiplier], out var multiplier)
+                ? multiplier
+                : 2;
+
+            _initialized = true;
+        }
+        finally
+        {
+            _initLock.Release();
+        }
+    }
+
     /// <inheritdoc />
     public override async Task<string> TranslateAsync(
-        string text, 
-        string sourceLanguage, 
+        string text,
+        string sourceLanguage,
         string targetLanguage,
-        List<string>? contextLinesBefore, 
-        List<string>? contextLinesAfter, 
+        List<string>? contextLinesBefore,
+        List<string>? contextLinesAfter,
         CancellationToken cancellationToken)
     {
+        await InitializeAsync();
+
         using var retry = new CancellationTokenSource();
         using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, retry.Token);
-        
-        const int maxRetries = 5;
-        var delay = TimeSpan.FromSeconds(1);
-        var maxDelay = TimeSpan.FromSeconds(32);
-        for (var attempt = 1; attempt <= maxRetries; attempt++)
+
+        var delay = _retryDelay;
+        for (var attempt = 1; attempt <= _maxRetries + 1; attempt++)
         {
             try
             {
                 var result = await _translator.TranslateAsync(
-                    text, 
-                    targetLanguage, 
-                    sourceLanguage)
+                        text,
+                        targetLanguage,
+                        sourceLanguage)
                     .WaitAsync(linked.Token)
                     .ConfigureAwait(false);
-                
+
                 return result.Translation;
             }
             catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.TooManyRequests or HttpStatusCode.ServiceUnavailable)
             {
-                if (attempt == maxRetries)
+                if (attempt > _maxRetries)
                 {
                     _logger.LogError(ex, "Max retries exhausted ({StatusCode}) for text: {Text}", ex.StatusCode, text);
                     throw new TranslationException($"Retry limit reached after {ex.StatusCode}.", ex);
                 }
 
                 _logger.LogWarning(
                     "{ServiceName} received {StatusCode}. Retrying in {Delay}... (Attempt {Attempt}/{MaxRetries})",
-                    "GTranslator", ex.StatusCode, delay, attempt, maxRetries);
+                    "GTranslator", ex.StatusCode, delay, attempt, _maxRetries);
 
                 await Task.Delay(delay, linked.Token).ConfigureAwait(false);
-                delay = TimeSpan.FromTicks(Math.Min(delay.Ticks * 2, maxDelay.Ticks));
+                delay = TimeSpan.FromTicks(delay.Ticks * _retryDelayMultiplier);
             }
             catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
@@ -79,4 +123,4 @@ public override async Task<string> TranslateAsync(
 
         throw new TranslationException("Translation failed after maximum retry attempts.");
     }
-}
+}