feat: new providers, supported languages

Author: dangeredwolf

.env.example

@@ -1,2 +1,3 @@
 DEEPLX_CLOUDFLARE_URL=""
-DEEPLX_VERCEL_URL=""
+DEEPLX_VERCEL_URL=""
+PORT=3220

README.md

@@ -1,12 +1,14 @@
 # 🌍 Polyglot - Scalable Translation Service
 
-A fast, reliable, and modular translation API that supports multiple translation providers with automatic fallback mechanisms. It uses a mixture of free undocumented APIs as well as official ones requiring API keys. It's been designed to be scalable to meet FxEmbed's needs, intended to run across many web hosting providers behind load balancing to have the highest rate limits possible across different translation services.
+A fast, reliable, and modular translation API designed to meet the scale required by FxEmbed.
 
-## ✨ Features
+## 🔄 Native Multi-Provider Architecture
 
-- 🔄 **Multi-Provider Architecture**: Google Translate, DeepLX, Bing Translate so far, more to come
+We support both scraping free translations from popular services like Google Translate, Bing Translate, and DeepL, as well as paid APIs such as Azure, DeepL API
+- 🎯 **Dynamic Selection**: Chooses between providers based on target language and availability
 - ⚖️ **Load Balancing and Rate Limit Leveling**: Distributes requests across providers
-- 🛡️ **Automatic Fallback**: If one provider fails, automatically tries others
+- 🛡️ **Automatic Failover**: If one provider fails, automatically tries others
+- 🐍 **Designed to Scale**: Use higher rate limits for free services by scaling across servers and network providers
 
 ## 🚀 Quick Start
 
@@ -30,6 +32,32 @@ bun run index.ts
 
 The API will be available at `http://localhost:3000`
 
+### Optional: Configure Paid APIs
+
+Relying on free services alone is not ideal since requests can be throttled or blocked (DeepL in particular is very aggressive at this). So we support a variety of paid translation providers, which luckily have free tiers:
+Azure AI Translator - 2M characters free per month
+DeepL - 500K characters free per month
+AWS - 2M characters free per month, 12 months only
+
+```
+# Azure AI Translator 
+AZURE_TRANSLATOR_KEY="your_azure_key"
+AZURE_TRANSLATOR_REGION="eastus"
+
+# DeepL Official API  
+DEEPL_API_KEY="your_deepl_key"
+
+# DeepLX Cloudflare Worker (if deployed)
+DEEPLX_CLOUDFLARE_URL="https://deeplx.example.workers.dev"
+
+# AWS Translate
+AWS_ACCESS_KEY_ID="your_access_key_id"
+AWS_SECRET_ACCESS_KEY="your_access_key"
+AWS_REGION="your_region" # optional, defaults to us-east-1
+```
+
+**Note**: Free providers (Google Translate, Bing, DeepLX) work without configuration. Paid APIs are only used if environment variables are provided.
+
 ## 📖 API Usage
 
 ### Translate Text
@@ -59,12 +87,12 @@ The API will be available at `http://localhost:3000`
 
 ```bash
 # Basic translation, auto detect language
-curl -X POST http://localhost:3000/translate \
+curl -X POST http://localhost:3220/translate \
   -H "Content-Type: application/json" \
   -d '{"text": "Hello, world!", "target_lang": "es"}'
 
 # With source language specified
-curl -X POST http://localhost:3000/translate \
+curl -X POST http://localhost:3220/translate \
   -H "Content-Type: application/json" \
   -d '{"text": "Bonjour le monde", "source_lang": "fr", "target_lang": "en"}'
 ```

index.ts

@@ -4,7 +4,7 @@ import type { TranslationRequest } from './providers/index.js';
 const translationService = new TranslationService();
 
 const server = Bun.serve({
-  port: 3000,
+  port: process.env.PORT ? parseInt(process.env.PORT) : 3220,
   async fetch(req) {
     const url = new URL(req.url);
 
@@ -52,4 +52,4 @@ const server = Bun.serve({
   },
 });
 
-console.log(`🚀 Server running at http://localhost:${server.port}/`);
+console.log(`🗣️ Ready to translate (http://localhost:${server.port})`);

providers/aws.ts

@@ -0,0 +1,181 @@
+import { TranslationProvider } from './types.js';
+import type { TranslationResponse } from './types.js';
+
+interface AWSTranslateResponse {
+  TranslatedText: string;
+  SourceLanguageCode: string;
+  TargetLanguageCode: string;
+}
+
+export class AWSProvider extends TranslationProvider {
+  name = 'aws';
+
+  // AWS Translate supported languages
+  // https://docs.aws.amazon.com/translate/latest/dg/what-is-languages.html
+  private supportedLanguages = new Set([
+    'af', 'sq', 'am', 'ar', 'hy', 'az', 'bn', 'bs', 'bg', 'ca', 'zh', 'zh-tw', 'hr', 'cs', 'da', 'fa-af', 
+    'nl', 'en', 'et', 'fa', 'tl', 'fi', 'fr', 'fr-ca', 'ka', 'de', 'el', 'gu', 'ht', 'ha', 'he', 'hi', 
+    'hu', 'is', 'id', 'ga', 'it', 'ja', 'kn', 'kk', 'ko', 'lv', 'lt', 'mk', 'ms', 'ml', 'mt', 'mr', 'mn', 
+    'no', 'ps', 'pl', 'pt', 'pt-pt', 'pa', 'ro', 'ru', 'sr', 'si', 'sk', 'sl', 'so', 'es', 'es-mx', 'sw', 
+    'sv', 'ta', 'te', 'th', 'tr', 'uk', 'ur', 'uz', 'vi', 'cy'
+  ]);
+
+  private async sign(stringToSign: string, key: string): Promise<string> {
+    const encoder = new TextEncoder();
+    const keyData = encoder.encode(key);
+    const dataToSign = encoder.encode(stringToSign);
+    
+    const cryptoKey = await crypto.subtle.importKey(
+      'raw',
+      keyData,
+      { name: 'HMAC', hash: 'SHA-256' },
+      false,
+      ['sign']
+    );
+    
+    const signature = await crypto.subtle.sign('HMAC', cryptoKey, dataToSign);
+    return Array.from(new Uint8Array(signature))
+      .map(b => b.toString(16).padStart(2, '0'))
+      .join('');
+  }
+
+  // 💀
+  // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html
+  private async createAuthHeader(
+    method: string,
+    url: string,
+    headers: Record<string, string>,
+    payload: string,
+    accessKey: string,
+    secretKey: string,
+    region: string
+  ): Promise<string> {
+    const urlObj = new URL(url);
+    const host = urlObj.hostname;
+    const path = urlObj.pathname;
+    
+    const timestamp = new Date().toISOString().replace(/[:\-]|\.\d{3}/g, '');
+    const date = timestamp.substr(0, 8);
+    
+    // Create canonical request
+    const canonicalHeaders = Object.entries(headers)
+      .map(([key, value]) => `${key.toLowerCase()}:${value}`)
+      .sort()
+      .join('\n') + '\n';
+    
+    const signedHeaders = Object.keys(headers)
+      .map(key => key.toLowerCase())
+      .sort()
+      .join(';');
+    
+    const payloadHash = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(payload));
+    const payloadHashHex = Array.from(new Uint8Array(payloadHash))
+      .map(b => b.toString(16).padStart(2, '0'))
+      .join('');
+    
+    const canonicalRequest = [
+      method,
+      path,
+      '', // query string (empty for POST)
+      canonicalHeaders,
+      signedHeaders,
+      payloadHashHex
+    ].join('\n');
+    
+    // Create string to sign
+    const algorithm = 'AWS4-HMAC-SHA256';
+    const credentialScope = `${date}/${region}/translate/aws4_request`;
+    const canonicalRequestHash = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(canonicalRequest));
+    const canonicalRequestHashHex = Array.from(new Uint8Array(canonicalRequestHash))
+      .map(b => b.toString(16).padStart(2, '0'))
+      .join('');
+    
+    const stringToSign = [
+      algorithm,
+      timestamp,
+      credentialScope,
+      canonicalRequestHashHex
+    ].join('\n');
+    
+    // Calculate signature
+    const kDate = await this.sign(date, `AWS4${secretKey}`);
+    const kRegion = await this.sign(region, kDate);
+    const kService = await this.sign('translate', kRegion);
+    const kSigning = await this.sign('aws4_request', kService);
+    const signature = await this.sign(stringToSign, kSigning);
+    
+    return `${algorithm} Credential=${accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
+  }
+
+  async translate(text: string, targetLang: string, sourceLang?: string): Promise<TranslationResponse> {
+    const accessKey = process.env.AWS_ACCESS_KEY_ID;
+    const secretKey = process.env.AWS_SECRET_ACCESS_KEY;
+    const region = process.env.AWS_REGION || 'us-east-1';
+
+    if (!accessKey || !secretKey) {
+      throw new Error('AWS credentials not configured');
+    }
+
+    const url = `https://translate.${region}.amazonaws.com/`;
+    const payload = JSON.stringify({
+      Text: text,
+      SourceLanguageCode: sourceLang || 'auto',
+      TargetLanguageCode: targetLang
+    });
+
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/x-amz-json-1.1',
+      'X-Amz-Target': 'AWSShineFrontendService_20170701.TranslateText',
+      'Host': `translate.${region}.amazonaws.com`,
+      'X-Amz-Date': new Date().toISOString().replace(/[:\-]|\.\d{3}/g, '')
+    };
+
+    const authHeader = await this.createAuthHeader(
+      'POST',
+      url,
+      headers,
+      payload,
+      accessKey,
+      secretKey,
+      region
+    );
+
+    headers['Authorization'] = authHeader;
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: payload
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`AWS Translate API error (${response.status}): ${errorText}`);
+    }
+
+    const data = await response.json() as AWSTranslateResponse;
+
+    if (!data.TranslatedText) {
+      throw new Error('Invalid response from AWS Translate API');
+    }
+
+    return {
+      text: data.TranslatedText,
+      source_lang: sourceLang || data.SourceLanguageCode || 'auto',
+      target_lang: targetLang,
+      provider: this.name
+    };
+  }
+
+  supportsLanguage(languageCode: string): boolean {
+    const normalized = languageCode.toLowerCase();
+    // Handle common variations
+    if (normalized === 'zh-cn') return this.supportedLanguages.has('zh');
+    if (normalized === 'zh-tw') return this.supportedLanguages.has('zh-tw');
+    return this.supportedLanguages.has(normalized);
+  }
+
+  isAvailable(): boolean {
+    return !!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);
+  }
+}

providers/azure.ts

@@ -0,0 +1,86 @@
+import { TranslationProvider } from './types.js';
+import type { TranslationResponse } from './types.js';
+
+interface AzureTranslateResponse {
+  translations: Array<{
+    text: string;
+    to: string;
+  }>;
+  detectedLanguage?: {
+    language: string;
+    score: number;
+  };
+}
+
+export class AzureProvider extends TranslationProvider {
+  name = 'azure';
+
+  // Azure Translator supported languages
+  // https://learn.microsoft.com/en-us/azure/ai-services/translator/language-support
+  private supportedLanguages = new Set([
+    'af', 'am', 'ar', 'as', 'az', 'ba', 'bg', 'bn', 'bo', 'bs', 'ca', 'cs', 'cy', 'da', 'de', 'dsb', 'dv', 
+    'el', 'en', 'es', 'et', 'eu', 'fa', 'fi', 'fj', 'fo', 'fr', 'fr-ca', 'ga', 'gl', 'gom', 'gu', 'ha', 
+    'he', 'hi', 'hr', 'hsb', 'ht', 'hu', 'hy', 'id', 'ig', 'ikt', 'is', 'it', 'iu', 'iu-latn', 'ja', 'ka', 
+    'kk', 'km', 'kmr', 'kn', 'ko', 'ks', 'ku', 'ky', 'ln', 'lo', 'lt', 'lug', 'lv', 'lzh', 'mai', 'mg', 
+    'mi', 'mk', 'ml', 'mn-cyrl', 'mn-mong', 'mni', 'mr', 'ms', 'mt', 'mww', 'my', 'nb', 'ne', 'nl', 'nso', 
+    'nya', 'or', 'otq', 'pa', 'pl', 'prs', 'ps', 'pt', 'pt-pt', 'ro', 'ru', 'run', 'rw', 'sd', 'si', 'sk', 
+    'sl', 'sm', 'sn', 'so', 'sq', 'sr-cyrl', 'sr-latn', 'st', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'ti', 
+    'tk', 'tlh-latn', 'tlh-piqd', 'tn', 'to', 'tr', 'tt', 'ty', 'ug', 'uk', 'ur', 'uz', 'vi', 'xh', 'yo', 
+    'yua', 'yue', 'zh-hans', 'zh-hant', 'zu'
+  ]);
+
+  async translate(text: string, targetLang: string, sourceLang?: string): Promise<TranslationResponse> {
+    const apiKey = process.env.AZURE_TRANSLATOR_KEY;
+    const region = process.env.AZURE_TRANSLATOR_REGION || 'global';
+    const endpoint = process.env.AZURE_TRANSLATOR_ENDPOINT || 'https://api.cognitive.microsofttranslator.com';
+
+    if (!apiKey) {
+      throw new Error('Azure Translator API key not configured');
+    }
+
+    const url = `${endpoint}/translate?api-version=3.0&to=${targetLang}${sourceLang ? `&from=${sourceLang}` : ''}`;
+    
+    const headers: Record<string, string> = {
+      'Ocp-Apim-Subscription-Key': apiKey,
+      'Content-Type': 'application/json',
+    };
+
+    // Add region header if not global
+    if (region !== 'global') {
+      headers['Ocp-Apim-Subscription-Region'] = region;
+    }
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify([{ text }])
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`Azure Translator API error (${response.status}): ${errorText}`);
+    }
+
+    const data = await response.json() as AzureTranslateResponse[];
+    const result = data[0];
+
+    if (!result?.translations?.[0]) {
+      throw new Error('Invalid response from Azure Translator API');
+    }
+
+    return {
+      text: result.translations[0].text,
+      source_lang: sourceLang || result.detectedLanguage?.language || 'auto',
+      target_lang: targetLang,
+      provider: this.name
+    };
+  }
+
+  supportsLanguage(languageCode: string): boolean {
+    return this.supportedLanguages.has(languageCode.toLowerCase());
+  }
+
+  isAvailable(): boolean {
+    return !!process.env.AZURE_TRANSLATOR_KEY;
+  }
+}

providers/bing.ts

@@ -1,10 +1,15 @@
 import { translate } from 'bing-translate-api';
+// @ts-ignore - Import JSON file directly
+import bingLanguages from 'bing-translate-api/src/lang.json' assert { type: 'json' };
 import { TranslationProvider } from './types.js';
 import type { TranslationResponse } from './types.js';
 
 export class BingTranslateProvider extends TranslationProvider {
   name = 'bing';
 
+  // Dynamic language support from Bing Translate API package
+  private supportedLanguages = new Set(Object.keys(bingLanguages));
+
   async translate(text: string, targetLang: string, sourceLang?: string): Promise<TranslationResponse> {
     // Use null for auto-detect if sourceLang is not provided
     const result = await translate(text, sourceLang || null, targetLang);
@@ -20,4 +25,13 @@ export class BingTranslateProvider extends TranslationProvider {
       provider: this.name
     };
   }
+
+  supportsLanguage(languageCode: string): boolean {
+    return this.supportedLanguages.has(languageCode.toLowerCase());
+  }
+
+  isAvailable(): boolean {
+    // Always available since we don't need API keys
+    return true;
+  }
 }