feat: setup translations using LLM

Author: alexmercerind

.github/scripts/translate.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Dict, Any, Set
+
+
+def load_json(file_path: Path) -> Dict[str, Any]:
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+
+def save_json(file_path: Path, data: Dict[str, Any]) -> None:
+    with open(file_path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+        f.write('\n')
+
+
+def get_changed_keys(en_file: Path) -> Set[str]:
+    # Get git diff to find which keys were added or modified in en_US.json.
+    print("Getting git diff...", flush=True)
+    
+    try:
+        result = subprocess.run(
+            ['git', 'diff', 'HEAD~1', 'HEAD', '--', str(en_file)],
+            capture_output=True,
+            text=True,
+            check=False,
+            cwd=en_file.parent.parent
+        )
+        
+        print(f"Git diff return code: {result.returncode}", flush=True)
+        
+        if result.returncode != 0:
+            print(f"Git diff error: {result.stderr}", flush=True)
+            sys.exit(1)
+        
+        if not result.stdout.strip():
+            print("No diff found - file unchanged", flush=True)
+            return set()
+        
+        # Parse diff output to extract changed keys.
+        changed_keys = set()
+        for line in result.stdout.split('\n'):
+            if line.startswith('+') and not line.startswith('+++'):
+                content = line[1:].strip()
+                if content.startswith('"') and '":' in content:
+                    try:
+                        key = content.split('"')[1]
+                        changed_keys.add(key)
+                    except IndexError:
+                        continue
+        
+        return changed_keys
+        
+    except Exception as e:
+        print(f"Exception in get_changed_keys: {e}", flush=True)
+        sys.exit(1)
+
+
+def translate_keys(keys_dict: Dict[str, str], target_language: str) -> Dict[str, str]:
+    # Use LLM to translate English strings to target language.
+    prompt = f"""You are a professional translator. Translate the following JSON object from English to {target_language}.
+
+IMPORTANT RULES:
+1. Keep all JSON keys EXACTLY the same (do not translate keys)
+2. Only translate the VALUES
+3. Preserve any special formatting like quotes (\"\"), placeholders (\"M\", \"N\", \"X\", \"ENTRY\", \"PLAYLIST\", etc.)
+4. Maintain the same meaning, punctuation, capitalization, structure and formatting
+5. Return ONLY the translated JSON object, no additional text
+6. Ensure the output is valid JSON
+7. Try to keep the same string length as the original string (if possible)
+
+Input JSON:
+{json.dumps(keys_dict, ensure_ascii=False, indent=2)}"""
+
+    print(f"Calling LLM...", flush=True)
+    
+    try:
+        result = subprocess.run(
+            ['llm', '-m', 'github/gpt-4o'],
+            input=prompt,
+            capture_output=True,
+            text=True,
+            check=False
+        )
+        
+        print(f"LLM returned with code {result.returncode}", flush=True)
+        
+        if result.returncode != 0:
+            print(f"Error: {result.stderr}", flush=True)
+            return keys_dict
+        
+        content = result.stdout.strip()
+        
+        if not content:
+            print(f"Empty response from LLM", flush=True)
+            return keys_dict
+        
+        # Strip markdown code block formatting if present.
+        if content.startswith('```'):
+            content = content.split('```')[1]
+            if content.startswith('json'):
+                content = content[4:]
+            content = content.split('```')[0].strip()
+        
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError as e:
+            print(f"JSON error: {e}", flush=True)
+            print(f"Content: {content[:200]}...", flush=True)
+            return keys_dict
+    except Exception as e:
+        print(f"Exception calling LLM: {e}", flush=True)
+        return keys_dict
+
+
+def main():
+    print("Starting translation script...", flush=True)
+    
+    # Setup paths.
+    script_dir = Path(__file__).parent
+    project_root = script_dir.parent.parent
+    localizations_dir = project_root / "localizations"
+    index_file = project_root / "index.json"
+    en_file = localizations_dir / "en_US.json"
+    
+    print(f"Paths:", flush=True)
+    print(f"  project_root: {project_root}", flush=True)
+    print(f"  en_file: {en_file}", flush=True)
+    
+    if not en_file.exists():
+        print(f"Error: {en_file} not found", flush=True)
+        sys.exit(1)
+    
+    # Load English localization file.
+    en_data = load_json(en_file)
+    print(f"Loaded {len(en_data)} keys from en_US.json", flush=True)
+    
+    # Get keys that were changed in the latest commit.
+    changed_keys = get_changed_keys(en_file)
+    
+    if not changed_keys:
+        print("No changed keys found - nothing to translate", flush=True)
+        sys.exit(0)
+    
+    print(f"Found {len(changed_keys)} changed keys: {', '.join(sorted(changed_keys))}", flush=True)
+    
+    # Load list of available languages from index.json.
+    if not index_file.exists():
+        print(f"Error: {index_file} not found", flush=True)
+        sys.exit(1)
+    
+    languages = load_json(index_file)
+    print(f"Loaded {len(languages)} languages", flush=True)
+    
+    # Translate changed keys for each language.
+    for lang_info in languages:
+        lang_code = lang_info['code']
+        lang_name = lang_info['name']
+        
+        # Skip English since it's the source language.
+        if lang_code == 'en_US':
+            continue
+        
+        print(f"\n[{lang_code}] {lang_name}", flush=True)
+        
+        target_file = localizations_dir / f"{lang_code}.json"
+        existing_data = load_json(target_file) if target_file.exists() else {}
+        
+        # Filter to only keys that need translation.
+        keys_to_translate = {k: en_data[k] for k in changed_keys if k in en_data}
+        
+        if not keys_to_translate:
+            print("Up to date", flush=True)
+            continue
+        
+        print(f"Translating {len(keys_to_translate)} keys...", flush=True)
+        
+        # Translate in batches to avoid overwhelming the LLM.
+        batch_size = 50
+        translated = {}
+        keys = list(keys_to_translate.keys())
+        
+        for i in range(0, len(keys), batch_size):
+            batch_keys = keys[i:i + batch_size]
+            batch_dict = {k: keys_to_translate[k] for k in batch_keys}
+            
+            batch_num = i // batch_size + 1
+            total_batches = (len(keys) + batch_size - 1) // batch_size
+            print(f"Batch {batch_num}/{total_batches}", flush=True)
+            
+            batch_translated = translate_keys(batch_dict, lang_name)
+            translated.update(batch_translated)
+        
+        # Merge translations with existing data and maintain key order from en_US.json.
+        final_data = {**existing_data, **translated}
+        ordered_data = {k: final_data.get(k, en_data[k]) for k in en_data.keys()}
+        
+        save_json(target_file, ordered_data)
+        print(f"✓ Saved", flush=True)
+    
+    print("\n✓ Done", flush=True)
+
+
+if __name__ == "__main__":
+    main()

.github/ci.py .github/scripts/validate.py.github/ci.py renamed to .github/scripts/validate.py

@@ -0,0 +1,53 @@
+name: Translate
+
+on:
+  push:
+    branches: ["main", "master"]
+    paths:
+      - "localizations/en_US.json"
+
+jobs:
+  translate:
+    if: github.event.pusher.name == 'alexmercerind' && github.event.pusher.email == 'saini123hitesh@gmail.com'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install llm-github-models CLI
+        run: |
+          pip install llm-github-models
+      - name: Configure llm
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          llm keys set github --value "$GITHUB_TOKEN"
+      - name: Generate translations
+        run: |
+          python3 .github/scripts/translate.py
+      - name: Check for changes
+        id: check_changes
+        run: |
+          if git diff --quiet; then
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          fi
+      - name: Create Pull Request
+        if: steps.check_changes.outputs.has_changes == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore: auto-translate localizations"
+          title: "Auto-Translate Localizations"
+          body: "This pull request contains automatically generated translations based on changes to `en_US.json`."
+          branch: auto-translate-${{ github.run_number }}
+          delete-branch: true

.github/workflows/translate.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Validate
 
 on:
   push:
@@ -14,4 +14,4 @@ jobs:
       - name: Validate JSON syntax
         run: for file in `ls | grep '.json' `; do  python -mjson.tool "$file" > /dev/null ; done
       - name: Validate index entries & localization values
-        run: python3 .github/ci.py
+        run: python3 .github/scripts/validate.py