feat: setup translations using LLM

Author: alexmercerind

.github/scripts/translate.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Dict, Any, Set, Optional, List
+
+BATCH_SIZE = 50
+LLM_MODEL = 'github/gpt-4o'
+SOURCE_LANGUAGE = 'en_US'
+
+
+def load_json(file_path: Path) -> Dict[str, Any]:
+    """Load and parse a JSON file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError) as e:
+        print(f"Error loading {file_path}: {e}", flush=True)
+        raise
+
+
+def save_json(file_path: Path, data: Dict[str, Any]) -> None:
+    """Save data to a JSON file with proper formatting."""
+    try:
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+            f.write('\n')
+    except Exception as e:
+        print(f"Error saving {file_path}: {e}", flush=True)
+        raise
+
+
+def get_changed_keys(en_file: Path) -> Set[str]:
+    """Extract changed keys from git diff of the English localization file."""
+    print("Getting git diff...", flush=True)
+    
+    try:
+        result = subprocess.run(
+            ['git', 'diff', 'HEAD~1', 'HEAD', '--', str(en_file)],
+            capture_output=True,
+            text=True,
+            check=False,
+            cwd=en_file.parent.parent
+        )
+        
+        print(f"Git diff return code: {result.returncode}", flush=True)
+        
+        if result.returncode != 0:
+            print(f"Git diff error: {result.stderr}", flush=True)
+            sys.exit(1)
+        
+        if not result.stdout.strip():
+            print("No diff found - file unchanged", flush=True)
+            return set()
+        
+        # Parse diff output to extract changed keys using regex for better accuracy
+        changed_keys = set()
+        # Match lines like: + "key": "value"
+        pattern = re.compile(r'^\+\s*"([^"]+)"\s*:', re.MULTILINE)
+        
+        for match in pattern.finditer(result.stdout):
+            key = match.group(1)
+            changed_keys.add(key)
+        
+        return changed_keys
+        
+    except subprocess.TimeoutExpired:
+        print("Git diff timed out", flush=True)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Exception in get_changed_keys: {e}", flush=True)
+        sys.exit(1)
+
+
+def strip_markdown_code_block(content: str) -> str:
+    """Remove markdown code block formatting from LLM response."""
+    content = content.strip()
+    
+    if content.startswith('```'):
+        # Remove opening ```json or ```
+        lines = content.split('\n')
+        if lines[0].strip() in ('```json', '```'):
+            lines = lines[1:]
+        
+        # Remove closing ```
+        if lines and lines[-1].strip() == '```':
+            lines = lines[:-1]
+        
+        content = '\n'.join(lines).strip()
+    
+    return content
+
+
+def call_llm(prompt: str) -> Optional[str]:
+    """Call the LLM with the given prompt and return the response."""
+    try:
+        process = subprocess.Popen(
+            ['llm', '-m', LLM_MODEL],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True
+        )
+        
+        stdout, stderr = process.communicate(input=prompt, timeout=300)
+        
+        if process.returncode != 0:
+            print(f"LLM error: {stderr}", flush=True)
+            return None
+        
+        return stdout.strip() if stdout.strip() else None
+        
+    except subprocess.TimeoutExpired:
+        print("LLM call timed out", flush=True)
+        process.kill()
+        return None
+    except Exception as e:
+        print(f"Exception calling LLM: {e}", flush=True)
+        return None
+
+
+def build_translation_prompt(
+    keys_dict: Dict[str, str],
+    target_language: str,
+    full_en_data: Dict[str, str],
+    existing_target_data: Dict[str, str]
+) -> str:
+    """Build the translation prompt for the LLM."""
+    return f"""You are a professional translator working on localization for Harmonoid, a music player application. Translate the following JSON object from English to {target_language}.
+
+CONTEXT: These strings are UI text for a music player app. They include terms related to music playback, playlists, albums, artists, audio settings, and media library management.
+
+FULL ENGLISH LOCALIZATION (all strings for reference):
+{json.dumps(full_en_data, ensure_ascii=False, indent=2)}
+
+EXISTING {target_language.upper()} TRANSLATIONS (for consistency reference):
+{json.dumps(existing_target_data, ensure_ascii=False, indent=2)}
+
+IMPORTANT RULES:
+1. Keep all JSON keys EXACTLY the same (do not translate keys)
+2. Only translate the VALUES
+3. Preserve any special formatting like quotes (""), placeholders ("M", "N", "X", "ENTRY", "PLAYLIST", etc.)
+4. Maintain the same meaning, punctuation, capitalization, structure and formatting as the English source
+5. Use appropriate music/audio terminology for the target language
+6. Maintain CONSISTENCY with the existing translations shown above - use the same style, tone, and terminology choices
+7. For technical terms (e.g., "playlist", "equalizer"), check if they were translated or kept in English in existing translations and follow the same pattern
+8. Return ONLY the translated JSON object, no additional text or explanations
+9. Ensure the output is valid JSON
+10. Try to keep similar string length as the original English string (if possible and natural in the target language)
+
+STRINGS TO TRANSLATE:
+{json.dumps(keys_dict, ensure_ascii=False, indent=2)}"""
+
+
+def translate_keys(
+    keys_dict: Dict[str, str],
+    target_language: str,
+    full_en_data: Dict[str, str],
+    existing_target_data: Dict[str, str]
+) -> Dict[str, str]:
+    """Translate a dictionary of keys using LLM."""
+    if not keys_dict:
+        return {}
+    
+    print(f"Calling LLM...", flush=True)
+    
+    prompt = build_translation_prompt(keys_dict, target_language, full_en_data, existing_target_data)
+    response = call_llm(prompt)
+    
+    if not response:
+        print("Empty or failed LLM response, returning original keys", flush=True)
+        return keys_dict
+    
+    print(f"LLM returned successfully", flush=True)
+    
+    # Strip markdown formatting
+    content = strip_markdown_code_block(response)
+    
+    # Parse JSON response
+    try:
+        translated = json.loads(content)
+        
+        # Validate that all keys are present
+        if not isinstance(translated, dict):
+            print("LLM response is not a dictionary", flush=True)
+            return keys_dict
+        
+        missing_keys = set(keys_dict.keys()) - set(translated.keys())
+        if missing_keys:
+            print(f"Warning: Missing keys in translation: {missing_keys}", flush=True)
+            # Fill in missing keys with original values
+            for key in missing_keys:
+                translated[key] = keys_dict[key]
+        
+        return translated
+        
+    except json.JSONDecodeError as e:
+        print(f"JSON decode error: {e}", flush=True)
+        print(f"Content preview: {content[:500]}...", flush=True)
+        return keys_dict
+
+
+def translate_language(
+    lang_code: str,
+    lang_name: str,
+    keys_to_translate: Dict[str, str],
+    en_data: Dict[str, str],
+    existing_data: Dict[str, str],
+    localizations_dir: Path
+) -> bool:
+    """Translate all keys for a specific language."""
+    if not keys_to_translate:
+        print("Up to date", flush=True)
+        return False
+    
+    print(f"Translating {len(keys_to_translate)} keys...", flush=True)
+    
+    # Translate in batches
+    translated = {}
+    keys = list(keys_to_translate.keys())
+    total_batches = (len(keys) + BATCH_SIZE - 1) // BATCH_SIZE
+    
+    for i in range(0, len(keys), BATCH_SIZE):
+        batch_keys = keys[i:i + BATCH_SIZE]
+        batch_dict = {k: keys_to_translate[k] for k in batch_keys}
+        
+        batch_num = i // BATCH_SIZE + 1
+        print(f"Batch {batch_num}/{total_batches} ({len(batch_keys)} keys)", flush=True)
+        
+        batch_translated = translate_keys(batch_dict, lang_name, en_data, existing_data)
+        translated.update(batch_translated)
+    
+    # Merge translations with existing data and maintain key order from en_US.json
+    final_data = {**existing_data, **translated}
+    ordered_data = {k: final_data.get(k, en_data[k]) for k in en_data.keys()}
+    
+    # Save the updated translations
+    target_file = localizations_dir / f"{lang_code}.json"
+    save_json(target_file, ordered_data)
+    print(f"✓ Saved to {target_file.name}", flush=True)
+    
+    return True
+
+
+def main() -> None:
+    """Main entry point for the translation script."""
+    print("Starting translation script...", flush=True)
+    
+    # Setup paths
+    script_dir = Path(__file__).parent
+    project_root = script_dir.parent.parent
+    localizations_dir = project_root / "localizations"
+    index_file = project_root / "index.json"
+    en_file = localizations_dir / f"{SOURCE_LANGUAGE}.json"
+    
+    print(f"Paths:", flush=True)
+    print(f"  project_root: {project_root}", flush=True)
+    print(f"  en_file: {en_file}", flush=True)
+    
+    # Validate English localization file exists
+    if not en_file.exists():
+        print(f"Error: {en_file} not found", flush=True)
+        sys.exit(1)
+    
+    # Load English localization file
+    try:
+        en_data = load_json(en_file)
+        print(f"Loaded {len(en_data)} keys from {SOURCE_LANGUAGE}.json", flush=True)
+    except Exception:
+        sys.exit(1)
+    
+    # Get keys that were changed in the latest commit
+    changed_keys = get_changed_keys(en_file)
+    
+    if not changed_keys:
+        print("No changed keys found - nothing to translate", flush=True)
+        sys.exit(0)
+    
+    print(f"Found {len(changed_keys)} changed keys: {', '.join(sorted(changed_keys))}", flush=True)
+    
+    # Load list of available languages from index.json
+    if not index_file.exists():
+        print(f"Error: {index_file} not found", flush=True)
+        sys.exit(1)
+    
+    try:
+        languages = load_json(index_file)
+        print(f"Loaded {len(languages)} languages", flush=True)
+    except Exception:
+        sys.exit(1)
+    
+    # Translate changed keys for each language
+    translated_count = 0
+    
+    for lang_info in languages:
+        lang_code = lang_info.get('code')
+        lang_name = lang_info.get('name')
+        
+        if not lang_code or not lang_name:
+            print(f"Warning: Invalid language entry: {lang_info}", flush=True)
+            continue
+        
+        # Skip English since it's the source language
+        if lang_code == SOURCE_LANGUAGE:
+            continue
+        
+        print(f"\n[{lang_code}] {lang_name}", flush=True)
+        
+        # Load existing translations for this language
+        target_file = localizations_dir / f"{lang_code}.json"
+        existing_data = load_json(target_file) if target_file.exists() else {}
+        
+        # Filter to only keys that need translation
+        keys_to_translate = {k: en_data[k] for k in changed_keys if k in en_data}
+        
+        # Translate the language
+        if translate_language(lang_code, lang_name, keys_to_translate, en_data, existing_data, localizations_dir):
+            translated_count += 1
+    
+    print(f"\n✓ Done - translated {translated_count} language(s)", flush=True)
+
+
+if __name__ == "__main__":
+    main()

.github/ci.py .github/scripts/validate.py.github/ci.py renamed to .github/scripts/validate.py

@@ -0,0 +1,66 @@
+name: Translate
+
+on:
+  push:
+    branches: ["main", "master"]
+    paths:
+      - "localizations/en_US.json"
+
+jobs:
+  translate:
+    if: github.event.pusher.name == 'alexmercerind' && github.event.pusher.email == 'saini123hitesh@gmail.com'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install llm-github-models CLI
+        run: |
+          pip install llm-github-models
+      - name: Configure llm
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          llm keys set github --value "$GITHUB_TOKEN"
+      - name: Generate translations
+        run: |
+          python3 .github/scripts/translate.py
+      - name: Check for changes
+        id: check_changes
+        run: |
+          if git diff --quiet; then
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          fi
+      - name: Create Pull Request
+        if: steps.check_changes.outputs.has_changes == 'true'
+        id: create_pr
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore: auto-translate localizations"
+          title: "Auto-Translate Localizations"
+          body: "This pull request contains automatically generated translations based on changes to `en_US.json`."
+          branch: auto-translate-${{ github.run_number }}
+          delete-branch: true
+      - name: Assign reviewer to PR
+        if: steps.check_changes.outputs.has_changes == 'true' && steps.create_pr.outputs.pull-request-number
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.pulls.requestReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: ${{ steps.create_pr.outputs.pull-request-number }},
+              reviewers: ['gemini-code-assist']
+            });