This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Translator All | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - 'scripts/**' | |
| - '.gitignore' | |
| - '.github/**' | |
| - Dockerfile | |
| workflow_dispatch: | |
| permissions: | |
| packages: write | |
| id-token: write | |
| contents: write | |
| jobs: | |
| translate: | |
| name: Translate → ${{ matrix.name }} (${{ matrix.branch }}) | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| # Run N languages in parallel (tune max-parallel if needed) | |
| strategy: | |
| fail-fast: false | |
| # max-parallel: 3 #Nothing to run all in parallel | |
| matrix: | |
| include: | |
| - { name: "Afrikaans", language: "Afrikaans", branch: "af" } | |
| - { name: "German", language: "German", branch: "de" } | |
| - { name: "Greek", language: "Greek", branch: "el" } | |
| - { name: "Spanish", language: "Spanish", branch: "es" } | |
| - { name: "French", language: "French", branch: "fr" } | |
| - { name: "Hindi", language: "Hindi", branch: "hi" } | |
| - { name: "Italian", language: "Italian", branch: "it" } | |
| - { name: "Japanese", language: "Japanese", branch: "ja" } | |
| - { name: "Korean", language: "Korean", branch: "ko" } | |
| - { name: "Polish", language: "Polish", branch: "pl" } | |
| - { name: "Portuguese", language: "Portuguese", branch: "pt" } | |
| - { name: "Serbian", language: "Serbian", branch: "sr" } | |
| - { name: "Swahili", language: "Swahili", branch: "sw" } | |
| - { name: "Turkish", language: "Turkish", branch: "tr" } | |
| - { name: "Ukrainian", language: "Ukrainian", branch: "uk" } | |
| - { name: "Chinese", language: "Chinese", branch: "zh" } | |
| # Ensure only one job per branch runs at a time (even across workflow runs) | |
| concurrency: | |
| group: translate-${{ matrix.branch }} | |
| cancel-in-progress: false | |
| container: | |
| image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest | |
| env: | |
| LANGUAGE: ${{ matrix.language }} | |
| BRANCH: ${{ matrix.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Update and download scripts | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install wget -y | |
| wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py | |
| wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py | |
| wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py | |
| - name: Run get_and_save_refs.py | |
| run: | | |
| python /tmp/get_and_save_refs.py | |
| - name: Download language branch & update refs | |
| run: | | |
| pwd | |
| ls -la | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| git config --global user.name 'Translator' | |
| git config --global user.email '[email protected]' | |
| git config pull.rebase false | |
| git checkout $BRANCH | |
| git pull | |
| python /tmp/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt | |
| git add . | |
| git commit -m "Fix unmatched refs" || echo "No changes to commit" | |
| git push || echo "No changes to push" | |
| - name: Run translation script on changed files | |
| run: | | |
| git checkout master | |
| cp src/SUMMARY.md /tmp/master-summary.md | |
| export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} | |
| git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do | |
| if echo "$file" | grep -qE '\.md$'; then | |
| echo -n ",$file" >> /tmp/file_paths.txt | |
| fi | |
| done | |
| echo "Files to translate:" | |
| cat /tmp/file_paths.txt | |
| echo "" | |
| echo "" | |
| touch /tmp/file_paths.txt | |
| if [ -s /tmp/file_paths.txt ]; then | |
| python /tmp/translator.py \ | |
| --language "$LANGUAGE" \ | |
| --branch "$BRANCH" \ | |
| --api-key "$OPENAI_API_KEY" \ | |
| -f "$(cat /tmp/file_paths.txt)" \ | |
| -t 3 | |
| else | |
| echo "No markdown files changed, skipping translation." | |
| fi | |
| - name: Sync SUMMARY.md from master | |
| run: | | |
| git checkout "$BRANCH" | |
| git pull | |
| if [ -f /tmp/master-summary.md ]; then | |
| cp /tmp/master-summary.md src/SUMMARY.md | |
| git add src/SUMMARY.md | |
| git commit -m "Sync SUMMARY.md with master" || echo "SUMMARY already up to date" | |
| git push || echo "No SUMMARY updates to push" | |
| else | |
| echo "master summary not exported; failing" | |
| exit 1 | |
| fi | |
| - name: Build mdBook | |
| run: | | |
| git checkout "$BRANCH" | |
| git pull | |
| MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) | |
| - name: Update searchindex.js in repo (purge history, keep current on HEAD) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Be explicit about workspace trust (avoids "dubious ownership") | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| git checkout "$BRANCH" | |
| git fetch origin "$BRANCH" --quiet | |
| git pull --ff-only | |
| # Choose the file to keep at HEAD: | |
| # 1) Prefer freshly built version from book/ | |
| # 2) Fallback to the file currently at HEAD (if it exists) | |
| HAS_FILE=0 | |
| if [ -f "book/searchindex.js" ]; then | |
| cp "book/searchindex.js" /tmp/sidx.js | |
| HAS_FILE=1 | |
| elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then | |
| git show "HEAD:searchindex.js" > /tmp/sidx.js | |
| HAS_FILE=1 | |
| fi | |
| # Skip if there's nothing to purge AND nothing to keep | |
| if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then | |
| # **Fail early if working tree is dirty** (prevents confusing filter results) | |
| git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; } | |
| # Make sure git-filter-repo is callable via `git filter-repo` | |
| python -m pip install --quiet --user git-filter-repo | |
| export PATH="$HOME/.local/bin:$PATH" | |
| # Rewrite ONLY this branch, dropping all historical blobs of searchindex.js | |
| git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH" | |
| # Re-add the current version on top of rewritten history (keep it in HEAD) | |
| if [ "$HAS_FILE" = "1" ]; then | |
| mv /tmp/sidx.js "searchindex.js" | |
| git add "searchindex.js" | |
| git commit -m "Update searchindex (purged history; keep current)" | |
| else | |
| echo "No current searchindex.js to re-add after purge." | |
| fi | |
| # **Safer force push** (prevents clobbering unexpected remote updates) | |
| git push --force-with-lease origin "$BRANCH" | |
| else | |
| echo "Nothing to purge; skipping." | |
| fi | |
| # Login in AWs | |
| - name: Configure AWS credentials using OIDC | |
| uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| # Sync the build to S3 | |
| - name: Sync to S3 | |
| run: | | |
| echo "Current branch:" | |
| git rev-parse --abbrev-ref HEAD | |
| echo "Syncing $BRANCH to S3" | |
| aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete | |
| echo "Sync completed" | |
| echo "Cat 3 files from the book" | |
| find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat |