Skip to content

Commit 7a61b79

Browse files
carlospolopcarlospolop
committed
improve workflows
1 parent 0af9072 commit 7a61b79

18 files changed

+241
-1558
lines changed

‎.github/workflows/build_master.yml‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,19 +35,60 @@ jobs:
3535
- name: Build mdBook
3636
run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
3737

38-
- name: Update searchindex.js in repo
38+
- name: Update searchindex in repo (purge history, keep current on HEAD)
39+
shell: bash
3940
run: |
41+
set -euo pipefail
42+
4043
ls -la
4144
ls -la book
42-
(git config --global --add safe.directory /__w/hacktricks-cloud/hacktricks-cloud
43-
git pull
45+
46+
git config --global --add safe.directory /__w/hacktricks/hacktricks-cloud
4447
git config --global user.email "[email protected]"
4548
git config --global user.name "Build master"
4649
git config pull.rebase false
47-
cp book/searchindex.js searchindex.js
48-
git add searchindex.js
49-
git commit -m "Update searchindex"
50-
git push) || echo "No changes to searchindex.js"
50+
51+
# Ensure we're on the target branch and up to date
52+
git pull --ff-only
53+
54+
# Choose the file to keep at HEAD:
55+
# 1) Prefer freshly built version from book/
56+
# 2) Fallback to the file currently at HEAD (if it exists)
57+
HAS_FILE=0
58+
if [ -f "book/searchindex.js" ]; then
59+
cp "book/searchindex.js" /tmp/sidx.js
60+
HAS_FILE=1
61+
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
62+
git show "HEAD:searchindex.js" > /tmp/sidx.js
63+
HAS_FILE=1
64+
fi
65+
66+
# Skip if there's nothing to purge AND nothing to keep
67+
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then
68+
# Fail early if working tree is dirty (avoid confusing rewrites)
69+
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
70+
71+
# Install git-filter-repo and ensure it's on PATH
72+
python -m pip install --quiet --user git-filter-repo
73+
export PATH="$HOME/.local/bin:$PATH"
74+
75+
# Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js
76+
git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)"
77+
78+
# Re-add the current version on top of rewritten history (keep it in HEAD)
79+
if [ "$HAS_FILE" = "1" ]; then
80+
mv /tmp/sidx.js "searchindex.js"
81+
git add "searchindex.js"
82+
git commit -m "Update searchindex (purged history; keep current)"
83+
else
84+
echo "No current searchindex.js to re-add after purge."
85+
fi
86+
87+
# Safer force push (only updates if remote hasn't advanced)
88+
git push --force-with-lease
89+
else
90+
echo "Nothing to purge; skipping."
91+
fi
5192
5293
# Login in AWs
5394
- name: Configure AWS credentials using OIDC

‎.github/workflows/translate_af.yml‎

Lines changed: 0 additions & 97 deletions
This file was deleted.

‎.github/workflows/translate_all.yml‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
name: Translator All
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
paths-ignore:
8+
- 'scripts/**'
9+
- '.gitignore'
10+
- '.github/**'
11+
- Dockerfile
12+
workflow_dispatch:
13+
14+
permissions:
15+
packages: write
16+
id-token: write
17+
contents: write
18+
19+
jobs:
20+
translate:
21+
name: Translate → ${{ matrix.name }} (${{ matrix.branch }})
22+
runs-on: ubuntu-latest
23+
24+
# Run N languages in parallel (tune max-parallel if needed)
25+
strategy:
26+
fail-fast: false
27+
# max-parallel: 3 #Nothing to run all in parallel
28+
matrix:
29+
include:
30+
- { name: "Afrikaans", language: "Afrikaans", branch: "af" }
31+
- { name: "German", language: "German", branch: "de" }
32+
- { name: "Greek", language: "Greek", branch: "el" }
33+
- { name: "Spanish", language: "Spanish", branch: "es" }
34+
- { name: "French", language: "French", branch: "fr" }
35+
- { name: "Hindi", language: "Hindi", branch: "hi" }
36+
- { name: "Italian", language: "Italian", branch: "it" }
37+
- { name: "Japanese", language: "Japanese", branch: "ja" }
38+
- { name: "Korean", language: "Korean", branch: "ko" }
39+
- { name: "Polish", language: "Polish", branch: "pl" }
40+
- { name: "Portuguese", language: "Portuguese", branch: "pt" }
41+
- { name: "Serbian", language: "Serbian", branch: "sr" }
42+
- { name: "Swahili", language: "Swahili", branch: "sw" }
43+
- { name: "Turkish", language: "Turkish", branch: "tr" }
44+
- { name: "Ukrainian", language: "Ukrainian", branch: "uk" }
45+
- { name: "Chinese", language: "Chinese", branch: "zh" }
46+
47+
# Ensure only one job per branch runs at a time (even across workflow runs)
48+
concurrency:
49+
group: translate-${{ matrix.branch }}
50+
cancel-in-progress: false
51+
52+
container:
53+
image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
54+
55+
env:
56+
LANGUAGE: ${{ matrix.language }}
57+
BRANCH: ${{ matrix.branch }}
58+
59+
steps:
60+
- name: Checkout code
61+
uses: actions/checkout@v4
62+
with:
63+
fetch-depth: 0
64+
65+
- name: Update and download scripts
66+
run: |
67+
sudo apt-get update
68+
sudo apt-get install wget -y
69+
mkdir scripts
70+
cd scripts
71+
wget -O get_and_save_refs.py https://raw.githubusercontent.com/carlospolop/hacktricks-cloud/master/scripts/get_and_save_refs.py
72+
wget -O compare_and_fix_refs.py https://raw.githubusercontent.com/carlospolop/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
73+
wget -O translator.py https://raw.githubusercontent.com/carlospolop/hacktricks-cloud/master/scripts/translator.py
74+
cd ..
75+
76+
- name: Run get_and_save_refs.py
77+
run: |
78+
python scripts/get_and_save_refs.py
79+
80+
- name: Download language branch & update refs
81+
run: |
82+
git config --global --add safe.directory /__w/hacktricks/hacktricks-cloud
83+
git config --global user.name 'Translator'
84+
git config --global user.email '[email protected]'
85+
git config pull.rebase false
86+
git checkout $BRANCH
87+
git pull
88+
python scripts/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt
89+
git add .
90+
git commit -m "Fix unmatched refs" || echo "No changes to commit"
91+
git push || echo "No changes to push"
92+
93+
- name: Run translation script on changed files
94+
run: |
95+
git checkout master
96+
export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
97+
git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do
98+
if echo "$file" | grep -qE '\.md$'; then
99+
echo -n ",$file" >> /tmp/file_paths.txt
100+
fi
101+
done
102+
103+
echo "Files to translate:"
104+
cat /tmp/file_paths.txt
105+
echo ""
106+
echo ""
107+
touch /tmp/file_paths.txt
108+
109+
if [ -s /tmp/file_paths.txt ]; then
110+
python scripts/translator.py \
111+
--language "$LANGUAGE" \
112+
--branch "$BRANCH" \
113+
--api-key "$OPENAI_API_KEY" \
114+
-f "$(cat /tmp/file_paths.txt)" \
115+
-t 3
116+
else
117+
echo "No markdown files changed, skipping translation."
118+
fi
119+
120+
- name: Build mdBook
121+
run: |
122+
git checkout "$BRANCH"
123+
git pull
124+
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
125+
126+
- name: Update searchindex.js in repo (purge history, keep current on HEAD)
127+
shell: bash
128+
run: |
129+
set -euo pipefail
130+
131+
# Be explicit about workspace trust (avoids "dubious ownership")
132+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
133+
134+
git checkout "$BRANCH"
135+
git fetch origin "$BRANCH" --quiet
136+
git pull --ff-only
137+
138+
# Choose the file to keep at HEAD:
139+
# 1) Prefer freshly built version from book/
140+
# 2) Fallback to the file currently at HEAD (if it exists)
141+
HAS_FILE=0
142+
if [ -f "book/searchindex.js" ]; then
143+
cp "book/searchindex.js" /tmp/sidx.js
144+
HAS_FILE=1
145+
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
146+
git show "HEAD:searchindex.js" > /tmp/sidx.js
147+
HAS_FILE=1
148+
fi
149+
150+
# Skip if there's nothing to purge AND nothing to keep
151+
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then
152+
# **Fail early if working tree is dirty** (prevents confusing filter results)
153+
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
154+
155+
# Make sure git-filter-repo is callable via `git filter-repo`
156+
python -m pip install --quiet --user git-filter-repo
157+
export PATH="$HOME/.local/bin:$PATH"
158+
159+
# Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
160+
git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
161+
162+
# Re-add the current version on top of rewritten history (keep it in HEAD)
163+
if [ "$HAS_FILE" = "1" ]; then
164+
mv /tmp/sidx.js "searchindex.js"
165+
git add "searchindex.js"
166+
git commit -m "Update searchindex (purged history; keep current)"
167+
else
168+
echo "No current searchindex.js to re-add after purge."
169+
fi
170+
171+
# **Safer force push** (prevents clobbering unexpected remote updates)
172+
git push --force-with-lease origin "$BRANCH"
173+
else
174+
echo "Nothing to purge; skipping."
175+
fi
176+
177+
# Login in AWs
178+
- name: Configure AWS credentials using OIDC
179+
uses: aws-actions/configure-aws-credentials@v3
180+
with:
181+
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
182+
aws-region: us-east-1
183+
184+
# Sync the build to S3
185+
- name: Sync to S3
186+
run: |
187+
echo "Current branch:"
188+
git rev-parse --abbrev-ref HEAD
189+
echo "Syncing $BRANCH to S3"
190+
aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete
191+
echo "Sync completed"
192+
echo "Cat 3 files from the book"
193+
find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat

0 commit comments

Comments
 (0)