Skip to content

Commit 338435e

Browse files
paoloredispaoloredis
authored
Add github workflow for synching redisvl documentation (#1318)
1 parent 8f12642 commit 338435e

File tree

1 file changed

+286
-0
lines changed

1 file changed

+286
-0
lines changed

‎.github/workflows/redisvl_docs_sync.yaml‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
name: redisvl_docs_sync
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * *' # run every day at midnight UTC time
6+
workflow_dispatch: # or run on manual trigger
7+
8+
jobs:
9+
redisvl_docs_sync:
10+
runs-on: ubuntu-latest
11+
permissions:
12+
contents: write
13+
pull-requests: write
14+
actions: write
15+
steps:
16+
- name: 'Checkout'
17+
uses: 'actions/checkout@v3'
18+
19+
- name: 'Install deps'
20+
run: |
21+
pip3 install -U sphinx
22+
pip3 install sphinx_design sphinx_copybutton nbsphinx sphinx_favicon myst_nb sphinx_markdown_builder==0.6.8 jupyter
23+
24+
- name: 'Fetch redisvl repo'
25+
run: |
26+
git clone https://github.com/redis/redis-vl-python
27+
28+
- name: 'Run sphinx-build'
29+
id: 'sphinx'
30+
env:
31+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
32+
run: |
33+
pushd redis-vl-python
34+
35+
# Get latest release
36+
latest_release=$(gh release -R redis/redis-vl-python list --json name,isLatest --jq '.[] | select(.isLatest)|.name')
37+
git checkout "tags/${latest_release}"
38+
pip3 install -e .
39+
40+
popd
41+
sphinx-build -M markdown ./redis-vl-python/docs ./build
42+
43+
echo "release=${latest_release}" >> "$GITHUB_OUTPUT"
44+
45+
- name: 'Format markdown files for hugo compatibility'
46+
run: |
47+
#!/bin/bash
48+
mkdir -p redis_vl_hugo/user_guide/
49+
mkdir redis_vl_hugo/overview/
50+
mkdir redis_vl_hugo/api/
51+
52+
function format() {
53+
src=$1
54+
55+
# Extract title from source file
56+
title=$(head -n 1 ${src} | sed 's/# //')
57+
linkTitle="${title}"
58+
59+
# All words except first should be lowercased in linkTitle, except for RedisVL, API, CLI, LLMs, JSON
60+
if [[ "$linkTitle" =~ \ ]]; then
61+
linkTitle=$(awk '{ $1=$1; for (i=2; i<=NF; i++) if ($i != "RedisVL" && $i != "API" && $i != "CLI" && $i != "LLMs" && $i != "JSON") $i=tolower($i); print }' <<< "${linkTitle}")
62+
fi
63+
64+
# Inject frontmatter in destination file
65+
cat >_tmp <<EOL
66+
---
67+
linkTitle: ${linkTitle}
68+
title: ${title}
69+
type: integration
70+
EOL
71+
72+
# Inject weight property for index pages to preserve order
73+
case "${title}" in
74+
"Overview")
75+
echo "weight: 3" >> _tmp ;;
76+
"User Guides")
77+
echo "weight: 4" >> _tmp ;;
78+
"RedisVL API")
79+
echo "weight: 5" >> _tmp ;;
80+
esac
81+
82+
# For user guides, strip the leading numbers from filename and use them for weight
83+
f=$(basename "${src}")
84+
if [[ "$f" =~ ^([0-9][0-9])_(.+) ]]; then
85+
echo "weight: ${BASH_REMATCH[1]}" >> _tmp
86+
fi
87+
88+
# For _index.md files, add hideListLinks property
89+
if [[ ${src} =~ _index.md$ ]]; then
90+
echo "hideListLinks: true" >> _tmp
91+
fi
92+
93+
# Close frontmatter
94+
echo "---" >> _tmp
95+
echo "" >> _tmp
96+
97+
# Write all of source file, except first line, in destination file
98+
tail -n+2 ${src} >> _tmp
99+
100+
# _index.md files need title removed
101+
if [[ ${src} =~ _index.md$ ]]; then
102+
sed -i "s/# ${title}//" _tmp
103+
fi
104+
105+
mv _tmp ${src}
106+
}
107+
108+
# Convert jupyter notebooks to markdown
109+
jupyter nbconvert --to markdown build/jupyter_execute/user_guide/*.ipynb --output-dir redis_vl_hugo/user_guide/ 2>/dev/null
110+
jupyter nbconvert --to markdown build/jupyter_execute/overview/cli.ipynb --output-dir redis_vl_hugo/overview/ 2>/dev/null
111+
112+
# Prepare markdown files
113+
rsync -a ./build/markdown/api/ ./redis_vl_hugo/api/ --exclude=index.md
114+
cp ./build/markdown/overview/installation.md ./redis_vl_hugo/overview/installation.md
115+
116+
# Format markdown files
117+
markdown_pages=(./redis_vl_hugo/**/*.md)
118+
119+
for markdown_page in "${markdown_pages[@]}"; do
120+
format "${markdown_page}"
121+
122+
# Remove blockquotes and ansi stuff
123+
sed -E -i 's/^> *//g; s/\x1b\[[0-9;]*m//g' "${markdown_page}"
124+
125+
# Replace https://docs.redisvl.com links
126+
sed -E -i 's#https://docs.redisvl.com/en/latest/.+/([^_]+).+\.html(\#[^)]+)#{{< relref "\1\2" >}}#g; s#https://docs.redisvl.com/en/latest/(.+)\.html#https://redis.io/docs/latest/integrate/redisvl/\1#g' "${markdown_page}"
127+
done
128+
129+
# Fix links in api pages
130+
api_markdown_pages=(./redis_vl_hugo/api/*.md)
131+
132+
for api_markdown_page in "${api_markdown_pages[@]}"; do
133+
gawk '
134+
# handle relrefs to other pages
135+
/\([A-Za-z]+\.md#redisvl\.[a-zA-Z0-9_.]+\)/ {
136+
while (match($0, /\([A-Za-z]+\.md#redisvl\.[a-zA-Z0-9_.]+\)/)) {
137+
138+
m = substr($0, RSTART+1, RLENGTH-2)
139+
split(m, parts, /[.#]/)
140+
ref = "({{< relref \"" parts[1] "/#" tolower(parts[length(parts)]) "\" >}})"
141+
$0 = substr($0, 1, RSTART-1) ref substr($0, RSTART+RLENGTH)
142+
143+
}
144+
}
145+
# handle relref same page
146+
/\(#redisvl\.[^)]+\)/ {
147+
while (match($0, /\[[^][]+\]\(#redisvl\.[^)]+\)/)) {
148+
149+
m = substr($0, RSTART+1, RLENGTH-2)
150+
split(m, parts, /[.#]/)
151+
ref = "[" parts[length(parts)] "]" "(" "#" tolower(parts[length(parts)]) ")"
152+
$0 = substr($0, 1, RSTART-1) ref substr($0, RSTART+RLENGTH)
153+
}
154+
}
155+
# format class/function signatures
156+
/^####?/ {
157+
# remove * from class, async, etc.
158+
if ($2 ~ "^*") {
159+
gsub(/\*/, "", $2)
160+
}
161+
162+
# insert opening backtick
163+
gsub(/^/,"`", $2)
164+
165+
if ($0 ~ /#### `.+: .*\[.+\]\(#.+\)/) {
166+
# fix relrefs
167+
gsub(/^/, "`", $4)
168+
gsub(/`$/, "", $4)
169+
gsub(/\*$/,"");
170+
} else {
171+
# insert closing backtick
172+
$0 = $0 "`";
173+
}
174+
# unescape double asterisks, e.g. **kwargs
175+
gsub(/\\\*\\\*/,"**");
176+
177+
# unespace underscores
178+
gsub(/\\_/,"_");
179+
180+
# remove asterisks
181+
gsub(/ ?\*:/,":");
182+
gsub(/\*`$/,"`");
183+
184+
# handle ClassVar[ConfigDict]
185+
gsub(/\* \*= \{/," = \{")
186+
187+
# handle *= in router.md
188+
gsub(/ \*\= /, " = ");
189+
190+
# format rel links
191+
if ($0 ~ /\[.+\]\(.+relref.+\)/) {
192+
gsub(/\[/,"`[`")
193+
gsub(/\]/,"`]")
194+
gsub(/\)/,")` ")
195+
}
196+
# unescape single asterisks, e.g. *args
197+
gsub(/\\\*/,"*");
198+
}
199+
{
200+
# fix parameters
201+
gsub(/\*\[\*\*/,"\*\[\* \*");
202+
gsub(/\*\(\*\*/,"\*\(\* \*");
203+
print $0
204+
}
205+
' "${api_markdown_page}" > _tmp && mv _tmp "${api_markdown_page}"
206+
done
207+
208+
# Format _index.md pages
209+
cp ./build/markdown/api/index.md ./redis_vl_hugo/api/_index.md
210+
cp ./build/markdown/user_guide/index.md ./redis_vl_hugo/user_guide/_index.md
211+
cp ./build/markdown/overview/index.md ./redis_vl_hugo/overview/_index.md
212+
213+
index_markdown_pages=(
214+
./redis_vl_hugo/api/_index.md
215+
./redis_vl_hugo/user_guide/_index.md
216+
./redis_vl_hugo/overview/_index.md
217+
)
218+
219+
for index_markdown_page in "${index_markdown_pages[@]}"; do
220+
format "${index_markdown_page}"
221+
222+
# Fix relrefs by removing .md extension and references to numbered pages
223+
sed -E -i 's/\.md/\//g; s/\([0-9]+_/\(/g' "${index_markdown_page}"
224+
done
225+
226+
# Rename user guides to strip leading numbers from filename
227+
user_guides=(./redis_vl_hugo/user_guide/*.md)
228+
for user_guide in "${user_guides[@]}"; do
229+
if [[ ! "$user_guide" =~ _index.md$ ]]; then
230+
renamed_user_guide=$(sed 's|[0-9]\+_||' <<<${user_guide})
231+
mv ${user_guide} ${renamed_user_guide}
232+
fi
233+
done
234+
235+
cp -r redis_vl_hugo/* content/integrate/redisvl/
236+
237+
- name: 'Create pull request if necessary'
238+
env:
239+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
240+
run: |
241+
release="${{ steps.sphinx.outputs.release }}"
242+
branch="redisvl_docs_sync_${release}"
243+
redisvl_change=false
244+
245+
# check if remote branch already exists
246+
git fetch --all
247+
set +e
248+
git ls-remote --exit-code --heads origin "refs/heads/${branch}"
249+
if [ "$?" -eq 0 ]; then
250+
set -e
251+
# if it does, create local branch off existing remote branch
252+
git checkout -b "${branch}" "origin/${branch}"
253+
git branch --set-upstream-to="origin/${branch}" "${branch}"
254+
git pull
255+
else
256+
set -e
257+
# otherwise, create local branch from main
258+
git checkout -b "${branch}"
259+
fi
260+
261+
redisvl_is_different=$(git diff content/integrate/redisvl/)
262+
263+
if [[ ! -z $redisvl_is_different ]]; then
264+
redisvl_change=true
265+
266+
git add "content/integrate/redisvl/"
267+
git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com"
268+
git config user.name "redisdocsapp[bot]"
269+
git commit -m "Update for redisvl ${release}"
270+
fi
271+
272+
if [ "$redisvl_change" = true ] ; then
273+
git push origin "${branch}"
274+
275+
# If a pr is not already open, create one
276+
set +e
277+
gh search prs -R redis/docs --state open --match title "update for redisvl ${release}" | grep -q "update for redisvl ${release}"
278+
if [ "$?" -eq 1 ]; then
279+
set -e
280+
gh pr create \
281+
--body "update for redisvl ${release}" \
282+
--title "update for redisvl ${release}" \
283+
--head "$branch" \
284+
--base "main"
285+
fi
286+
fi

0 commit comments

Comments
 (0)