Skip to content

Commit 5040828

Browse files
jaimergpbeckermr
jaimergp
and
beckermr
authored
Render Sphinx /docs in docusaurus (#2077)
* Render Sphinx to markdown * Adjust CFEP index format * Adjust some lists and headers * Fix broken link? (works locally) * Only throw errors on GHA CI (warn on local and Netlify) * push a couple broken links to demo link checker * add an additional link checker GHA step * Use lychee * configure lychee * less verbosity * remap instead * try with this one * where's the report? * remove verbosity * exclude comma separated URLs * a different broken anchor * raise if document not in sidebar * do not show editUrl on docs/ for now (broken) * remove intentionally broken links; this should pass * upgrade prism highlighting * fix indentation in some lists * Publish _static assets * pathname:// for linked files * Use :ref: and :doc: for internal links instead of https://conda-forge.org/ ones * Adjust FAQ output * Formatting adjustments * Fix Maintainer FAQ too * add basic local search for now * add goatcounter * Use Algolia search * exclude algolia from link checker * more explicit * Upgrade to docusaurus 3.1.1 and enable onBrokenAnchors * Fix broken generation of anchor * Remove todo list from misc * vendor goatcounter ourselves * only deploy stats on GHA site * BUG attempt to fix rendering of core and emeritus * BUG maybe one more level? * Fix Jinja rendering in governance and cfep-index * get cfep list from repo archive, not api --------- Co-authored-by: Matthew R. Becker <[email protected]>
1 parent d6d4d60 commit 5040828

37 files changed

+1378
-857
lines changed

‎.ci_scripts/environment.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,5 @@ dependencies:
1818
- python-rapidjson
1919
- termcolor
2020
- nodejs 20.*
21+
- pip:
22+
- https://github.com/jaimergp/sphinx-markdown-builder/archive/admonitions.tar.gz

‎.ci_scripts/generate_cfep_index.py

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
import re
22
import requests
3+
import sys
4+
import tarfile
35
from dataclasses import dataclass
46
from pathlib import Path
7+
from tempfile import TemporaryDirectory
58

9+
REPO_URL = "https://github.com/conda-forge/cfep"
10+
REPO_ARCHIVE = "https://github.com/conda-forge/cfep/archive/main.tar.gz"
611
REPO_CONTENTS = "https://api.github.com/repos/conda-forge/cfep/contents/"
712
TITLE_PATTERN = "<td>\s*Title\s*</td><td>\s*(.*)\s*</td>"
813
STATUS_PATTERN = "<td>\s*Status\s*</td><td>\s*(.*)\s*</td>"
@@ -31,7 +36,7 @@ def md_link(self) -> str:
3136
)
3237

3338

34-
def get_cfeps():
39+
def get_cfeps_from_gh_api():
3540
"""Generator that returns all CFEPs from GitHub repo"""
3641
response = requests.get(
3742
REPO_CONTENTS, headers={"Accept": "application/vnd.github.v3+json"}
@@ -56,10 +61,48 @@ def get_cfeps():
5661
yield Cfep(content["name"], title, status, content["html_url"])
5762

5863

64+
def get_cfeps():
65+
"""Return a generator of CFEPs, by traversing the contents of the repo archive"""
66+
r = requests.get(REPO_ARCHIVE, stream=True)
67+
r.raise_for_status()
68+
with TemporaryDirectory() as tmp:
69+
# Write the tarball to a temporary directory
70+
tarball = Path(tmp) / "cfep.tar.gz"
71+
with tarball.open("wb") as f:
72+
for chunk in r.iter_content(chunk_size=8192):
73+
f.write(chunk)
74+
# Extract the tarball
75+
extracted_dir = Path(tmp) / "cfep"
76+
extracted_dir.mkdir()
77+
with tarfile.open(tarball) as tar:
78+
tar.extractall(extracted_dir)
79+
# Traverse the extracted directory and return all CFEPs
80+
for cfep in sorted(extracted_dir.rglob("cfep-*.md")):
81+
name = cfep.name
82+
url = f"{REPO_URL}/blob/main/{name}"
83+
if name == "cfep-00.md":
84+
# Hardcode title and status for CFEP-00
85+
yield Cfep(name, "CFEP Template", "Proposed", url)
86+
continue
87+
cfep_text = cfep.read_text()
88+
m = re.search(TITLE_PATTERN, cfep_text)
89+
title = m.group(1).strip() if m else ""
90+
m = re.search(STATUS_PATTERN, cfep_text)
91+
status = m.group(1).strip() if m else ""
92+
yield Cfep(name, title, status, url)
93+
94+
5995
def write_cfep_index():
60-
with CFEP_INDEX_RST.open("a") as f:
61-
for cfep in get_cfeps():
62-
f.write(f"* {cfep.rst_link()}\n")
96+
contents = CFEP_INDEX_RST.read_text()
97+
if ".. REPLACE-THIS-LINE-WITH-THE-INDEX-OF-CFEPs" not in contents:
98+
print("!!! Skipping writing CFEP index. Already rendered?", file=sys.stderr)
99+
return
100+
rst_links = [f"- {cfep.rst_link()}" for cfep in get_cfeps()]
101+
contents = contents.replace(
102+
".. REPLACE-THIS-LINE-WITH-THE-INDEX-OF-CFEPs",
103+
"\n".join(rst_links)
104+
)
105+
CFEP_INDEX_RST.write_text(contents)
63106

64107

65108
if __name__ == "__main__":

‎.ci_scripts/sphinx_markdown_to_docusaurus.py

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
"""
2+
Given the Markdown output of a sphinx site, place it in the correct place for Docusaurus.
3+
"""
4+
import re
5+
import sys
6+
from pathlib import Path
7+
8+
# The ordering of the sidebar is defined here. This is a list of paths to the markdown files
9+
# as they would appear after being moved into /docs. In other words, how Docusaurus would see them.
10+
SIDEBAR_ORDERING = [
11+
"/docs/index.md",
12+
"/docs/user/index.md",
13+
"/docs/user/introduction.md",
14+
"/docs/user/tipsandtricks.md",
15+
"/docs/user/ci-skeleton.md",
16+
"/docs/user/faq.md",
17+
"/docs/user/contributing.md",
18+
"/docs/user/how_to_get_help.md",
19+
"/docs/user/talks.md",
20+
"/docs/maintainer/index.md",
21+
"/docs/maintainer/infrastructure.md",
22+
"/docs/maintainer/adding_pkgs.md",
23+
"/docs/maintainer/updating_pkgs.md",
24+
"/docs/maintainer/pinning_deps.md",
25+
"/docs/maintainer/conda_forge_yml.md",
26+
"/docs/maintainer/knowledge_base.md",
27+
"/docs/maintainer/maintainer_faq.md",
28+
"/docs/orga/index.md",
29+
"/docs/orga/guidelines.md",
30+
"/docs/orga/governance.md",
31+
"/docs/orga/subteams.md",
32+
"/docs/orga/joining-the-team.md",
33+
"/docs/orga/funding",
34+
"/docs/orga/minutes/",
35+
"/docs/orga/cfep-index.md",
36+
"/docs/orga/getting-in-touch.md",
37+
"/docs/orga/funding/index.md",
38+
"/docs/misc/index.md",
39+
"/docs/contracting/index.md",
40+
]
41+
# Note we also ignore the /minutes/.* files later in the code
42+
SIDEBAR_ORDERING_IGNORED = {
43+
"/docs/orga/funding/gsod-2023.md",
44+
"/docs/orga/funding/gsoc-2023.md",
45+
"/docs/orga/funding/sdg-2023-1.md",
46+
}
47+
48+
49+
def get_mds(basedir):
50+
for path in Path(basedir).glob("**/*.md"):
51+
yield path
52+
53+
54+
def sphinx_md_to_docusaurus_md(basedir, mdpath, targetdir, ordering=None):
55+
relmd = mdpath.relative_to(basedir)
56+
target_path = Path(targetdir, relmd)
57+
target_path.parent.mkdir(parents=True, exist_ok=True)
58+
text = mdpath.read_text()
59+
text = text.replace("00_intro.md", "index.md")
60+
text = text.replace("(/_static/", "(pathname:///_static/")
61+
text = re.sub(r"\]\((/\S+\.\S+)\)", r"](pathname://\1)", text)
62+
title = next(re.finditer(r"^# (.+)$", text, re.MULTILINE), None)
63+
if not text.lstrip().startswith("---"):
64+
frontmatter = []
65+
if title:
66+
frontmatter.append(f"title: '{title.group(1)}'")
67+
if ordering is not None:
68+
frontmatter.append(f"sidebar_position: {ordering}")
69+
if frontmatter:
70+
text = "---\n" + "\n".join(frontmatter) + "\n---\n\n" + text
71+
if mdpath.name == "00_intro.md":
72+
target_path = target_path.parent / "index.md"
73+
target_path.write_text(text)
74+
75+
76+
def main(build_dir, targetdir):
77+
md_dir = Path(build_dir)
78+
not_in_sidebar = []
79+
for path in md_dir.glob("**/*.md"):
80+
print("Processing MD", path)
81+
try:
82+
relmd = path.relative_to(md_dir).as_posix()
83+
relmd_key = f"/docs/{relmd}".replace("00_intro.md", "index.md")
84+
ordering = SIDEBAR_ORDERING.index(relmd_key)
85+
except ValueError:
86+
if "/minutes/" in relmd_key or relmd_key in SIDEBAR_ORDERING_IGNORED:
87+
ordering = 1000
88+
else:
89+
not_in_sidebar.append(path)
90+
continue
91+
sphinx_md_to_docusaurus_md(md_dir, path, targetdir, ordering=ordering)
92+
if not_in_sidebar:
93+
print(
94+
"The following files are not in the sidebar:",
95+
*not_in_sidebar,
96+
sep="\n- ",
97+
file=sys.stderr,
98+
)
99+
print(
100+
"Edit SIDEBAR_ORDERING in .ci_scripts/sphinx_markdown_to_docusaurus.py",
101+
file=sys.stderr,
102+
)
103+
sys.exit(1)
104+
105+
106+
if __name__ == "__main__":
107+
build_dir, targetdir = sys.argv[1:3]
108+
main(build_dir, targetdir)

‎.ci_scripts/update_docs

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,27 @@
22

33
set -ex
44

5+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6+
BASE_DIR=$(dirname $HERE)
7+
58
# configure bot account
69
if [[ "$CI" == "1" ]]; then
710
git config --global user.email "[email protected]"
811
git config --global user.name "conda-forge-admin"
912
git checkout -b new_site_content
1013
fi
1114

12-
python .ci_scripts/generate_cfep_index.py
13-
14-
pushd sphinx/src
15-
15+
pushd "$BASE_DIR/sphinx/src"
16+
make clean
17+
rm -rf "$BASE_DIR/static-sphinx"
1618
# -W --keep-going: list all warnings but fail build in case there are any
17-
# -n: check validity of all links
18-
make html SPHINXOPTS="-W --keep-going -n"
19-
linkcheck_failed=0
20-
make linkcheck > /dev/null || linkcheck_failed=1
21-
python ../../.ci_scripts/display_linkcheck.py _build/linkcheck/output.json
22-
23-
if [[ "${GHREF}" != "refs/heads/main" ]]; then
24-
test "$linkcheck_failed" -eq 0
25-
fi
26-
27-
# Move rendered Sphinx docs to a static directory for Docusaurus to use
28-
rm -rf ../../static-sphinx || true
29-
mkdir -p ../../static-sphinx
30-
mv _build/html ../../static-sphinx/docs
31-
rm -rf _build
19+
make markdown SPHINXOPTS="-W --keep-going"
3220
popd
3321

22+
# Move rendered Sphinx markdown to Docusaurus
23+
python "$BASE_DIR/.ci_scripts/sphinx_markdown_to_docusaurus.py" "$BASE_DIR/sphinx/src/_build/markdown" docs/
24+
mkdir -p "$BASE_DIR/static-sphinx/_static"
25+
cp -r "$BASE_DIR/sphinx/src/_static/" "$BASE_DIR/static-sphinx/_static/"
3426
# Build docusaurus site
3527
npm install
3628
npm run build

‎.github/workflows/deploy.yml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,20 @@ jobs:
4141
publish_dir: ./build
4242
user_name: conda-forge-admin
4343
user_email: [email protected]
44+
45+
- name: Link Checker
46+
uses: lycheeverse/lychee-action@c053181aa0c3d17606addfe97a9075a32723548a # for v1.9.3
47+
with:
48+
token: ${{ secrets.GITHUB_TOKEN }}
49+
lycheeVersion: '0.14.3'
50+
args: >
51+
--no-progress
52+
--exclude 'https://polys.me/?$'
53+
--exclude 'https://kb43fqob7u-dsn.algolia.net/'
54+
--exclude '.*/404.html/'
55+
--exclude '.*,.*'
56+
--exclude-path './build/docs/orga/minutes/'
57+
--remap "https://conda-forge.org/status https://conda-forge.org/status"
58+
--remap "https://conda-forge.org/(.*) file://$(pwd)/build/\$1"
59+
'./build/**/*.html'
60+
'*.md'

‎.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ sphinx/newsfeed/demo/_build
2222

2323
# Sphinx output gets moved to Docusaurus' static folder
2424
/static-sphinx
25+
/docs
2526

2627
# Docusaurus dependencies
2728
/node_modules
@@ -42,3 +43,7 @@ sphinx/newsfeed/demo/_build
4243
npm-debug.log*
4344
yarn-debug.log*
4445
yarn-error.log*
46+
47+
# pyc files
48+
*.pyc
49+
__pycache__/

0 commit comments

Comments
 (0)