minor: simplify docs build process & pin pip package versions (#17816)

Author: Jefffrey

.github/dependabot.yml

@@ -50,3 +50,8 @@ updates:
       interval: "daily"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
+  - package-ecosystem: "pip"
+    directory: "/docs"
+    schedule:
+      interval: "weekly"
+    labels: [auto-dependencies]

docs/.gitignore

@@ -15,7 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-build
-temp
+temp/
+build/
 venv/
 .python-version
+__pycache__/

docs/Makefile

@@ -21,7 +21,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    ?= -W
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build

docs/README.md

@@ -28,22 +28,36 @@ https://datafusion.apache.org/ as part of the release process.
 It's recommended to install build dependencies and build the documentation
 inside a Python virtualenv.
 
-- Python
-- `pip install -r requirements.txt`
+```sh
+python3 -m venv venv
+pip install -r requirements.txt
+```
+
+If using [uv](https://docs.astral.sh/uv/) the script can be run like so without
+needing to create a virtual environment:
+
+```sh
+uv run --with-requirements requirements.txt bash build.sh
+```
 
 ## Build & Preview
 
 Run the provided script to build the HTML pages.
 
 ```bash
+# If using venv, ensure you have activated it
 ./build.sh
 ```
 
-The HTML will be generated into a `build` directory.
+The HTML will be generated into a `build` directory. Open `build/html/index.html`
+in your preferred browser, e.g.
 
 Preview the site on Linux by running this command.
 
 ```bash
+# On macOS
+open build/html/index.html
+# On Linux with Firefox
 firefox build/html/index.html
 ```
 

docs/build.sh

@@ -20,12 +20,5 @@
 
 set -e
 rm -rf build 2> /dev/null
-rm -rf temp 2> /dev/null
-mkdir temp
-cp -rf source/* temp/
-# replace relative URLs with absolute URLs
-sed -i -e 's/\.\.\/\.\.\/\.\.\//https:\/\/github.com\/apache\/arrow-datafusion\/blob\/main\//g' temp/contributor-guide/index.md
 
-python rustdoc_trim.py
-
-make SOURCEDIR=`pwd`/temp SPHINXOPTS=-W html
+make html

docs/make.bat

@@ -23,7 +23,8 @@ REM Command file for Sphinx documentation
 
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
-)
+)
+set SPHINXOPTS=-W
 set SOURCEDIR=source
 set BUILDDIR=build
 

docs/requirements.txt

@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-sphinx
-sphinx-reredirects
+sphinx==8.2.3
+sphinx-reredirects==1.0.0
 pydata-sphinx-theme==0.8.0
-myst-parser
-maturin
-jinja2
-setuptools>=48.0.0
+myst-parser==4.0.1
+maturin==1.9.4
+jinja2==3.1.6
+setuptools==80.9.0

docs/rustdoc_trim.py

@@ -16,8 +16,7 @@
 # under the License.
 
 import re
-
-from pathlib import Path
+from sphinx.application import Sphinx
 
 # Regex pattern to match Rust code blocks in Markdown
 RUST_CODE_BLOCK_PATTERN = re.compile(r"```rust\s*(.*?)```", re.DOTALL)
@@ -46,30 +45,16 @@ def _process_code_block(match):
     return RUST_CODE_BLOCK_PATTERN.sub(_process_code_block, markdown_content)
 
 
-# Example usage
-def process_markdown_file(file_path):
-    # Read the Markdown file
-    with open(file_path, "r", encoding="utf-8") as file:
-        markdown_content = file.read()
-
+def process_source_file(app: Sphinx, docname: str, source: list[str]):
+    original_content = source[0]
     # Remove lines starting with '#' in Rust code blocks
-    updated_markdown_content = remove_hashtag_lines_in_rust_blocks(markdown_content)
-
-    # Write the updated content back to the Markdown file
-    with open(file_path, "w", encoding="utf-8") as file:
-        file.write(updated_markdown_content)
-
-    print(f"Done processing file: {file_path}")
-
-
-root_directory = Path("./temp/library-user-guide")
-for file_path in root_directory.rglob("*.md"):
-    print(f"Processing file: {file_path}")
-    process_markdown_file(file_path)
+    modified_content = remove_hashtag_lines_in_rust_blocks(original_content)
+    source[0] = modified_content
 
-root_directory = Path("./temp/user-guide")
-for file_path in root_directory.rglob("*.md"):
-    print(f"Processing file: {file_path}")
-    process_markdown_file(file_path)
 
-print("All Markdown files processed.")
+def setup(app: Sphinx):
+    app.connect("source-read", process_source_file)
+    return dict(
+        parallel_read_safe=True,
+        parallel_write_safe=True,
+    )

docs/source/conf.py

@@ -26,16 +26,17 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+
+# To pickup rustdoc_trim.py
+sys.path.insert(0, os.path.abspath(".."))
 
 # -- Project information -----------------------------------------------------
 
-project = 'Apache DataFusion'
-copyright = '2019-2025, Apache Software Foundation'
-author = 'Apache Software Foundation'
+project = "Apache DataFusion"
+copyright = "2019-2025, Apache Software Foundation"
+author = "Apache Software Foundation"
 
 
 # -- General configuration ---------------------------------------------------
@@ -44,24 +45,25 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.doctest',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'myst_parser',
-    'sphinx_reredirects',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.doctest",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
+    "myst_parser",
+    "sphinx_reredirects",
+    "rustdoc_trim",
 ]
 
 source_suffix = {
-    '.rst': 'restructuredtext',
-    '.md': 'markdown',
+    ".rst": "restructuredtext",
+    ".md": "markdown",
 }
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -83,7 +85,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'pydata_sphinx_theme'
+html_theme = "pydata_sphinx_theme"
 
 html_theme_options = {
     "use_edit_page_button": True,
@@ -99,13 +101,11 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 html_logo = "_static/images/2x_bgwhite_original.png"
 
-html_css_files = [
-    "theme_overrides.css"
-]
+html_css_files = ["theme_overrides.css"]
 
 html_sidebars = {
     "**": ["docs-sidebar.html"],
@@ -121,9 +121,9 @@
 # presence of some special characters like: 🚀, å, {,... But this isn’t a major
 # issue for our documentation. So, suppress these warnings to keep our build
 # log cleaner.
-suppress_warnings = ['misc.highlighting_failure']
+suppress_warnings = ["misc.highlighting_failure"]
 
 redirects = {
     "library-user-guide/adding-udfs": "functions/index.html",
     "user-guide/runtime_configs": "configs.html",
-}
+}