Improve the Sphinx configs for laTeX/PDF builds (#12662)

* Improve the Sphinx configs for Latex/PDF builds

This also cleans up a bunch of RST and UML formats and files, which
should work and improve HTML output as well.

We also try to automate more of the builds, setting up the appropraite
pyenv and also making sure we deal with macOS app focus.

Finally, this upgrades PlantUML to v1.2025.10.

Build improvements

* Cleaning up some existing docs for formatting

* More TOC reorgs

* Chapter page breaks

Author: zwoop

doc/.gitignore

@@ -1,3 +1,4 @@
 uml/images
 ext/local-config.py
 Pipfile.lock
+conf.py

doc/CMakeLists.txt

@@ -15,19 +15,19 @@
 #
 #######################
 
-set(PLANTUML_DATE "1.2018.1")
+set(PLANTUML_DATE "1.2025.10")
 set(PLANTUML_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/plantuml-${PLANTUML_DATE}.tar.bz2)
 set(PLANTUML_JAR ${CMAKE_CURRENT_BINARY_DIR}/plantuml-${PLANTUML_DATE}/plantuml.jar)
 
 file(DOWNLOAD https://ci.trafficserver.apache.org/bintray/plantuml-${PLANTUML_DATE}.tar.bz2 ${PLANTUML_ARCHIVE}
-     EXPECTED_HASH SHA1=4dbf218641a777007f9bc72ca8017a41a23e1081
+     EXPECTED_HASH SHA1=248b535bb361119d154e3435dfe5382ddeca55d6
 )
 file(ARCHIVE_EXTRACT INPUT ${PLANTUML_ARCHIVE} PATTERNS *.jar)
 
 configure_file(ext/local-config.cmake.in.py ext/local-config.py)
 configure_file(ext/traffic-server.cmake.in.py ext/traffic-server.py)
 configure_file(ext/extras/txnbox.cmake.in.py ext/extras/txnbox.py)
-configure_file(conf.cmake.in.py conf.py)
+configure_file(conf.cmake.in.py conf.py @ONLY)
 configure_file(manpages.cmake.in.py manpages.py)
 
 # copy static directory to build
@@ -65,31 +65,73 @@ foreach(UML ${UML_FILES})
   list(APPEND SVG_FILES ${CMAKE_CURRENT_SOURCE_DIR}/uml/images/${uml_name}.svg)
   add_custom_command(
     OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/uml/images/${uml_name}.svg
-    COMMAND ${Java_JAVA_EXECUTABLE} -jar ${PLANTUML_JAR} -o ${CMAKE_CURRENT_SOURCE_DIR}/uml/images -tsvg -graphvizdot
-            ${GRAPHVIZ_DOT} ${CMAKE_CURRENT_SOURCE_DIR}/${UML}
+    COMMAND ${CMAKE_COMMAND} -E env GRAPHVIZ_DOT=${GRAPHVIZ_DOT} ${Java_JAVA_EXECUTABLE} -Djava.awt.headless=true
+            -Dapple.awt.UIElement=true -jar ${PLANTUML_JAR} -o ${CMAKE_CURRENT_SOURCE_DIR}/uml/images -tsvg
+            ${CMAKE_CURRENT_SOURCE_DIR}/${UML}
     DEPENDS ${UML}
     VERBATIM
   )
 endforeach()
 add_custom_target(generate_svg_from_uml DEPENDS ${SVG_FILES})
 
 # Docs are built with python so we need a target to setup pipenv
-set(RUNPIPENV PIPENV_PIPFILE=${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${PipEnv})
+# Copy Pipfile to build directory and explicitly point pipenv to it
+# PIPENV_VENV_IN_PROJECT=1 creates .venv next to the Pipfile location (if not already in a venv)
+# PIPENV_VERBOSITY=-1 suppresses warnings
+# If user is already in a virtualenv, pipenv will use it; otherwise creates one in build dir
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${CMAKE_CURRENT_BINARY_DIR}/Pipfile COPYONLY)
+set(RUNPIPENV PIPENV_PIPFILE=${CMAKE_CURRENT_BINARY_DIR}/Pipfile PIPENV_VENV_IN_PROJECT=1 PIPENV_VERBOSITY=-1 ${PipEnv})
+# Create a marker file to indicate pipenv setup is complete
+# This is more reliable than using Pipfile.lock which might already exist
 add_custom_command(
-  OUTPUT Pipfile.lock
-  COMMAND ${RUNPIPENV} install
-  COMMENT "Setup pipenv"
-  DEPENDS Pipfile
+  OUTPUT .pipenv_installed
+  COMMAND ${RUNPIPENV} install --python ${Python3_EXECUTABLE}
+  COMMAND ${CMAKE_COMMAND} -E touch .pipenv_installed
+  COMMENT "Setup pipenv in build directory (${CMAKE_CURRENT_BINARY_DIR}/.venv) and install packages"
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Pipfile
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  VERBATIM
 )
 
 add_custom_target(
   generate_docs
-  COMMAND ${RUNPIPENV} run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version
-  COMMAND ${RUNPIPENV} run python -m sphinx -W -c ${CMAKE_CURRENT_BINARY_DIR} -b html ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python -m sphinx -W -c ${CMAKE_CURRENT_BINARY_DIR} -b html ${CMAKE_CURRENT_SOURCE_DIR}
           ${CMAKE_CURRENT_BINARY_DIR}/docbuild/html
-  DEPENDS generate_docs_setup generate_svg_from_uml Pipfile.lock
+  DEPENDS generate_docs_setup .pipenv_installed
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Sphinx PlantUML extension now generates diagrams automatically in build tree"
+)
+# Generate PDF documentation (Letter paper size - US standard)
+add_custom_target(
+  generate_pdf
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_paper ${CMAKE_CURRENT_SOURCE_DIR}
+          ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex
+  COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex latexmk -pdf -interaction=nonstopmode -f
+          ApacheTrafficServer.tex
+  DEPENDS generate_docs_setup .pipenv_installed
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building PDF documentation with Sphinx and LaTeX (Letter paper, PlantUML generates PNG automatically)"
+)
+
+# Generate PDF documentation (A4 paper size - International standard)
+add_custom_target(
+  generate_pdf_a4
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version
+  COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV}
+          run python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_a4 ${CMAKE_CURRENT_SOURCE_DIR}
+          ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex-a4
+  COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex-a4 latexmk -pdf -interaction=nonstopmode
+          -f ApacheTrafficServer.tex
+  DEPENDS generate_docs_setup .pipenv_installed
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building PDF documentation with Sphinx and LaTeX (A4 paper, PlantUML generates PNG automatically)"
 )
 
 #add_custom_command(

doc/README.md

@@ -15,20 +15,30 @@ documentation will also require Java and graphviz system packages to be
 installed.
 
 ## Build Steps
-Building the docs is a relatively simple matter of passing
-`-DENABLE_DOCS=ON` to cmake (docs generation is off by default), and
-then using the `generate_docs` build target. The build steps for the
-`generate_docs` target will install a Pipenv using `docs/Pipfile` and do
-what is necessary to build and install the docs. Thus the following
-steps should build the docs:
+Building the docs requires passing `-DENABLE_DOCS=ON` to cmake (docs generation
+is off by default), and then using the appropriate build target. The build steps
+will automatically install a Pipenv virtual environment using `docs/Pipfile` and
+do what is necessary to build the docs.
 
+### Building HTML Documentation
 ```sh
 cmake -B docs-build -DENABLE_DOCS=ON
 cmake --build docs-build --target generate_docs
 ```
 
-The generated HTML docs will be installed in the cmake build's `doc/docbuild/html`
-directory, `docs-build/doc/docbuild/html` per the above example.
+The generated HTML docs will be in `docs-build/doc/docbuild/html`.
+
+### Building PDF Documentation
+```sh
+# Letter paper size (US standard)
+cmake --build docs-build --target generate_pdf
+
+# A4 paper size (International standard)
+cmake --build docs-build --target generate_pdf_a4
+```
+
+The generated PDF will be in `docs-build/doc/docbuild/latex/ApacheTrafficServer.pdf`
+(or `docs-build/doc/docbuild/latex-a4/` for A4 format).
 
 Once the build completes, you can use Python's
 [http.server](https://docs.python.org/3/library/http.server.html) module to

doc/admin-guide/configuration/hrw4u.en.rst

@@ -68,7 +68,7 @@ virtualenv or system-wide using:
 
 .. code-block:: none
 
-   pipx install dist/hrw4u-1.0.0-py3-none-any.whl
+   pipx install dist/hrw4u-1.4.0-py3-none-any.whl
 
 Using
 -----
@@ -86,6 +86,10 @@ Doing a compile is simply:
 
    hrw4u some_file.hrw4u
 
+in Addition to ``hrw4u``, you also have the reverse tool, converting existing ``header_rewrite``
+configurations to ``hrw4u``. This tool is named ``u4wrh``. For people using IDEs, the package also
+provides an LSP for this language, named ``hrw4u-lsp``.
+
 Syntax Differences
 ==================
 

doc/admin-guide/layer-4-routing.en.rst

@@ -28,7 +28,7 @@ accomplished by examining the initial data from the inbound connection to decide
 destination. The initial data is then sent to the destination and subsequently |TS| forwards all
 data read on one connection to the other and vice versa.
 
-.. figure:: ../uml/images/l4-basic-sequence.svg
+.. uml:: ../uml/l4-basic-sequence.uml
    :align: center
 
 In this way it acts similarly to `nc <https://linux.die.net/man/1/nc>`__.
@@ -81,11 +81,9 @@ services.
 
 The basic set up is therefore
 
-.. figure:: ../uml/images/l4-example-cdn-layout.svg
+.. uml:: ../uml/l4-example-cdn-layout.uml
    :align: center
-
-   A Client connects to an edge |TS| which forwards the connection to the internal Service.
-   The Client then negotiates TLS with the Service.
+   :caption: A Client connects to an edge |TS| which forwards the connection to the internal Service. The Client then negotiates TLS with the Service.
 
 
 For the example, let us define two services inside the corporate network of Example, Inc.
@@ -118,7 +116,7 @@ In addition to this, in the :file:`records.yaml` file, edit ``connect_ports`` li
 
 The sequence of network activity for a Client connecting to ``service-2`` is
 
-.. figure:: ../uml/images/l4-sni-routing-seq.svg
+.. uml:: ../uml/l4-sni-routing-seq.uml
    :align: center
 
 Note the destination for the outbound TCP connection and the HTTP ``CONNECT`` is the same. If this
@@ -138,7 +136,7 @@ Pre-warming TLS Tunnel
 Pre-warming TLS Tunnel reduces the latency of TLS connections (``forward_route`` and ``partial_blind_route`` type SNI
 Routing). When this feature is enabled, each ET_NET thread makes TLS connections pool per routing type, SNI, and ALPN.
 
-.. figure:: ../uml/images/l4-pre-warming-overview.svg
+.. uml:: ../uml/l4-pre-warming-overview.uml
    :align: center
 
 Stats for connection pools are registered dynamically on start up. Details in :ref:`pre-warming-tls-tunnel-stats`.