From e5e7a3b273a86bbb41f264803f9aae526841feb5 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 16 Aug 2025 12:26:34 +0800 Subject: [PATCH 1/7] Migrate pdf-doc build to meson --- .github/workflows/doc-build-pdf.yml | 157 +++++++++------------------- .github/workflows/doc-build.yml | 2 +- pyproject.toml | 4 + src/sage_docbuild/builders.py | 34 +++++- tools/update-conda.py | 2 + 5 files changed, 87 insertions(+), 112 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 9e2726751cd..c861d0be0e6 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -2,18 +2,20 @@ name: Build documentation (PDF) on: pull_request: + merge_group: push: + tags: + # Match all release tags including beta, rc + - '[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.rc[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+' + branches: + - develop workflow_dispatch: # Allow to run manually - inputs: - platform: - description: 'Platform' - required: true - default: 'ubuntu-noble-standard' - docker_tag: - description: 'Docker tag' - required: true - default: 'dev' concurrency: # Cancel previous runs of this workflow for the same branch @@ -21,129 +23,72 @@ concurrency: cancel-in-progress: true env: - # Same as in build.yml - TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-incremental" - BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-with-targets:ci" - FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" - FROM_DOCKER_TARGET: "with-targets" - FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} - EXTRA_CONFIGURE_ARGS: --enable-fat-binary + PYTHON_VERSION: 3.11 jobs: - build-doc-pdf: + doc-pdf: runs-on: ubuntu-latest - services: - # https://docs.docker.com/build/ci/github-actions/local-registry/ - registry: - image: registry:2 - ports: - - 5000:5000 steps: - - name: Maximize build disk space - uses: easimon/maximize-build-space@v10 - with: - # need space in /var for Docker images - root-reserve-mb: 30000 - remove-dotnet: true - remove-android: true - remove-haskell: true - remove-codeql: true - remove-docker-images: true - name: Checkout uses: actions/checkout@v4 - - name: Install test prerequisites - # From docker.yml - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install tox - sudo apt-get clean - df -h + - name: Merge CI fixes from sagemath/sage run: | - mkdir -p upstream - .github/workflows/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log + .github/workflows/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - - # Building - - - name: Generate Dockerfile - # From docker.yml - run: | - tox -e ${{ env.TOX_ENV }} - cp .tox/${{ env.TOX_ENV }}/Dockerfile . - env: - # Only generate the Dockerfile, do not run 'docker build' here - DOCKER_TARGETS: "" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Cache conda packages + uses: actions/cache@v4 with: - driver-opts: network=host - - - name: Build Docker image - id: image - uses: docker/build-push-action@v6 + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ hashFiles('environment-${{ env.PYTHON_VERSION }}-linux.yml') }} + + - name: Compiler cache + uses: hendrikmuhs/ccache-action@v1.2 with: - # push and load may not be set together at the moment - push: true - load: false - context: . - tags: ${{ env.BUILD_IMAGE }} - target: with-targets - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - NUMPROC=6 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse - TARGETS_PRE=build/make/Makefile - TARGETS=ci-build-with-fallback - - - name: Start container - id: container - # Try to continue when "exporting to GitHub Actions Cache" failed with timeout - if: (success() || failure()) - run: | - docker run --name BUILD -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + key: ${{ runner.os }}-meson-${{ env.PYTHON_VERSION }} - # - # On PRs and pushes to branches - # - - - name: Update system packages - id: packages - if: (success() || failure()) && steps.container.outcome == 'success' + - name: Setup Conda environment + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + # Disabled for now due to + # https://github.com/conda-incubator/setup-miniconda/issues/379 + # miniforge-version: latest + use-mamba: true + channels: conda-forge + channel-priority: true + activate-environment: sage-dev + environment-file: environment-${{ env.PYTHON_VERSION }}-linux.yml + + - name: Build Sage + shell: bash -l {0} run: | - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy) - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export CC="ccache $CC" + export CXX="ccache $CXX" + pip install --no-build-isolation --config-settings=builddir=builddir . -v - - name: Build doc (PDF) - id: docbuild - if: (success() || failure()) && steps.packages.outcome == 'success' + - name: Build documentation + shell: bash -l {0} run: | - export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 - make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive sudo apt-get install texlive-latex-base + meson compile -C builddir doc-pdf + env: + SAGE_DOCBUILD_OPTS: "--include-tests-blocks" - name: Copy doc id: copy - if: (success() || failure()) && steps.docbuild.outcome == 'success' run: | mkdir -p ./doc - # We copy everything to a local folder - docker cp BUILD:/sage/local/share/doc/sage/pdf doc + cp -r builddir/src/doc/sage/pdf doc/ # Zip everything for increased performance zip -r doc-pdf.zip doc - name: Upload doc - if: (success() || failure()) && steps.copy.outcome == 'success' uses: actions/upload-artifact@v4 with: name: doc-pdf diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index c56e9a06346..ea4d0f95969 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -26,7 +26,7 @@ env: PYTHON_VERSION: 3.11 jobs: - build-doc: + doc-html: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/pyproject.toml b/pyproject.toml index d4d023ae190..01b867e7274 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,6 +205,10 @@ extra = [ "pkg:generic/qepcad", "pkg:generic/tides", ] +docs-pdf = [ + "pkg:generic/latexmk", + "pkg:generic/lualatex", +] [dependency-groups] test = ["pytest", "pytest-xdist", "coverage"] diff --git a/src/sage_docbuild/builders.py b/src/sage_docbuild/builders.py index 91035a01f1c..55d9aa94d30 100644 --- a/src/sage_docbuild/builders.py +++ b/src/sage_docbuild/builders.py @@ -265,13 +265,37 @@ def pdf(self): with open(tex_file, 'w') as f: f.write(ref) - make_target = "cd '%s' && $MAKE %s && mv -f *.pdf '%s'" - error_message = "failed to run $MAKE %s in %s" + make_cmd = os.environ.get('MAKE', 'make') command = 'all-pdf' + logger.debug(f"Running {make_cmd} {command} in {tex_dir}") + + proc = subprocess.run( + [make_cmd, command], + check=False, cwd=tex_dir, + capture_output=True, + text=True, + ) + + if proc.returncode != 0: + logger.error(f"stdout from {make_cmd}:\n{proc.stdout}") + logger.error(f"stderr from {make_cmd}:\n{proc.stderr}") + raise RuntimeError(f"failed to run {make_cmd} {command} in {tex_dir}") + + if proc.stdout: + logger.debug(f"make stdout:\n{proc.stdout}") + if proc.stderr: + # Still surface stderr even on success, but at debug level + logger.debug(f"make stderr:\n{proc.stderr}") + + # Move generated PDFs + for pdf in tex_dir.glob("*.pdf"): + try: + shutil.move(str(pdf), pdf_dir) + except Exception as e: + logger.error(f"Failed moving {pdf} to {pdf_dir}: {e}") + raise - if subprocess.call(make_target % (tex_dir, command, pdf_dir), close_fds=False, shell=True): - raise RuntimeError(error_message % (command, tex_dir)) - logger.warning(f"Build finished. The built documents can be found in {pdf_dir}.") + logger.info(f"Build finished. The built documents can be found in {pdf_dir}.") def clean(self, *args): shutil.rmtree(self._doctrees_dir()) diff --git a/tools/update-conda.py b/tools/update-conda.py index 5696b7ea90b..cbc50cb107f 100644 --- a/tools/update-conda.py +++ b/tools/update-conda.py @@ -168,6 +168,7 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st .replace("symengine", "python-symengine") .replace("memory_allocator", "memory-allocator") .replace("pkg:generic/r-lattice", "r-lattice") + .replace("pkg:generic/latexmk", "latexmk") for req in all_requirements } # Exclude requirements not available on conda (for a given platform) @@ -177,6 +178,7 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st "sagemath_giac", "pynormaliz", # due to https://github.com/sagemath/sage/issues/40214 "latte-integrale", # due to https://github.com/sagemath/sage/issues/40216 + "pkg:generic/lualatex", # texlive-core doesn't include lualatex } if platform in ("linux-aarch64", "osx-arm64"): exclude_packages |= { From bbbbf63a80c208958e99975e6afd86458767c668 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 16 Aug 2025 12:45:16 +0800 Subject: [PATCH 2/7] Install all required texlive packages --- .github/workflows/doc-build-pdf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index c861d0be0e6..4c8dcb283fb 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -1,4 +1,4 @@ -name: Build documentation (PDF) +name: Build documentation on: pull_request: @@ -75,7 +75,7 @@ jobs: shell: bash -l {0} run: | sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive sudo apt-get install texlive-latex-base + sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex) meson compile -C builddir doc-pdf env: SAGE_DOCBUILD_OPTS: "--include-tests-blocks" From d9cdd68bf0559dcc46b48b19aec9c5a32446ad36 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 16 Aug 2025 13:15:26 +0800 Subject: [PATCH 3/7] Install missing free_fonts --- .github/workflows/doc-build-pdf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 4c8dcb283fb..1a3e37a16f1 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -1,4 +1,4 @@ -name: Build documentation +name: Build documentation (PDF) on: pull_request: @@ -75,7 +75,7 @@ jobs: shell: bash -l {0} run: | sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex) + sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex free_fonts) meson compile -C builddir doc-pdf env: SAGE_DOCBUILD_OPTS: "--include-tests-blocks" From e57175d023727c6d1f7f596ea0a4c7fa123042c9 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 16 Aug 2025 13:41:57 +0800 Subject: [PATCH 4/7] Install xindy as well --- .github/workflows/doc-build-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 1a3e37a16f1..7c9b87828bb 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -75,7 +75,7 @@ jobs: shell: bash -l {0} run: | sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex free_fonts) + sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex free_fonts xindy) meson compile -C builddir doc-pdf env: SAGE_DOCBUILD_OPTS: "--include-tests-blocks" From 62a9bfd6365ea73cf954c6ee1ae169aa209a0ec0 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 16 Aug 2025 15:15:02 +0800 Subject: [PATCH 5/7] Fix location of produced pdfs --- .github/workflows/doc-build-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 7c9b87828bb..f278dae841e 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -84,7 +84,7 @@ jobs: id: copy run: | mkdir -p ./doc - cp -r builddir/src/doc/sage/pdf doc/ + cp -r builddir/src/doc/pdf doc/ # Zip everything for increased performance zip -r doc-pdf.zip doc From fb0e62770e74ed16402ecc5fb4826d20d7163b01 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Aug 2025 13:36:51 +0800 Subject: [PATCH 6/7] Properly handle `MAKE='make -j5'` --- src/sage_docbuild/builders.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sage_docbuild/builders.py b/src/sage_docbuild/builders.py index 55d9aa94d30..20721ab43a1 100644 --- a/src/sage_docbuild/builders.py +++ b/src/sage_docbuild/builders.py @@ -66,6 +66,7 @@ import os import pickle import re +import shlex import shutil import subprocess import sys @@ -266,11 +267,11 @@ def pdf(self): f.write(ref) make_cmd = os.environ.get('MAKE', 'make') - command = 'all-pdf' - logger.debug(f"Running {make_cmd} {command} in {tex_dir}") + command = shlex.split(make_cmd) + ['all-pdf'] + logger.debug(f"Running {' '.join(command)} in {tex_dir}") proc = subprocess.run( - [make_cmd, command], + command, check=False, cwd=tex_dir, capture_output=True, text=True, @@ -279,7 +280,7 @@ def pdf(self): if proc.returncode != 0: logger.error(f"stdout from {make_cmd}:\n{proc.stdout}") logger.error(f"stderr from {make_cmd}:\n{proc.stderr}") - raise RuntimeError(f"failed to run {make_cmd} {command} in {tex_dir}") + raise RuntimeError(f"failed to run {' '.join(command)} in {tex_dir}") if proc.stdout: logger.debug(f"make stdout:\n{proc.stdout}") From 4487fc2a879eb7b967aacba0e33215c9146020c6 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 23 Aug 2025 14:09:13 +0800 Subject: [PATCH 7/7] Improve "docs-pdf" metadata --- pyproject.toml | 11 +++++++---- tools/update-conda.py | 1 - 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1248cc6bf15..22b0bae5511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,10 +205,13 @@ extra = [ "pkg:generic/qepcad", "pkg:generic/tides", ] -docs-pdf = [ - "pkg:generic/latexmk", - "pkg:generic/lualatex", -] +# Uncommented for now as most devs probably don't need to run it locally +# docs-pdf = [ +# "pkg:generic/latexmk", +# "pkg:generic/lualatex", +# "pkg:fonts/freefonts", +# "pkg:generic/xindy", +# ] [dependency-groups] test = ["pytest", "pytest-xdist", "coverage"] diff --git a/tools/update-conda.py b/tools/update-conda.py index cbc50cb107f..6dc88b7fde8 100644 --- a/tools/update-conda.py +++ b/tools/update-conda.py @@ -178,7 +178,6 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st "sagemath_giac", "pynormaliz", # due to https://github.com/sagemath/sage/issues/40214 "latte-integrale", # due to https://github.com/sagemath/sage/issues/40216 - "pkg:generic/lualatex", # texlive-core doesn't include lualatex } if platform in ("linux-aarch64", "osx-arm64"): exclude_packages |= {