diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 9e2726751cd..f278dae841e 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -2,18 +2,20 @@ name: Build documentation (PDF) on: pull_request: + merge_group: push: + tags: + # Match all release tags including beta, rc + - '[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.rc[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+' + branches: + - develop workflow_dispatch: # Allow to run manually - inputs: - platform: - description: 'Platform' - required: true - default: 'ubuntu-noble-standard' - docker_tag: - description: 'Docker tag' - required: true - default: 'dev' concurrency: # Cancel previous runs of this workflow for the same branch @@ -21,129 +23,72 @@ concurrency: cancel-in-progress: true env: - # Same as in build.yml - TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-incremental" - BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-with-targets:ci" - FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" - FROM_DOCKER_TARGET: "with-targets" - FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} - EXTRA_CONFIGURE_ARGS: --enable-fat-binary + PYTHON_VERSION: 3.11 jobs: - build-doc-pdf: + doc-pdf: runs-on: ubuntu-latest - services: - # https://docs.docker.com/build/ci/github-actions/local-registry/ - registry: - image: registry:2 - ports: - - 5000:5000 steps: - - name: Maximize build disk space - uses: easimon/maximize-build-space@v10 - with: - # need space in /var for Docker images - root-reserve-mb: 30000 - remove-dotnet: true - remove-android: true - remove-haskell: true - remove-codeql: true - remove-docker-images: true - name: Checkout uses: actions/checkout@v4 - - name: Install test prerequisites - # From docker.yml - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install tox - sudo apt-get clean - df -h + - name: Merge CI fixes from sagemath/sage run: | - mkdir -p upstream - .github/workflows/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log + .github/workflows/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - - # Building - - - name: Generate Dockerfile - # From docker.yml - run: | - tox -e ${{ env.TOX_ENV }} - cp .tox/${{ env.TOX_ENV }}/Dockerfile . - env: - # Only generate the Dockerfile, do not run 'docker build' here - DOCKER_TARGETS: "" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Cache conda packages + uses: actions/cache@v4 with: - driver-opts: network=host - - - name: Build Docker image - id: image - uses: docker/build-push-action@v6 + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ hashFiles('environment-${{ env.PYTHON_VERSION }}-linux.yml') }} + + - name: Compiler cache + uses: hendrikmuhs/ccache-action@v1.2 with: - # push and load may not be set together at the moment - push: true - load: false - context: . - tags: ${{ env.BUILD_IMAGE }} - target: with-targets - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - NUMPROC=6 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse - TARGETS_PRE=build/make/Makefile - TARGETS=ci-build-with-fallback - - - name: Start container - id: container - # Try to continue when "exporting to GitHub Actions Cache" failed with timeout - if: (success() || failure()) - run: | - docker run --name BUILD -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + key: ${{ runner.os }}-meson-${{ env.PYTHON_VERSION }} - # - # On PRs and pushes to branches - # - - - name: Update system packages - id: packages - if: (success() || failure()) && steps.container.outcome == 'success' + - name: Setup Conda environment + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + # Disabled for now due to + # https://github.com/conda-incubator/setup-miniconda/issues/379 + # miniforge-version: latest + use-mamba: true + channels: conda-forge + channel-priority: true + activate-environment: sage-dev + environment-file: environment-${{ env.PYTHON_VERSION }}-linux.yml + + - name: Build Sage + shell: bash -l {0} run: | - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy) - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export CC="ccache $CC" + export CXX="ccache $CXX" + pip install --no-build-isolation --config-settings=builddir=builddir . -v - - name: Build doc (PDF) - id: docbuild - if: (success() || failure()) && steps.packages.outcome == 'success' + - name: Build documentation + shell: bash -l {0} run: | - export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 - make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex free_fonts xindy) + meson compile -C builddir doc-pdf + env: + SAGE_DOCBUILD_OPTS: "--include-tests-blocks" - name: Copy doc id: copy - if: (success() || failure()) && steps.docbuild.outcome == 'success' run: | mkdir -p ./doc - # We copy everything to a local folder - docker cp BUILD:/sage/local/share/doc/sage/pdf doc + cp -r builddir/src/doc/pdf doc/ # Zip everything for increased performance zip -r doc-pdf.zip doc - name: Upload doc - if: (success() || failure()) && steps.copy.outcome == 'success' uses: actions/upload-artifact@v4 with: name: doc-pdf diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index c56e9a06346..ea4d0f95969 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -26,7 +26,7 @@ env: PYTHON_VERSION: 3.11 jobs: - build-doc: + doc-html: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/pyproject.toml b/pyproject.toml index 143f89600d9..22b0bae5511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,6 +205,13 @@ extra = [ "pkg:generic/qepcad", "pkg:generic/tides", ] +# Uncommented for now as most devs probably don't need to run it locally +# docs-pdf = [ +# "pkg:generic/latexmk", +# "pkg:generic/lualatex", +# "pkg:fonts/freefonts", +# "pkg:generic/xindy", +# ] [dependency-groups] test = ["pytest", "pytest-xdist", "coverage"] diff --git a/src/sage_docbuild/builders.py b/src/sage_docbuild/builders.py index 91035a01f1c..20721ab43a1 100644 --- a/src/sage_docbuild/builders.py +++ b/src/sage_docbuild/builders.py @@ -66,6 +66,7 @@ import os import pickle import re +import shlex import shutil import subprocess import sys @@ -265,13 +266,37 @@ def pdf(self): with open(tex_file, 'w') as f: f.write(ref) - make_target = "cd '%s' && $MAKE %s && mv -f *.pdf '%s'" - error_message = "failed to run $MAKE %s in %s" - command = 'all-pdf' + make_cmd = os.environ.get('MAKE', 'make') + command = shlex.split(make_cmd) + ['all-pdf'] + logger.debug(f"Running {' '.join(command)} in {tex_dir}") + + proc = subprocess.run( + command, + check=False, cwd=tex_dir, + capture_output=True, + text=True, + ) + + if proc.returncode != 0: + logger.error(f"stdout from {make_cmd}:\n{proc.stdout}") + logger.error(f"stderr from {make_cmd}:\n{proc.stderr}") + raise RuntimeError(f"failed to run {' '.join(command)} in {tex_dir}") + + if proc.stdout: + logger.debug(f"make stdout:\n{proc.stdout}") + if proc.stderr: + # Still surface stderr even on success, but at debug level + logger.debug(f"make stderr:\n{proc.stderr}") + + # Move generated PDFs + for pdf in tex_dir.glob("*.pdf"): + try: + shutil.move(str(pdf), pdf_dir) + except Exception as e: + logger.error(f"Failed moving {pdf} to {pdf_dir}: {e}") + raise - if subprocess.call(make_target % (tex_dir, command, pdf_dir), close_fds=False, shell=True): - raise RuntimeError(error_message % (command, tex_dir)) - logger.warning(f"Build finished. The built documents can be found in {pdf_dir}.") + logger.info(f"Build finished. The built documents can be found in {pdf_dir}.") def clean(self, *args): shutil.rmtree(self._doctrees_dir()) diff --git a/tools/update-conda.py b/tools/update-conda.py index 5696b7ea90b..6dc88b7fde8 100644 --- a/tools/update-conda.py +++ b/tools/update-conda.py @@ -168,6 +168,7 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st .replace("symengine", "python-symengine") .replace("memory_allocator", "memory-allocator") .replace("pkg:generic/r-lattice", "r-lattice") + .replace("pkg:generic/latexmk", "latexmk") for req in all_requirements } # Exclude requirements not available on conda (for a given platform)