diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f8c468475470c..7db0100c6b3e9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,6 +143,22 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + check_refcounts: + name: 'Check if refcounts.dat is up to date' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Check refcounts.dat + run: make -C Doc/tools check_refcounts + build_windows: name: >- Windows @@ -554,6 +570,7 @@ jobs: - check-docs - check_autoconf_regen - check_generated_files + - check_refcounts - build_macos - build_ubuntu - build_ubuntu_ssltests @@ -590,6 +607,7 @@ jobs: && ' check_autoconf_regen, check_generated_files, + check_refcounts, build_macos, build_ubuntu, build_ubuntu_ssltests, diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ccde03f91983df..4b175fbbe90279 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,3 +24,17 @@ jobs: with: python-version: "3.x" - uses: pre-commit/action@v3.0.1 + + check_refcounts_format: + name: 'Check if refcounts.dat is correctly formatted' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Check refcounts.dat format + run: make -C Doc/tools check_refcounts diff --git a/Doc/tools/Makefile b/Doc/tools/Makefile new file mode 100644 index 00000000000000..6b51f89a5e1a2a --- /dev/null +++ b/Doc/tools/Makefile @@ -0,0 +1,335 @@ +# Makefile for Python documentation tools +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# You can set these variables from the command line. +PYTHON = python3 +VENVDIR = ./venv +SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build +BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb +JOBS = auto +PAPER = +SOURCES = +DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) +REQUIREMENTS = requirements.txt +SPHINXERRORHANDLING = -W + +# Internal variables. +PAPEROPT_a4 = -D latex_elements.papersize=a4paper +PAPEROPT_letter = -D latex_elements.papersize=letterpaper + +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove build files" + @echo " venv to create a venv with necessary tools" + @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" + @echo " htmlview to open the index page built by the html target in your browser" + @echo " htmllive to rebuild and reload HTML files in your browser" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " text to make plain text files" + @echo " texinfo to make Texinfo file" + @echo " epub to make EPUB files" + @echo " changes to make an overview over all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " coverage to check documentation coverage for library and C API" + @echo " doctest to run doctests in the documentation" + @echo " pydoc-topics to regenerate the pydoc topics file" + @echo " dist to create a \"dist\" directory with archived docs for download" + @echo " check to run a check for frequent markup errors" + +.PHONY: build +build: + -mkdir -p build +# Look first for a Misc/NEWS file (building from a source release tarball +# or old repo) and use that, otherwise look for a Misc/NEWS.d directory +# (building from a newer repo) and use blurb to generate the NEWS file. + @if [ -f ../Misc/NEWS ] ; then \ + echo "Using existing Misc/NEWS file"; \ + cp ../Misc/NEWS build/NEWS; \ + elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + if [ -d ../Misc/NEWS.d ]; then \ + echo "Building NEWS from Misc/NEWS.d with blurb"; \ + $(BLURB) merge -f build/NEWS; \ + else \ + echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + exit 1; \ + fi \ + else \ + echo ""; \ + echo "Missing the required blurb or sphinx-build tools."; \ + echo "Please run 'make venv' to install local copies."; \ + echo ""; \ + exit 1; \ + fi + $(SPHINXBUILD) $(ALLSPHINXOPTS) + @echo + +.PHONY: html +html: BUILDER = html +html: build + @echo "Build finished. The HTML pages are in build/html." + +.PHONY: htmlhelp +htmlhelp: BUILDER = htmlhelp +htmlhelp: build + @echo "Build finished; now you can run HTML Help Workshop with the" \ + "build/htmlhelp/pydoc.hhp project file." + +.PHONY: latex +latex: BUILDER = latex +latex: build + @echo "Build finished; the LaTeX files are in build/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +.PHONY: text +text: BUILDER = text +text: build + @echo "Build finished; the text files are in build/text." + +.PHONY: texinfo +texinfo: BUILDER = texinfo +texinfo: build + @echo "Build finished; the python.texi file is in build/texinfo." + @echo "Run \`make info' in that directory to run it through makeinfo." + +.PHONY: epub +epub: BUILDER = epub +epub: build + @echo "Build finished; the epub files are in build/epub." + +.PHONY: changes +changes: BUILDER = changes +changes: build + @echo "The overview file is in build/changes." + +.PHONY: linkcheck +linkcheck: BUILDER = linkcheck +linkcheck: + @$(MAKE) build BUILDER=$(BUILDER) || { \ + echo "Link check complete; look for any errors in the above output" \ + "or in build/$(BUILDER)/output.txt"; \ + false; } + +.PHONY: coverage +coverage: BUILDER = coverage +coverage: build + @echo "Coverage finished; see c.txt and python.txt in build/coverage" + +.PHONY: doctest +doctest: BUILDER = doctest +doctest: + @$(MAKE) build BUILDER=$(BUILDER) || { \ + echo "Testing of doctests in the sources finished, look at the" \ + "results in build/doctest/output.txt"; \ + false; } + +.PHONY: pydoc-topics +pydoc-topics: BUILDER = pydoc-topics +pydoc-topics: build + @echo "Building finished; now run this:" \ + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + +.PHONY: gettext +gettext: BUILDER = gettext +gettext: SPHINXOPTS += -d build/doctrees-gettext +gettext: build + +.PHONY: htmlview +htmlview: html + $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" + +.PHONY: htmllive +htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 +htmllive: _ensure-sphinx-autobuild html + +.PHONY: clean +clean: clean-venv + -rm -rf build/* + +.PHONY: clean-venv +clean-venv: + rm -rf $(VENVDIR) + +.PHONY: venv +venv: + @if [ -d $(VENVDIR) ] ; then \ + echo "venv already exists."; \ + echo "To recreate it, remove it first with \`make clean-venv'."; \ + else \ + echo "Creating venv in $(VENVDIR)"; \ + if $(UV) --version >/dev/null 2>&1; then \ + $(UV) venv $(VENVDIR); \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ + $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + fi; \ + echo "The venv has been created in the $(VENVDIR) directory"; \ + fi + +.PHONY: dist-no-html +dist-no-html: dist-text dist-pdf dist-epub dist-texinfo + +.PHONY: dist +dist: + rm -rf dist + mkdir -p dist + $(MAKE) dist-html + $(MAKE) dist-text + $(MAKE) dist-pdf + $(MAKE) dist-epub + $(MAKE) dist-texinfo + +.PHONY: dist-html +dist-html: + # archive the HTML + @echo "Building HTML..." + mkdir -p dist + rm -rf build/html + find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; + $(MAKE) html + cp -pPR build/html dist/python-$(DISTVERSION)-docs-html + tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) + rm -r dist/python-$(DISTVERSION)-docs-html + rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" + +.PHONY: dist-text +dist-text: + # archive the text build + @echo "Building text..." + mkdir -p dist + rm -rf build/text + find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \; + $(MAKE) text + cp -pPR build/text dist/python-$(DISTVERSION)-docs-text + tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) + rm -r dist/python-$(DISTVERSION)-docs-text + rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" + +.PHONY: dist-pdf +dist-pdf: + # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." + mkdir -p dist + rm -rf build/latex + find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \; + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip + cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" + +.PHONY: dist-epub +dist-epub: + # copy the epub build + @echo "Building EPUB..." + mkdir -p dist + rm -rf build/epub + rm -f dist/python-$(DISTVERSION)-docs.epub + $(MAKE) epub + cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" + +.PHONY: dist-texinfo +dist-texinfo: + # archive the texinfo build + @echo "Building Texinfo..." + mkdir -p dist + rm -rf build/texinfo + find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \; + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo + cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo + tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) + rm -r dist/python-$(DISTVERSION)-docs-texinfo + rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" + +.PHONY: _ensure-package +_ensure-package: venv + if $(UV) --version >/dev/null 2>&1; then \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \ + else \ + $(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \ + fi + +.PHONY: _ensure-pre-commit +_ensure-pre-commit: + $(MAKE) _ensure-package PACKAGE=pre-commit + +.PHONY: _ensure-sphinx-autobuild +_ensure-sphinx-autobuild: + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild + +.PHONY: check +check: _ensure-pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files + +.PHONY: serve +serve: + @echo "The serve target was removed, use htmllive instead (see gh-80510)" + +# Targets for daily automated doc build +# By default, Sphinx only rebuilds pages where the page content has changed. +# This means it doesn't always pick up changes to preferred link targets, etc +# To ensure such changes are picked up, we build the published docs with +# `-E` (to ignore the cached environment) and `-a` (to ignore already existing +# output files) + +# for development releases: always build +.PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) +autobuild-dev: + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) + +# for HTML-only rebuilds +.PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) +autobuild-dev-html: + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) + +# for stable releases: only build if not in pre-release stage (alpha, beta) +# release candidate downloads are okay, since the stable tree can be in that stage +.PHONY: autobuild-stable +autobuild-stable: + @case $(DISTVERSION) in *[ab]*) \ + echo "Not building; $(DISTVERSION) is not a release version."; \ + exit 1;; \ + esac + @$(MAKE) autobuild-dev + +.PHONY: autobuild-stable-html +autobuild-stable-html: + @case $(DISTVERSION) in *[ab]*) \ + echo "Not building; $(DISTVERSION) is not a release version."; \ + exit 1;; \ + esac + @$(MAKE) autobuild-dev-html + +.PHONY: check_refcounts +check_refcounts: + $(PYTHON) tools/check_refcounts.py diff --git a/Doc/tools/check_refcounts.py b/Doc/tools/check_refcounts.py new file mode 100644 index 00000000000000..3c05630f90dd32 --- /dev/null +++ b/Doc/tools/check_refcounts.py @@ -0,0 +1,55 @@ +import os +import sys +import argparse + +def read_refcounts_file(file_path): + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + with open(file_path, 'r') as file: + lines = file.readlines() + return [line.strip() for line in lines if line.strip()] + +def read_stable_abi_file(file_path): + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + with open(file_path, 'r') as file: + lines = file.readlines() + return [line.strip() for line in lines if line.strip()] + +def is_sorted(file_lines): + return file_lines == sorted(file_lines) + +def check_refcounts(refcounts_file, stable_abi_file, auto_sort=False): + refcounts = read_refcounts_file(refcounts_file) + stable_abi = read_stable_abi_file(stable_abi_file) + + missing_entries = [entry for entry in stable_abi if entry not in refcounts] + if missing_entries: + print("Missing entries in refcounts.dat:") + for entry in missing_entries: + print(entry) + print(f"Total entries in stable_abi.dat: {len(stable_abi)}") + print(f"Total entries in refcounts.dat: {len(refcounts)}") + print(f"Missing entries: {len(missing_entries)}") + return False + + if not is_sorted(refcounts): + print("refcounts.dat is not sorted alphabetically.") + if auto_sort: + with open(refcounts_file, 'w') as file: + file.writelines('\n'.join(sorted(refcounts)) + '\n') + print("refcounts.dat has been sorted.") + return False + + print("refcounts.dat is up-to-date and sorted.") + return True + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Validate refcounts.dat.") + parser.add_argument("--refcounts", default='../data/refcounts.dat', help="Path to refcounts.dat") + parser.add_argument("--stable_abi", default='../data/stable_abi.dat', help="Path to stable_abi.dat") + parser.add_argument("--auto_sort", action='store_true', help="Auto-sort refcounts.dat if not sorted") + args = parser.parse_args() + + if not check_refcounts(args.refcounts, args.stable_abi, args.auto_sort): + sys.exit(1)