diff --git a/.github/workflows/build-package-docs.yaml b/.github/workflows/build-package-docs.yaml index 04a581664a3..cb9b4cf478f 100644 --- a/.github/workflows/build-package-docs.yaml +++ b/.github/workflows/build-package-docs.yaml @@ -1,6 +1,6 @@ --- name: Build and deploy docs -on: +"on": schedule: # Run at 05:17 on Tuesday and Thursday - cron: '17 5 * * 2,4' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c52912470b2..3d11f6a7e02 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,8 @@ name: Ansible Docsite CI -on: +"on": schedule: - # Daily + # Daily - cron: "23 7 * * *" push: branches-ignore: diff --git a/.github/workflows/reusable-nox.yml b/.github/workflows/reusable-nox.yml index 146e334d43d..45cfffb3825 100644 --- a/.github/workflows/reusable-nox.yml +++ b/.github/workflows/reusable-nox.yml @@ -25,6 +25,8 @@ jobs: python-versions: "3.11" - session: "checkers(rstcheck)" python-versions: "3.11" + - session: "checkers(rst-yamllint)" + python-versions: "3.11" - session: "checkers(docs-build)" python-versions: "3.11" - session: "actionlint" diff --git a/.mypy.ini b/.mypy.ini index e176756d259..59913e808b4 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -3,3 +3,6 @@ check_untyped_defs = True [mypy-ansible.*] ignore_missing_imports = True + +[mypy-yamllint.*] +ignore_missing_imports = True diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000000..29dc60a1e82 --- /dev/null +++ b/.yamllint @@ -0,0 +1,68 @@ +--- +extends: default + +ignore: + - /lib/ansible/ + +rules: + line-length: disable + # max: 160 + # level: warning + document-start: disable + document-end: disable + truthy: + level: warning + allowed-values: + - 'true' + - 'false' + - 'True' + - 'False' + - 'TRUE' + - 'FALSE' + - 'yes' + - 'no' + - 'Yes' + - 'No' + - 'YES' + - 'NO' + indentation: disable + # level: warning + # spaces: 2 + # indent-sequences: consistent + key-duplicates: disable + # level: warning + # forbid-duplicated-merge-keys: true + trailing-spaces: enable + hyphens: disable + # max-spaces-after: 1 + # level: warning + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + level: warning + commas: disable + # max-spaces-before: 0 + # min-spaces-after: 1 + # max-spaces-after: 1 + # level: warning + colons: disable + # max-spaces-before: 0 + # max-spaces-after: 1 + # level: warning + brackets: disable + # min-spaces-inside: 0 + # max-spaces-inside: 0 + # level: warning + braces: + min-spaces-inside: 0 + max-spaces-inside: 1 + level: warning + octal-values: disable + # forbid-implicit-octal: true + # forbid-explicit-octal: true + # level: warning + comments: disable + # min-spaces-from-content: 1 + # level: warning + comments-indentation: disable diff --git a/docs/docsite/rst/getting_started_ee/yaml/hosts.yml b/docs/docsite/rst/getting_started_ee/yaml/hosts.yml index 3fb711e9c71..8e131a86329 100644 --- a/docs/docsite/rst/getting_started_ee/yaml/hosts.yml +++ b/docs/docsite/rst/getting_started_ee/yaml/hosts.yml @@ -1,3 +1,3 @@ all: hosts: - 192.168.0.2 # Replace with the IP of your target host \ No newline at end of file + 192.168.0.2 # Replace with the IP of your target host diff --git a/docs/docsite/rst/playbook_guide/playbooks_error_handling.rst b/docs/docsite/rst/playbook_guide/playbooks_error_handling.rst index 8d99da6e3e4..45953dbbbe8 100644 --- a/docs/docsite/rst/playbook_guide/playbooks_error_handling.rst +++ b/docs/docsite/rst/playbook_guide/playbooks_error_handling.rst @@ -179,16 +179,16 @@ You can reference simple variables in conditionals to avoid repeating certain te - name: Example playbook hosts: myHosts vars: - log_path: /home/ansible/logfolder/ - log_file: log.log + log_path: /home/ansible/logfolder/ + log_file: log.log - tasks: + tasks: - name: Create empty log file - ansible.builtin.shell: mkdir {{ log_path }} || touch {{log_path }}{{ log_file }} - register: tmp - changed_when: - - tmp.rc == 0 - - 'tmp.stderr != "mkdir: cannot create directory ‘" ~ log_path ~ "’: File exists"' + ansible.builtin.shell: mkdir {{ log_path }} || touch {{log_path }}{{ log_file }} + register: tmp + changed_when: + - tmp.rc == 0 + - 'tmp.stderr != "mkdir: cannot create directory ‘" ~ log_path ~ "’: File exists"' .. note:: Notice the missing double curly braces ``{{ }}`` around the ``log_path`` variable in the ``changed_when`` statement. diff --git a/examples/DOCUMENTATION.yml b/examples/DOCUMENTATION.yml index b6ccb866298..a0700ca9975 100644 --- a/examples/DOCUMENTATION.yml +++ b/examples/DOCUMENTATION.yml @@ -22,15 +22,15 @@ options: default: a string or the word null choices: - enable - - disable + - disable aliases: - repo_name version_added: "1.X" requirements: - list of required things - like the factor package - - zypper >= 1.0 -seealso: + - zypper >= 1.0 +seealso: - specify references to other modules, useful guides, and so on notes: - other things consumers of your module should know diff --git a/noxfile.py b/noxfile.py index 5450142dae4..000f39d2e70 100644 --- a/noxfile.py +++ b/noxfile.py @@ -15,6 +15,7 @@ "hacking/tagger/tag.py", "noxfile.py", *iglob("docs/bin/*.py"), + *iglob("tests/checkers/rst-yamllint*.py"), # TODO: also lint others ) PINNED = os.environ.get("PINNED", "true").lower() in {"1", "true"} nox.options.sessions = ("clone-core", "lint", "checkers", "make") diff --git a/tests/checkers/rst-yamllint.json b/tests/checkers/rst-yamllint.json new file mode 100644 index 00000000000..0f9dcfad1c4 --- /dev/null +++ b/tests/checkers/rst-yamllint.json @@ -0,0 +1,9 @@ +{ + "extensions": [ + ".rst", + ".txt" + ], + "ignore_regexs": [ + "^docs/docsite/rst/porting_guides/porting_guide_[0-9]+\\.rst$" + ] +} diff --git a/tests/checkers/rst-yamllint.py b/tests/checkers/rst-yamllint.py new file mode 100644 index 00000000000..45eef975367 --- /dev/null +++ b/tests/checkers/rst-yamllint.py @@ -0,0 +1,330 @@ +"""Sanity test using rstcheck and sphinx.""" + +from __future__ import annotations + +import io +import pathlib +import sys +import traceback + +from docutils import nodes +from docutils.core import Publisher +from docutils.io import StringInput +from docutils.parsers.rst import Directive +from docutils.parsers.rst.directives import register_directive +from docutils.parsers.rst.directives import unchanged as directive_param_unchanged +from docutils.utils import Reporter, SystemMessage +from yamllint import linter +from yamllint.config import YamlLintConfig +from yamllint.linter import PROBLEM_LEVELS + +REPORT_LEVELS: set[PROBLEM_LEVELS] = { + "warning", + "error", +} + +ALLOWED_LANGUAGES = { + "ansible-output", + "bash", + "console", + "csharp", + "diff", + "ini", + "jinja", + "json", + "md", + "none", + "powershell", + "python", + "rst", + "sh", + "shell", + "shell-session", + "text", +} + + +class IgnoreDirective(Directive): + has_content = True + + def run(self) -> list: + return [] + + +class CodeBlockDirective(Directive): + has_content = True + optional_arguments = 1 + + # These are all options Sphinx allows for code blocks. + # We need to have them here so that docutils successfully parses this extension. + option_spec = { + "caption": directive_param_unchanged, + "class": directive_param_unchanged, + "dedent": directive_param_unchanged, + "emphasize-lines": directive_param_unchanged, + "name": directive_param_unchanged, + "force": directive_param_unchanged, + "linenos": directive_param_unchanged, + "lineno-start": directive_param_unchanged, + } + + def run(self) -> list[nodes.literal_block]: + code = "\n".join(self.content) + literal = nodes.literal_block(code, code) + literal["classes"].append("code-block") + literal["ansible-code-language"] = self.arguments[0] if self.arguments else None + literal["ansible-code-block"] = True + literal["ansible-code-lineno"] = self.lineno + return [literal] + + +class YamlLintVisitor(nodes.SparseNodeVisitor): + def __init__( + self, + document: nodes.document, + path: str, + results: list[dict], + content: str, + yamllint_config: YamlLintConfig, + ): + super().__init__(document) + self.__path = path + self.__results = results + self.__content_lines = content.splitlines() + self.__yamllint_config = yamllint_config + + def visit_system_message(self, node: nodes.system_message) -> None: + raise nodes.SkipNode + + def visit_error(self, node: nodes.error) -> None: + raise nodes.SkipNode + + def visit_literal_block(self, node: nodes.literal_block) -> None: + if "ansible-code-block" not in node.attributes: + if node.attributes["classes"]: + self.__results.append( + { + "path": self.__path, + "line": node.line or "unknown", + "col": 0, + "message": ( + "Warning: found unknown literal block! Check for double colons '::'." + " If that is not the cause, please report this warning." + " It might indicate a bug in the checker or an unsupported Sphinx directive." + f" Node: {node!r}; attributes: {node.attributes}; content: {node.rawsource!r}" + ), + } + ) + raise nodes.SkipNode + + language = node.attributes["ansible-code-language"] + lineno = node.attributes["ansible-code-lineno"] + + # Ok, we have to find both the row and the column offset for the actual code content + row_offset = lineno + found_empty_line = False + found_content_lines = False + content_lines = node.rawsource.count("\n") + 1 + min_indent = None + for offset, line in enumerate(self.__content_lines[lineno:]): + stripped_line = line.strip() + if not stripped_line: + if not found_empty_line: + row_offset = lineno + offset + 1 + found_empty_line = True + elif not found_content_lines: + found_content_lines = True + row_offset = lineno + offset + + if found_content_lines and content_lines > 0: + if stripped_line: + indent = len(line) - len(line.lstrip()) + if min_indent is None or min_indent > indent: + min_indent = indent + content_lines -= 1 + elif not content_lines: + break + + min_source_indent = None + for line in node.rawsource.split("\n"): + stripped_line = line.lstrip() + if stripped_line: + indent = len(line) - len(line.lstrip()) + if min_source_indent is None or min_source_indent > indent: + min_source_indent = indent + + col_offset = max(0, (min_indent or 0) - (min_source_indent or 0)) + + # Now that we have the offsets, we can actually do some processing... + if language not in {"YAML", "yaml", "yaml+jinja", "YAML+Jinja"}: + if language is None: + allowed_languages = ", ".join(sorted(ALLOWED_LANGUAGES)) + self.__results.append( + { + "path": self.__path, + "line": row_offset + 1, + "col": col_offset + 1, + "message": ( + "Literal block without language!" + f" Allowed languages are: {allowed_languages}." + ), + } + ) + return + if language not in ALLOWED_LANGUAGES: + allowed_languages = ", ".join(sorted(ALLOWED_LANGUAGES)) + self.__results.append( + { + "path": self.__path, + "line": row_offset + 1, + "col": col_offset + 1, + "message": ( + f"Warning: literal block with disallowed language: {language}." + " If the language should be allowed, the checker needs to be updated." + f" Currently allowed languages are: {allowed_languages}." + ), + } + ) + raise nodes.SkipNode + + # So we have YAML. Let's lint it! + try: + problems = linter.run( + io.StringIO(node.rawsource.rstrip() + "\n"), + self.__yamllint_config, + self.__path, + ) + for problem in problems: + if problem.level not in REPORT_LEVELS: + continue + msg = f"{problem.level}: {problem.desc}" + if problem.rule: + msg += f" ({problem.rule})" + self.__results.append( + { + "path": self.__path, + "line": row_offset + problem.line, + "col": col_offset + problem.column, + "message": msg, + } + ) + except Exception as exc: + error = str(exc).replace("\n", " / ") + self.__results.append( + { + "path": self.__path, + "line": row_offset + 1, + "col": col_offset + 1, + "message": ( + f"Internal error while linting YAML: exception {type(exc)}:" + f" {error}; traceback: {traceback.format_exc()!r}" + ), + } + ) + + raise nodes.SkipNode + + +def main(): + paths = sys.argv[1:] or sys.stdin.read().splitlines() + results = [] + + for directive in ( + "code", + "code-block", + "sourcecode", + ): + register_directive(directive, CodeBlockDirective) + + # The following docutils directives should better be ignored: + for directive in ("parsed-literal",): + register_directive(directive, IgnoreDirective) + + # TODO: should we handle the 'literalinclude' directive? maybe check file directly if right extension? + # (https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-literalinclude) + + repo_root = pathlib.Path(__file__).resolve().parent.parent.parent + docs_root = repo_root / "docs" / "docsite" / "rst" + + with open(repo_root / ".yamllint", encoding="utf-8") as f: + yamllint_config = YamlLintConfig(f.read()) + + for path in paths: + with open(path, "rt", encoding="utf-8") as f: + content = f.read() + + # We create a Publisher only to have a mechanism which gives us the settings object. + # Doing this more explicit is a bad idea since the classes used are deprecated and will + # eventually get replaced. Publisher.get_settings() looks like a stable enough API that + # we can 'just use'. + publisher = Publisher(source_class=StringInput) + publisher.set_components("standalone", "restructuredtext", "pseudoxml") + override = { + "root_prefix": docs_root, + "input_encoding": "utf-8", + "file_insertion_enabled": False, + "raw_enabled": False, + "_disable_config": True, + "report_level": Reporter.ERROR_LEVEL, + "warning_stream": io.StringIO(), + } + publisher.process_programmatic_settings(None, override, None) + publisher.set_source(content, path) + + # Parse the document + try: + doc = publisher.reader.read( + publisher.source, publisher.parser, publisher.settings + ) + except SystemMessage as exc: + error = str(exc).replace("\n", " / ") + results.append( + { + "path": path, + "line": 0, + "col": 0, + "message": f"Cannot parse document: {error}", + } + ) + continue + except Exception as exc: + error = str(exc).replace("\n", " / ") + results.append( + { + "path": path, + "line": 0, + "col": 0, + "message": f"Cannot parse document, unexpected error {type(exc)}: {error}; traceback: {traceback.format_exc()!r}", + } + ) + continue + + # Process the document + try: + visitor = YamlLintVisitor(doc, path, results, content, yamllint_config) + doc.walk(visitor) + except Exception as exc: + error = str(exc).replace("\n", " / ") + results.append( + { + "path": path, + "line": 0, + "col": 0, + "message": f"Cannot process document: {type(exc)} {error}; traceback: {traceback.format_exc()!r}", + } + ) + + for result in sorted( + results, + key=lambda result: ( + result["path"], + result["line"], + result["col"], + result["message"], + ), + ): + print("{path}:{line}:{col}: {message}".format(**result)) + + +if __name__ == "__main__": + main() diff --git a/tests/requirements.in b/tests/requirements.in index 19502f6eae1..ecbd8d739d2 100644 --- a/tests/requirements.in +++ b/tests/requirements.in @@ -11,6 +11,7 @@ sphinx-notfound-page # extension used for the custom 404 page (cowsay) sphinx-ansible-theme # extension used for the custom docs theme sphinx-rtd-theme rstcheck +yamllint sphinx-copybutton jinja2 # used by hacking/build_library/build_ansible/command_plugins/generate_man.py and dump_keywords.py pyyaml # used by ansible-core diff --git a/tests/requirements.txt b/tests/requirements.txt index f13ce994fce..0269a49f410 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,7 +6,7 @@ aiofiles==24.1.0 # antsibull-fileutils aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.11.13 +aiohttp==3.11.14 # via # antsibull-core # antsibull-docs @@ -34,7 +34,7 @@ antsibull-docs-parser==1.2.0 # via antsibull-docs antsibull-docutils==1.1.0 # via antsibull-changelog -antsibull-fileutils==1.1.0 +antsibull-fileutils==1.2.0 # via # antsibull-changelog # antsibull-core @@ -90,7 +90,7 @@ jinja2==3.1.6 # sphinx markupsafe==3.0.2 # via jinja2 -multidict==6.1.0 +multidict==6.2.0 # via # aiohttp # yarl @@ -102,6 +102,8 @@ packaging==24.2 # antsibull-docs # build # sphinx +pathspec==0.12.1 + # via yamllint perky==0.9.3 # via antsibull-core propcache==0.3.0 @@ -129,6 +131,7 @@ pyyaml==6.0.2 # -r tests/requirements.in # antsibull-docs # antsibull-fileutils + # yamllint requests==2.32.3 # via sphinx resolvelib==1.0.1 @@ -146,7 +149,7 @@ semantic-version==2.10.0 # antsibull-changelog # antsibull-core # antsibull-docs -setuptools==76.0.0 +setuptools==77.0.3 # via sphinx-intl six==1.17.0 # via twiggy @@ -203,5 +206,7 @@ typing-extensions==4.12.2 # rstcheck urllib3==2.3.0 # via requests +yamllint==1.36.2 + # via -r tests/requirements.in yarl==1.18.3 # via aiohttp diff --git a/tests/typing.in b/tests/typing.in index 9701e2c5780..121d2e35d57 100644 --- a/tests/typing.in +++ b/tests/typing.in @@ -2,3 +2,4 @@ -r tag.in mypy nox +types-docutils diff --git a/tests/typing.txt b/tests/typing.txt index 8dbc9eb3d1e..a8bd08e8373 100644 --- a/tests/typing.txt +++ b/tests/typing.txt @@ -2,7 +2,7 @@ # uv pip compile --universal --output-file tests/typing.txt tests/typing.in argcomplete==3.5.3 # via nox -certifi==2024.12.14 +certifi==2025.1.31 # via requests cffi==1.17.1 # via @@ -24,11 +24,11 @@ colorlog==6.9.0 # via nox cryptography==44.0.0 # via pyjwt -deprecated==1.2.15 +deprecated==1.2.18 # via pygithub distlib==0.3.9 # via virtualenv -filelock==3.16.1 +filelock==3.17.0 # via virtualenv gitdb==4.0.12 # via gitpython @@ -60,7 +60,7 @@ pycparser==2.22 # via cffi pygithub==2.5.0 # via -r tests/../hacking/pr_labeler/requirements.txt -pygments==2.18.0 +pygments==2.19.1 # via rich pyjwt==2.10.1 # via pygithub @@ -78,6 +78,8 @@ typer==0.15.1 # via -r tests/tag.in typer-slim==0.15.1 # via -r tests/../hacking/pr_labeler/requirements.txt +types-docutils==0.21.0.20241128 + # via -r tests/typing.in typing-extensions==4.12.2 # via # codeowners @@ -89,7 +91,7 @@ urllib3==2.3.0 # via # pygithub # requests -virtualenv==20.28.1 +virtualenv==20.29.1 # via nox -wrapt==1.17.0 +wrapt==1.17.2 # via deprecated