Support default lexer and more language configurations (#167)

* Support default lexer, literal blocks, highlight directive and highlight_language config (#166)
* Better docs
* Recover from default parse fail and add confval

Author: felix-hilden

docs/src/about.rst

@@ -18,7 +18,7 @@ Caveats
 -------
 - **Only works with HTML documentation**, disabled otherwise. If the extension
   is off, it silently removes directives that would produce output.
-- **Only processes literal blocks, not inline code**. Sphinx has great tools
+- **Only processes blocks, not inline code**. Sphinx has great tools
   for linking definitions inline, and longer code should be in a block anyway.
 - **Doesn't run example code**. Therefore all possible resolvable types are not
   found, and the runtime correctness of code cannot be validated.

docs/src/examples.rst

@@ -21,15 +21,16 @@ be analysed and linked to known reference documentation entries.
 
 Different import styles are supported, along with all Python syntax.
 Star imports might be particularly handy in code examples.
-`Doctest <https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
-#doctest-blocks>`_ and console blocks using :code:`.. code:: pycon` work too.
-Including code via :rst:dir:`literalinclude` requires using one of the following parameters:
-:code:`:language: python`, :code:`:language: python3`,
-:code:`:language: py`, :code:`:language: py3`, :code:`:language: pyi`,
-:code:`:language: sage`, :code:`:language: bazel`, :code:`:language: starlark`
-(i.e., the list of
-`Pygments Lexers <https://pygments.org/docs/lexers/#pygments.lexers.python.PythonLexer>`_).
 
+A number of other types of blocks are also supported
+along with standard Sphinx configuration, like:
+
+- Literal blocks, the ``.. highlight::`` directive
+  and ``highlight_language`` configuration
+- `Doctest blocks <https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#doctest-blocks>`_
+- Console blocks using :code:`.. code:: pycon`
+- Including code via :rst:dir:`literalinclude`, but requires using one of the
+  `Python Lexers <https://pygments.org/docs/lexers/#pygments.lexers.python.PythonLexer>`_
 
 .. code:: pycon
 

docs/src/index.rst

@@ -70,7 +70,7 @@ Caveats
 For a more thorough explanation, see :ref:`about`.
 
 - Only works with HTML documentation
-- Only processes literal blocks, not inline code
+- Only processes blocks, not inline code
 - Doesn't run example code
 - Parsing and type hint resolving is incomplete
 

docs/src/reference.rst

@@ -57,21 +57,27 @@ Configuration
 
 .. confval:: codeautolink_warn_on_missing_inventory
 
-   Type: ``bool``. Issue warning when an object cannot be found in
+   Type: ``bool``. Warn when an object cannot be found in
    the inventory (autodoc or intersphinx). Defaults to :code:`False`.
 
 .. confval:: codeautolink_warn_on_failed_resolve
 
-   Type: ``bool``. Issue warning when failing to resolve the canonical location
+   Type: ``bool``. Warn when failing to resolve the canonical location
    of an object that a code element references. Defaults to :code:`False`.
 
 .. confval:: codeautolink_warn_on_no_backreference
 
-   Type: ``bool``. Issue warning when no backreference could be found
+   Type: ``bool``. Warn when no backreference could be found
    from reference documentation using the :rst:dir:`autolink-examples` table.
    This highlights objects for which no tutorial, example or how-to exists.
    Defaults to :code:`False`.
 
+.. confval:: codeautolink_warn_on_default_parse_fail
+
+   Type: ``bool``. Warn when a code block using the ``default`` lexer
+   cannot be parsed as Python. By default these cases are ignored by the
+   syntax highlighter, so we match the behavior. Defaults to :code:`False`.
+
 Directives
 ----------
 .. rst:directive:: .. autolink-examples:: object
@@ -96,7 +102,7 @@ Directives
 
 .. rst:directive:: .. autolink-concat:: [mode]
 
-   Toggle literal block concatenation.
+   Toggle block concatenation.
    Concatenated code blocks are treated as a continuous source,
    so that imports and statements in previous blocks affect later blocks.
    Concatenation is begun at the directive, not applied retroactively.

docs/src/release_notes.rst

@@ -15,6 +15,10 @@ sphinx-codeautolink adheres to
   a corresponding tutorial or how-to (:issue:`161`)
 - Add more Pygments lexer aliases in code blocks (:issue:`160`)
 - Fix undocumented class attribute leading to a crash (:issue:`165`)
+- Support the ``default`` lexer, literal blocks, ``.. highlight::`` directive
+  and ``highlight_language`` configuration (:issue:`166`)
+- Add :confval:`codeautolink_warn_on_default_parse_fail` to warn about
+  failing to link code blocks without a language parameter (:issue:`166`)
 
 0.16.2 (2025-01-16)
 -------------------

src/sphinx_codeautolink/__init__.py

@@ -49,6 +49,12 @@ def setup(app: Sphinx):
         rebuild="html",
         types=[bool],
     )
+    app.add_config_value(
+        "codeautolink_warn_on_default_parse_fail",
+        default=False,
+        rebuild="html",
+        types=[bool],
+    )
 
     app.add_directive("autolink-concat", directive.Concat)
     app.add_directive("autolink-examples", directive.Examples)

src/sphinx_codeautolink/extension/__init__.py

@@ -75,6 +75,8 @@ def __init__(self) -> None:
         self.warn_missing_inventory = None
         self.warn_failed_resolve = None
         self.warn_no_backreference = None
+        self.warn_default_parse_fail = None
+        self.highlight_lang = None
 
         # Populated once
         self.outdated_docs: set[str] = set()
@@ -105,6 +107,10 @@ def build_inited(self, app) -> None:
         self.warn_missing_inventory = app.config.codeautolink_warn_on_missing_inventory
         self.warn_failed_resolve = app.config.codeautolink_warn_on_failed_resolve
         self.warn_no_backreference = app.config.codeautolink_warn_on_no_backreference
+        self.warn_default_parse_fail = (
+            app.config.codeautolink_warn_on_default_parse_fail
+        )
+        self.highlight_lang = app.config.highlight_language
 
         # Append static resources path so references in setup() are valid
         app.config.html_static_path.append(
@@ -138,6 +144,8 @@ def parse_blocks(self, app, doctree) -> None:
             global_preface=self.global_preface,
             custom_blocks=self.custom_blocks,
             concat_default=self.concat_default,
+            default_highlight_lang=self.highlight_lang,
+            warn_default_parse_fail=self.warn_default_parse_fail,
         )
         doctree.walkabout(visitor)
         self.cache.transforms[visitor.current_document] = visitor.source_transforms

src/sphinx_codeautolink/extension/block.py

@@ -19,6 +19,7 @@
 
 # list from https://pygments.org/docs/lexers/#pygments.lexers.python.PythonLexer
 BUILTIN_BLOCKS = {
+    "default": None,
     "python": None,
     "python3": None,
     "py": None,
@@ -104,7 +105,7 @@ def clean_ipython(source: str) -> tuple[str, str]:
 
 
 class CodeBlockAnalyser(nodes.SparseNodeVisitor):
-    """Transform literal blocks of Python with links to reference documentation."""
+    """Transform blocks of Python with links to reference documentation."""
 
     def __init__(
         self,
@@ -113,6 +114,8 @@ def __init__(
         global_preface: list[str],
         custom_blocks: dict[str, Callable[[str], str]],
         concat_default: bool,
+        default_highlight_lang: str | None,
+        warn_default_parse_fail: bool,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
@@ -130,6 +133,8 @@ def __init__(
         self.concat_section = False
         self.concat_sources = []
         self.skip = None
+        self.highlight_lang = default_highlight_lang
+        self.warn_default_parse_fail = warn_default_parse_fail
 
     def unknown_visit(self, node) -> None:
         """Handle and delete custom directives, ignore others."""
@@ -162,6 +167,10 @@ def unknown_visit(self, node) -> None:
     def unknown_departure(self, node) -> None:
         """Ignore unknown nodes."""
 
+    def visit_highlightlang(self, node) -> None:
+        """Set expected highlight language."""
+        self.highlight_lang = node["lang"]
+
     def visit_title(self, node) -> None:
         """Track section names and break concatenation and skipping."""
         self.title_stack.append(node.astext())
@@ -185,7 +194,7 @@ def visit_doctest_block(self, node):
 
     def visit_literal_block(self, node: nodes.literal_block):
         """Visit a generic literal block."""
-        return self.parse_source(node, node.get("language", None))
+        return self.parse_source(node, node.get("language", self.highlight_lang))
 
     def parse_source(
         self, node: nodes.literal_block | nodes.doctest_block, language: str | None
@@ -231,6 +240,9 @@ def parse_source(
         try:
             names = parse_names(modified_source, node)
         except SyntaxError as e:
+            if language == "default" and not self.warn_default_parse_fail:
+                return
+
             show_source = self._format_source_for_error(
                 self.global_preface, self.concat_sources, prefaces, source
             )

src/sphinx_codeautolink/extension/directive.py

@@ -62,7 +62,7 @@ def copy(self):
 
 
 class Concat(Directive):
-    """Toggle and cut literal block concatenation in a document."""
+    """Toggle and cut block concatenation in a document."""
 
     has_content = False
     required_arguments = 0