tree-sitter upgrade (#5320)

* Using separate language packages and update lockfile

* Fixing some but not all languages

* Theme updates

* More theme fixes

* Improving the VS Code theme

* Fixes

* Fixing CSS rendering

* Fixing SQL and JavaScript highlight queries

* Update GitHub Actions to run syntax highlighting tests on 3.13, and avoid those tests on 3.8.

* Update CHANGELOG and remove `SyntaxAwareDocument.language_name` property.

* Remove unused Kotlin language syntax highlighting snapshot

* Update CHANGELOG to note that Kotlin syntax highlighting was removed

* Bail out of TextArea.language watcher early if tree-sitter is not installed

* Skip a test on 3.8 as it relates to syntax highlighting (tree-sitter) being available

Author: darrenburns

.github/workflows/pythonpackage.yml

@@ -35,18 +35,18 @@ jobs:
           cache: "poetry"
       - name: Install dependencies
         run: poetry install --no-interaction --extras syntax
-        if: ${{ matrix.python-version != '3.13' }}
-      - name: Install dependencies for 3.13
+        if: ${{ matrix.python-version != '3.8' }}
+      - name: Install dependencies for 3.8
         run: poetry install --no-interaction
-        if: ${{ matrix.python-version == '3.13' }}
-      - name: Test with pytest
+        if: ${{ matrix.python-version == '3.8' }}
+      - name: Test with pytest (Py39+ - with syntax highlighting)
         run: |
           poetry run pytest tests -v --cov=./src/textual --cov-report=xml:./coverage.xml --cov-report term-missing
-        if: ${{ matrix.python-version != '3.13' }}
-      - name: Test with pytest for 3.13
+        if: ${{ matrix.python-version != '3.8' }}
+      - name: Test with pytest (Py38 - without syntax highlighting)
         run: |
           poetry run pytest tests -v --cov=./src/textual --cov-report=xml:./coverage.xml --cov-report term-missing -m 'not syntax'
-        if: ${{ matrix.python-version == '3.13' }}
+        if: ${{ matrix.python-version == '3.8' }}
       - name: Upload snapshot report
         if: always()
         uses: actions/upload-artifact@v3

CHANGELOG.md

@@ -10,10 +10,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## Added
 
 - Added "tab" border style https://github.com/Textualize/textual/pull/5335
+- Added support for XML syntax highlighting https://github.com/Textualize/textual/pull/5320
+- Added `TextArea.update_highlight_query` https://github.com/Textualize/textual/pull/5320
+
 
 ## Fixed
 
 - Fixed Select not scrolling highlight in to view when clicked https://github.com/Textualize/textual/issues/5255
+- Upgraded tree-sitter to 0.23+ (`syntax` extras) https://github.com/Textualize/textual/pull/5320
+- Some syntax highlighting changes due to tree-sitter updates https://github.com/Textualize/textual/pull/5320
+- Breaking change: `Document.query_syntax_tree` signature changed https://github.com/Textualize/textual/pull/5320
+- Breaking change: `TextArea.register_language` signature changed https://github.com/Textualize/textual/pull/5320
+- Breaking change: `SyntaxAwareDocument.language_name` property removed https://github.com/Textualize/textual/pull/5320
+- Breaking change: Kotlin syntax highlighting removed from `TextArea` https://github.com/Textualize/textual/pull/5320
 - Fixed selection list wrapping https://github.com/Textualize/textual/pull/5331
 - Fixed CSS encoding issue on Windows https://github.com/Textualize/textual/pull/5324
 

poetry.lock

@@ -46,12 +46,48 @@ markdown-it-py = { extras = ["plugins", "linkify"], version = ">=2.1.0" }
 rich = ">=13.3.3"
 #rich = {path="../rich", develop=true}
 typing-extensions = "^4.4.0"
-tree-sitter = { version = "^0.20.1", optional = true }
-tree-sitter-languages = { version = "1.10.2", optional = true }
 platformdirs = ">=3.6.0,<5"
 
+# start of [syntax] extras
+# Require tree-sitter >= 0.23.0 and python >= 3.9
+# Windows, MacOS and Linux binary wheels are available for all of the languages below.
+tree-sitter = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-python = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-markdown = { version = ">=0.3.0", optional = true, python = ">=3.9"}
+tree-sitter-json = { version = ">=0.24.0", optional = true, python = ">=3.9" }
+tree-sitter-toml = { version = ">=0.6.0", optional = true, python = ">=3.9" }
+tree-sitter-yaml = { version = ">=0.6.0", optional = true, python = ">=3.9" }
+tree-sitter-html = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-css = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-javascript = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-rust = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-go = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-regex = { version = ">=0.24.0", optional = true, python = ">=3.9" }
+tree-sitter-xml = { version = ">=0.7.0", optional = true, python = ">=3.9" }
+tree-sitter-sql = { version = ">=0.3.0", optional = true, python = ">=3.9" }
+tree-sitter-java = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter-bash = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+# end of [syntax] extras
+
 [tool.poetry.extras]
-syntax = ["tree-sitter", "tree_sitter_languages"]
+syntax = [
+    "tree-sitter",
+    "tree-sitter-python",
+    "tree-sitter-markdown",
+    "tree-sitter-json",
+    "tree-sitter-toml",
+    "tree-sitter-yaml",
+    "tree-sitter-html",
+    "tree-sitter-css",
+    "tree-sitter-javascript",
+    "tree-sitter-rust",
+    "tree-sitter-go",
+    "tree-sitter-regex",
+    "tree-sitter-xml",
+    "tree-sitter-sql",
+    "tree-sitter-java",
+    "tree-sitter-bash",
+]
 
 [tool.poetry.group.dev.dependencies]
 black = "24.4.2"
@@ -72,8 +108,6 @@ pytest-asyncio = "*"
 pytest-cov = "^5.0.0"
 textual-dev = "^1.7.0"
 types-setuptools = "^67.2.0.1"
-types-tree-sitter = "^0.20.1.4"
-types-tree-sitter-languages = "^1.7.0.1"
 isort = "^5.13.2"
 pytest-textual-snapshot = "^1.0.0"
 

pyproject.toml

@@ -148,7 +148,7 @@ def apply_css(self, text_area: TextArea) -> None:
                 self.selection_style = selection_style
             else:
                 selection_background_color = background_color.blend(
-                    app_theme.primary, factor=0.75
+                    app_theme.primary, factor=0.5
                 )
                 self.selection_style = Style.from_color(
                     bgcolor=selection_background_color.rich_color
@@ -204,8 +204,9 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "string": Style(color="#E6DB74"),
         "string.documentation": Style(color="#E6DB74"),
         "comment": Style(color="#75715E"),
+        "heading.marker": Style(color="#90908a"),
         "keyword": Style(color="#F92672"),
-        "operator": Style(color="#F92672"),
+        "operator": Style(color="#f8f8f2"),
         "repeat": Style(color="#F92672"),
         "exception": Style(color="#F92672"),
         "include": Style(color="#F92672"),
@@ -216,7 +217,10 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "number": Style(color="#AE81FF"),
         "float": Style(color="#AE81FF"),
         "class": Style(color="#A6E22E"),
+        "type": Style(color="#A6E22E"),
         "type.class": Style(color="#A6E22E"),
+        "type.builtin": Style(color="#F92672"),
+        "variable.builtin": Style(color="#f8f8f2"),
         "function": Style(color="#A6E22E"),
         "function.call": Style(color="#A6E22E"),
         "method": Style(color="#A6E22E"),
@@ -232,12 +236,18 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "json.label": Style(color="#F92672", bold=True),
         "toml.type": Style(color="#F92672"),
         "toml.datetime": Style(color="#AE81FF"),
+        "css.property": Style(color="#AE81FF"),
         "heading": Style(color="#F92672", bold=True),
         "bold": Style(bold=True),
         "italic": Style(italic=True),
         "strikethrough": Style(strike=True),
-        "link": Style(color="#66D9EF", underline=True),
+        "link.label": Style(color="#F92672"),
+        "link.uri": Style(color="#66D9EF", underline=True),
+        "list.marker": Style(color="#90908a"),
         "inline_code": Style(color="#E6DB74"),
+        "punctuation.bracket": Style(color="#f8f8f2"),
+        "punctuation.delimiter": Style(color="#f8f8f2"),
+        "punctuation.special": Style(color="#f8f8f2"),
     },
 )
 
@@ -254,8 +264,9 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "string": Style(color="#f1fa8c"),
         "string.documentation": Style(color="#f1fa8c"),
         "comment": Style(color="#6272a4"),
+        "heading.marker": Style(color="#6272a4"),
         "keyword": Style(color="#ff79c6"),
-        "operator": Style(color="#ff79c6"),
+        "operator": Style(color="#f8f8f2"),
         "repeat": Style(color="#ff79c6"),
         "exception": Style(color="#ff79c6"),
         "include": Style(color="#ff79c6"),
@@ -266,12 +277,15 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "number": Style(color="#bd93f9"),
         "float": Style(color="#bd93f9"),
         "class": Style(color="#50fa7b"),
+        "type": Style(color="#ff79c6"),
         "type.class": Style(color="#50fa7b"),
+        "type.builtin": Style(color="#bd93f9"),
+        "variable.builtin": Style(color="#f8f8f2"),
         "function": Style(color="#50fa7b"),
         "function.call": Style(color="#50fa7b"),
         "method": Style(color="#50fa7b"),
         "method.call": Style(color="#50fa7b"),
-        "boolean": Style(color="#bd93f9"),
+        "boolean": Style(color="#50fa7b"),
         "constant.builtin": Style(color="#bd93f9"),
         "json.null": Style(color="#bd93f9"),
         "regex.punctuation.bracket": Style(color="#ff79c6"),
@@ -282,12 +296,18 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "json.label": Style(color="#ff79c6", bold=True),
         "toml.type": Style(color="#ff79c6"),
         "toml.datetime": Style(color="#bd93f9"),
+        "css.property": Style(color="#bd93f9"),
         "heading": Style(color="#ff79c6", bold=True),
         "bold": Style(bold=True),
         "italic": Style(italic=True),
         "strikethrough": Style(strike=True),
-        "link": Style(color="#bd93f9", underline=True),
+        "link.label": Style(color="#ff79c6"),
+        "link.uri": Style(color="#bd93f9", underline=True),
+        "list.marker": Style(color="#6272a4"),
         "inline_code": Style(color="#f1fa8c"),
+        "punctuation.bracket": Style(color="#f8f8f2"),
+        "punctuation.delimiter": Style(color="#f8f8f2"),
+        "punctuation.special": Style(color="#f8f8f2"),
     },
 )
 
@@ -304,8 +324,9 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "string": Style(color="#ce9178"),
         "string.documentation": Style(color="#ce9178"),
         "comment": Style(color="#6A9955"),
-        "keyword": Style(color="#569cd6"),
-        "operator": Style(color="#569cd6"),
+        "heading.marker": Style(color="#6E7681"),
+        "keyword": Style(color="#C586C0"),
+        "operator": Style(color="#CCCCCC"),
         "conditional": Style(color="#569cd6"),
         "keyword.function": Style(color="#569cd6"),
         "keyword.return": Style(color="#569cd6"),
@@ -316,25 +337,35 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "number": Style(color="#b5cea8"),
         "float": Style(color="#b5cea8"),
         "class": Style(color="#4EC9B0"),
+        "type": Style(color="#EFCB43"),
         "type.class": Style(color="#4EC9B0"),
-        "function": Style(color="#4EC9B0"),
-        "function.call": Style(color="#4EC9B0"),
+        "type.builtin": Style(color="#9CDCFE"),
+        "function": Style(color="#DCDCAA"),
+        "function.call": Style(color="#DCDCAA"),
         "method": Style(color="#4EC9B0"),
         "method.call": Style(color="#4EC9B0"),
+        "constructor": Style(color="#4EC9B0"),
         "boolean": Style(color="#7DAF9C"),
         "constant.builtin": Style(color="#7DAF9C"),
         "json.null": Style(color="#7DAF9C"),
         "tag": Style(color="#EFCB43"),
         "yaml.field": Style(color="#569cd6", bold=True),
         "json.label": Style(color="#569cd6", bold=True),
         "toml.type": Style(color="#569cd6"),
+        "toml.datetime": Style(color="#C586C0", italic=True),
+        "css.property": Style(color="#569cd6"),
         "heading": Style(color="#569cd6", bold=True),
         "bold": Style(bold=True),
         "italic": Style(italic=True),
         "strikethrough": Style(strike=True),
-        "link": Style(color="#40A6FF", underline=True),
+        "link.uri": Style(color="#40A6FF", underline=True),
+        "link.label": Style(color="#569cd6"),
+        "list.marker": Style(color="#6E7681"),
         "inline_code": Style(color="#ce9178"),
         "info_string": Style(color="#ce9178", bold=True, italic=True),
+        "punctuation.bracket": Style(color="#CCCCCC"),
+        "punctuation.delimiter": Style(color="#CCCCCC"),
+        "punctuation.special": Style(color="#CCCCCC"),
     },
 )
 
@@ -351,6 +382,10 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "string": Style(color="#093069"),
         "string.documentation": Style(color="#093069"),
         "comment": Style(color="#6a737d"),
+        "heading.marker": Style(color="#A4A4A4"),
+        "type": Style(color="#A4A4A4"),
+        "type.class": Style(color="#A4A4A4"),
+        "type.builtin": Style(color="#7DAF9C"),
         "keyword": Style(color="#d73a49"),
         "operator": Style(color="#0450AE"),
         "conditional": Style(color="#CF222E"),
@@ -373,12 +408,18 @@ def builtin_themes(cls) -> list[TextAreaTheme]:
         "yaml.field": Style(color="#6639BB"),
         "json.label": Style(color="#6639BB"),
         "toml.type": Style(color="#6639BB"),
+        "css.property": Style(color="#6639BB"),
         "heading": Style(color="#24292e", bold=True),
         "bold": Style(bold=True),
         "italic": Style(italic=True),
         "strikethrough": Style(strike=True),
-        "link": Style(color="#40A6FF", underline=True),
+        "link.uri": Style(color="#40A6FF", underline=True),
+        "link.label": Style(color="#6639BB"),
+        "list.marker": Style(color="#A4A4A4"),
         "inline_code": Style(color="#093069"),
+        "punctuation.bracket": Style(color="#24292e"),
+        "punctuation.delimiter": Style(color="#24292e"),
+        "punctuation.special": Style(color="#24292e"),
     },
 )
 

src/textual/_text_area_theme.py

@@ -1,10 +1,49 @@
 from __future__ import annotations
 
 try:
-    from tree_sitter import Language, Parser, Tree
-    from tree_sitter.binding import Query
-    from tree_sitter_languages import get_language, get_parser
+    import tree_sitter_bash
+    import tree_sitter_css
+    import tree_sitter_go
+    import tree_sitter_html
+    import tree_sitter_java
+    import tree_sitter_javascript
+    import tree_sitter_json
+    import tree_sitter_markdown
+    import tree_sitter_python
+    import tree_sitter_regex
+    import tree_sitter_rust
+    import tree_sitter_sql
+    import tree_sitter_toml
+    import tree_sitter_xml
+    import tree_sitter_yaml
+    from tree_sitter import Language
+
+    _tree_sitter = True
+
+    _languages = {
+        "python": Language(tree_sitter_python.language()),
+        "json": Language(tree_sitter_json.language()),
+        "markdown": Language(tree_sitter_markdown.language()),
+        "yaml": Language(tree_sitter_yaml.language()),
+        "toml": Language(tree_sitter_toml.language()),
+        "rust": Language(tree_sitter_rust.language()),
+        "html": Language(tree_sitter_html.language()),
+        "css": Language(tree_sitter_css.language()),
+        "xml": Language(tree_sitter_xml.language_xml()),
+        "regex": Language(tree_sitter_regex.language()),
+        "sql": Language(tree_sitter_sql.language()),
+        "javascript": Language(tree_sitter_javascript.language()),
+        "java": Language(tree_sitter_java.language()),
+        "bash": Language(tree_sitter_bash.language()),
+        "go": Language(tree_sitter_go.language()),
+    }
+
+    def get_language(language_name: str) -> Language | None:
+        return _languages.get(language_name)
 
-    TREE_SITTER = True
 except ImportError:
-    TREE_SITTER = False
+    _tree_sitter = False
+    _languages = {}
+
+TREE_SITTER = _tree_sitter
+BUILTIN_LANGUAGES: dict[str, "Language"] = _languages

src/textual/_tree_sitter.py

@@ -731,7 +731,6 @@ def compose(self) -> ComposeResult:
                 "Java",
                 "Javascript",
                 "JSON",
-                "Kotlin",
                 "Markdown",
                 "Python",
                 "Rust",