feat(tree-sitter)!: Remove dependency on nvim-treesitter (#707)

Author: kristijanhusak

.gitignore

@@ -27,8 +27,6 @@ luac.out
 
 # Shared objects (inc. Windows DLLs)
 *.dll
-*.so
-*.so.*
 *.dylib
 
 # Executables

DOCS.md

@@ -1676,12 +1676,9 @@ More optimized version would be to create a lua file that has only necessary plu
 -- ~/.config/nvim/lua/partials/org_cron.lua
 
 -- If you are using lazy.vim do this:
-local treesitter = vim.fn.stdpath('data') .. '/lazy/nvim-treesitter'
 local orgmode = vim.fn.stdpath('data') .. '/lazy/orgmode'
 vim.opt.runtimepath:append(orgmode)
-vim.opt.runtimepath:append(treesitter)
 -- If you are using Packer or any other package manager that uses built-in package manager, do this:
-vim.cmd('packadd nvim-treesitter')
 vim.cmd('packadd orgmode')
 
 -- Run the orgmode cron

README.md

@@ -20,7 +20,6 @@
 ### Requirements
 
 * Neovim 0.9.2 or later
-* [nvim-treesitter](https://github.com/nvim-treesitter/nvim-treesitter)
 
 ### Installation
 
@@ -33,39 +32,31 @@ Use your favourite package manager:
 ```lua
 {
   'nvim-orgmode/orgmode',
-  dependencies = {
-    { 'nvim-treesitter/nvim-treesitter', lazy = true },
-  },
   event = 'VeryLazy',
   config = function()
-    -- Load treesitter grammar for org
-    require('orgmode').setup_ts_grammar()
-
-    -- Setup treesitter
-    require('nvim-treesitter.configs').setup({
-      highlight = {
-        enable = true,
-      },
-      ensure_installed = { 'org' },
-    })
-
     -- Setup orgmode
     require('orgmode').setup({
       org_agenda_files = '~/orgfiles/**/*',
       org_default_notes_file = '~/orgfiles/refile.org',
     })
+
+    -- NOTE: If you are using nvim-treesitter with `ensure_installed = "all"` option
+    -- add `org` to ignore_install
+    -- require('nvim-treesitter.configs').setup({
+    --   ensure_installed = 'all',
+    --   ignore_install = { 'org' },
+    -- })
   end,
 }
 ```
 
 </details>
 
-<details open>
+<details>
   <summary><b><a href="https://github.com/wbthomason/packer.nvim">packer.nvim</a></b></summary>
   </br>
 
 ```lua
-use {'nvim-treesitter/nvim-treesitter'}
 use {'nvim-orgmode/orgmode', config = function()
   require('orgmode').setup{}
 end
@@ -79,7 +70,6 @@ end
   </br>
 
 ```vim
-Plug 'nvim-treesitter/nvim-treesitter'
 Plug 'nvim-orgmode/orgmode'
 ```
 
@@ -90,7 +80,6 @@ Plug 'nvim-orgmode/orgmode'
   </br>
 
 ```vim
-call dein#add('nvim-treesitter/nvim-treesitter')
 call dein#add('nvim-orgmode/orgmode')
 ```
 
@@ -104,29 +93,27 @@ since instructions above covers full setup
 ```lua
 -- init.lua
 
--- Load custom treesitter grammar for org filetype
-require('orgmode').setup_ts_grammar()
-
--- Treesitter configuration
-require('nvim-treesitter.configs').setup {
-  highlight = {
-    enable = true,
-  },
-  ensure_installed = {'org'}, -- Or run :TSUpdate org
-}
-
 require('orgmode').setup({
   org_agenda_files = {'~/Dropbox/org/*', '~/my-orgs/**/*'},
   org_default_notes_file = '~/Dropbox/org/refile.org',
 })
-```
+
+-- NOTE: If you are using nvim-treesitter with `ensure_installed = "all"` option
+-- add `org` to ignore_install
+-- require('nvim-treesitter.configs').setup({
+--   ensure_installed = 'all',
+--   ignore_install = { 'org' },
+-- })
 
 Or if you are using `init.vim`, wrap the above snippet like so:
 ```vim
 " init.vim
 lua << EOF
 
-require('orgmode').setup_ts_grammar() ...
+require('orgmode').setup({
+  org_agenda_files = {'~/Dropbox/org/*', '~/my-orgs/**/*'},
+  org_default_notes_file = '~/Dropbox/org/refile.org',
+})
 
 EOF
 ```
@@ -191,29 +178,17 @@ or a hands-on [tutorial](https://github.com/nvim-orgmode/orgmode/wiki/Getting-St
 
 ## Treesitter Info
 The built-in treesitter parser is used for parsing the org files.
-Highlights are experimental and partially supported.
-
-### Advantages of treesitter over built in parsing/syntax:
-* More reliable, since parsing is done with a proper parsing tool
-* Better highlighting (Experimental, still requires improvements)
-* Future features will be easier to implement because the grammar already parses some things that were not parsed before (tables, latex, etc.)
-* Allows for easier hacking (custom motions that can work with TS nodes, etc.)
 
 ### Known highlighting issues and limitations
 * LaTex is still highlighted through syntax file
 
-### Improvements over Vim's syntax highlighting
-* Better highlighting of certain parts (tags, deadline/schedule/closed dates)
-* [Treesitter highlight injections](https://github.com/nvim-treesitter/nvim-treesitter/blob/4f2265632becabcd2c5b1791fa31ef278f1e496c/CONTRIBUTING.md#injections) through `#BEGIN_SRC filetype` blocks
-* Headline markup highlighting (https://github.com/nvim-orgmode/orgmode/issues/67)
-
 ## Troubleshoot
 ### Indentation is not working
 Make sure you are not overriding indentexpr in Org buffers with [nvim-treesitter indentation](https://github.com/nvim-treesitter/nvim-treesitter#indentation)
 
 ### I get `treesitter/query.lua` errors when opening agenda/capture prompt or org files
-Make sure you are using latest changes from [tree-sitter-org](https://github.com/milisims/tree-sitter-org) grammar.<br />
-by running `:TSUpdate org` and restarting the editor.
+Tree-sitter parser might not be installed.
+Try running `:lua require('orgmode.config'):reinstall_grammar()` to reinstall it.
 
 ### Dates are not in English
 Dates are generated with Lua native date support, and it reads your current locale when creating them.<br />

ftplugin/org.lua

@@ -9,6 +9,8 @@ local utils = require('orgmode.utils')
 
 vim.b.org_bufnr = vim.api.nvim_get_current_buf()
 
+vim.treesitter.start()
+
 config:setup_mappings('org', vim.b.org_bufnr)
 config:setup_mappings('text_objects', vim.b.org_bufnr)
 config:setup_foldlevel()
@@ -21,7 +23,7 @@ require('orgmode.org.indent').setup_virtual_indent()
 vim.bo.modeline = false
 vim.opt_local.fillchars:append('fold: ')
 vim.opt_local.foldmethod = 'expr'
-vim.opt_local.foldexpr = 'nvim_treesitter#foldexpr()'
+vim.opt_local.foldexpr = 'v:lua.require("orgmode.org.fold").foldexpr()'
 if utils.has_version_10() then
   vim.opt_local.foldtext = ''
 else

lua/orgmode/config/init.lua

@@ -28,6 +28,19 @@ function Config:__index(key)
   return rawget(getmetatable(self), key)
 end
 
+function Config:install_grammar()
+  local ok = pcall(vim.treesitter.language.add, 'org')
+  if ok then
+    return
+  end
+  require('orgmode.utils.treesitter.install').run()
+end
+
+---@param url? string
+function Config:reinstall_grammar(url)
+  return require('orgmode.utils.treesitter.install').run(url)
+end
+
 ---@param opts table
 ---@return OrgConfig
 function Config:extend(opts)

lua/orgmode/init.lua

@@ -1,6 +1,4 @@
 _G.orgmode = _G.orgmode or {}
-local ts_revision = 'f8c6b1e72f82f17e41004e04e15f62a83ecc27b0'
-local setup_ts_grammar_used = false
 ---@type Org | nil
 local instance = nil
 
@@ -98,52 +96,18 @@ function Org:setup_autocmds()
   })
 end
 
---- @param revision string?
-function Org.setup_ts_grammar(revision)
-  setup_ts_grammar_used = true
-  local parser_config = require('nvim-treesitter.parsers').get_parser_configs()
-  ---@diagnostic disable-next-line: inject-field
-  parser_config.org = {
-    install_info = {
-      url = 'https://github.com/nvim-orgmode/tree-sitter-org',
-      revision = revision or ts_revision,
-      files = { 'src/parser.c', 'src/scanner.c' },
-    },
-    filetype = 'org',
-  }
-end
-
----@private
-function Org._check_ts_grammar()
-  vim.defer_fn(function()
-    if setup_ts_grammar_used then
-      return
-    end
-    local parser_config = require('nvim-treesitter.parsers').get_parser_configs()
-    if parser_config and parser_config.org and parser_config.org.install_info.revision then
-      if parser_config.org.install_info.revision ~= ts_revision then
-        require('orgmode.utils').echo_error({
-          'You are using outdated version of tree-sitter grammar for Orgmode.',
-          'To use latest version, replace current grammar installation with "require(\'orgmode\').setup_ts_grammar()" and run :TSUpdate org.',
-          'More info in setup section of readme: https://github.com/nvim-orgmode/orgmode#setup',
-        })
-      end
-    else
-      require('orgmode.utils').echo_error({
-        'Cannot detect parser revision.',
-        "Please check your org grammar's install info.",
-        'Maybe you forgot to call "require(\'orgmode\').setup_ts_grammar()" before setup.',
-      })
-    end
-  end, 200)
+function Org.setup_ts_grammar()
+  require('orgmode.utils').echo_info(
+    'calling require("orgmode").setup_ts_grammar() is no longer necessary. Dependency on nvim-treesitter was removed'
+  )
 end
 
 ---@param opts? OrgDefaultConfig
 ---@return Org
 function Org.setup(opts)
   opts = opts or {}
-  Org._check_ts_grammar()
   local config = require('orgmode.config'):extend(opts)
+  config:install_grammar()
   instance = Org:new()
   vim.defer_fn(function()
     if config.notifications.enabled and #vim.api.nvim_list_uis() > 0 then

lua/orgmode/org/fold.lua

@@ -0,0 +1,108 @@
+-- Taken from https://github.com/nvim-treesitter/nvim-treesitter
+
+local api = vim.api
+local ts_utils = require('orgmode.utils.treesitter')
+
+---@type vim.treesitter.Query
+local query = nil
+
+local M = {}
+
+-- This is cached on buf tick to avoid computing that multiple times
+-- Especially not for every line in the file when `zx` is hit
+local folds_levels = ts_utils.memoize_by_buf_tick(function(bufnr)
+  local max_fold_level = api.nvim_get_option_value('foldnestmax', { win = 0 })
+  local trim_level = function(level)
+    if level > max_fold_level then
+      return max_fold_level
+    end
+    return level
+  end
+
+  query = query or vim.treesitter.query.get('org', 'folds')
+  local trees = vim.treesitter.get_parser(bufnr):parse()
+  local root = trees[1]:root()
+
+  local matches = {}
+  for _, node in query:iter_captures(root, bufnr) do
+    table.insert(matches, node)
+  end
+
+  ---@type table<number, number>
+  local start_counts = {}
+  ---@type table<number, number>
+  local stop_counts = {}
+
+  local prev_start = -1
+  local prev_stop = -1
+
+  local min_fold_lines = api.nvim_get_option_value('foldminlines', { win = 0 })
+
+  for _, match in ipairs(matches) do
+    local start, _, stop, stop_col = match:range() ---@type integer, integer, integer, integer
+
+    if stop_col == 0 then
+      stop = stop - 1
+    end
+
+    local fold_length = stop - start + 1
+    local should_fold = fold_length > min_fold_lines
+
+    -- Fold only multiline nodes that are not exactly the same as previously met folds
+    -- Checking against just the previously found fold is sufficient if nodes
+    -- are returned in preorder or postorder when traversing tree
+    if should_fold and not (start == prev_start and stop == prev_stop) then
+      start_counts[start] = (start_counts[start] or 0) + 1
+      stop_counts[stop] = (stop_counts[stop] or 0) + 1
+      prev_start = start
+      prev_stop = stop
+    end
+  end
+
+  ---@type string[]
+  local levels = {}
+  local current_level = 0
+
+  -- We now have the list of fold opening and closing, fill the gaps and mark where fold start
+  for lnum = 0, api.nvim_buf_line_count(bufnr) do
+    local prefix = ''
+
+    local last_trimmed_level = trim_level(current_level)
+    current_level = current_level + (start_counts[lnum] or 0)
+    local trimmed_level = trim_level(current_level)
+    current_level = current_level - (stop_counts[lnum] or 0)
+    local next_trimmed_level = trim_level(current_level)
+
+    -- Determine if it's the start/end of a fold
+    -- NB: vim's fold-expr interface does not have a mechanism to indicate that
+    -- two (or more) folds start at this line, so it cannot distinguish between
+    --  ( \n ( \n )) \n (( \n ) \n )
+    -- versus
+    --  ( \n ( \n ) \n ( \n ) \n )
+    -- If it did have such a mechanism, (trimmed_level - last_trimmed_level)
+    -- would be the correct number of starts to pass on.
+    if trimmed_level - last_trimmed_level > 0 then
+      prefix = '>'
+    elseif trimmed_level - next_trimmed_level > 0 then
+      -- Ending marks tend to confuse vim more than it helps, particularly when
+      -- the fold level changes by at least 2; we can uncomment this if
+      -- vim's behavior gets fixed.
+      -- prefix = "<"
+      prefix = ''
+    end
+
+    levels[lnum + 1] = prefix .. tostring(trimmed_level)
+  end
+
+  return levels
+end)
+
+---@return string
+function M.foldexpr()
+  local buf = api.nvim_get_current_buf()
+  local levels = folds_levels(buf) or {}
+
+  return levels[vim.v.lnum] or '0'
+end
+
+return M