Line/Column Numbers of tokens in the AST

[DeanHnter]

It would be good if we could have line numbers/column numbers in the AST produced by the --print-ast command. This could open up tool creation from other languages.

[kmafeni04]

If you're using lua, a lot of that information can be gotten directly

[DeanHnter]

ah awesome, is there any examples of getting this kind of information that you know of or do you know where i should be debugging to try extending this functionality so i can expose it to other languages?

[kmafeni04]

Well there's the lsp I’m writing as an example, the actual nelua source code is also probably the best example

If you want a code snippet of how to get the line and character position, I could provide that too

[DeanHnter]

Your LSP utils seem to be reasonably clear (i dont do much LUA) but a snippet of course would be appreciated since it would isolate everything.

[kmafeni04]

This will print all the nodes with a name with the starting line and starting character position
I've copied a lot of things from my project so all the code may not be as necessary as it seems but I wanted to create something quickly to show

To run it, you'd need nelua installed and the command would look like: nelua --script test.lua as the required modules are only available inside of nelua

local except = require("nelua.utils.except")
local analyzer = require("nelua.analyzer")
local AnalyzerContext = require("nelua.analyzercontext")
local aster = require("nelua.aster")
local generator = require("nelua.cgenerator")

---@param current_file string
---@param current_file_path string
---@return table? ast
---@return table? err
local function analyze_ast(current_file, current_file_path)
  local ast
  local ok, err = except.trycall(function()
    ast = aster.parse(current_file, current_file_path)
    local context = AnalyzerContext(analyzer.visitors, ast, generator)
    except.try(function()
      context = analyzer.analyze(context)
    end, function(e)
      e.message = ("%s%s"):format(context:get_visiting_traceback(1), e:get_message())
    end)
  end)
  if ok then
    return ast, nil
  else
    if type(err) == "string" then
      local actual_err = err
      err = {}
      err.__index = err
      setmetatable(err.__index, err)
      err.message = actual_err
      err.__tostring = function()
        return err.message
      end
    end
    return ast, err
  end
end

---@param node table
---@param pos integer
---@param foundnodes table
---@return table?
---@return string? err
local function find_nodes_in_scope_node(node, pos, foundnodes)
  if type(node) ~= "table" then
    return nil, "Node passed to find_nodes_in_scope_node was not a table"
  end

  local parent_node
  if node and node.attr and node.attr.scope and node.attr.scope.parent then
    parent_node = node.attr.scope.parent.node
  elseif node[2] and type(node[2]) == "table" and node[2].attr and node[2].attr.scope and node[2].attr.scope.node then
    parent_node = node[2].attr.scope.node
  end

  if
    node._astnode
    and (
      (node.pos and node.pos <= pos and node.endpos and node.endpos >= pos)
      or (
        parent_node
        and parent_node.pos
        and parent_node.pos <= pos
        and parent_node.endpos
        and parent_node.endpos >= pos
      )
    )
  then
    foundnodes[#foundnodes + 1] = node
  end

  for i = 1, node.nargs or #node do
    find_nodes_in_scope_node(node[i], pos, foundnodes)
  end
  return foundnodes, nil
end

---@param pos integer
---@param current_file string
---@return integer line
---@return integer char
local function pos_to_line_and_char(pos, current_file)
  local s_line = 0
  local pos_at_line = 0
  local text = current_file:sub(1, pos)
  for line in text:gmatch("[^\r\n]*\r?\n") do
    pos_at_line = pos_at_line + #line
    s_line = s_line + 1
  end
  local s_char = pos - pos_at_line - 1
  return s_line, s_char
end

local current_file = [[
require 'hashmap'
require 'io'
require 'os'
require 'arg'
require 'traits'

local optional = facultative

local Option = @record{
  opt_short: string,
  opt_desc: string,
  opt_default: string,
}

local Argparse = @record{
  name: string,
  desc: string,
  epilog: string,
  options: hashmap(string, Option),
  args: hashmap(string, string)
}

function Argparse.new(name: optional(string), desc: optional(string), epilog: optional(string)): Argparse
  local parser: Argparse
  ## if name.type.is_string then
    parser.name = name
  ## end
  ## if desc.type.is_string then
    parser.desc = desc
  ## end
  ## if epilog.type.is_string then
    parser.epilog = epilog
  ## end
  return parser
end
  
]]
local current_file_path = "/home/user/Desktop/test.nelua"

local ast, err = analyze_ast(current_file, current_file_path)

if not ast then
  io.stderr:write(err .. "/n")
  os.exit(1)
end

local scope_nodes, err = find_nodes_in_scope_node(ast, ast.endpos, {})

if scope_nodes then
  for _, node in ipairs(scope_nodes) do
    if node.pos and node.attr and node.attr.name then
      local s_line, s_char = pos_to_line_and_char(node.pos, current_file)
      print(("node_name: %s, s_line: %s, s_char: %s"):format(node.attr.name, s_line, s_char))
    end
  end
else
  io.stderr:write(err .. "/n")
  os.exit(1)
end

[DeanHnter]

Super helpful thanks so much!