Create Rowan parse tree, store document contents as text, and use Biome position encoding converters

Cargo.toml

@@ -23,3 +23,11 @@ rust-version = "1.85"
 edition = "2021"
 license = "MIT"
 authors = ["Posit Software, PBC"]
+
+[workspace.dependencies]
+biome_line_index = { git = "https://github.com/biomejs/biome", rev = "c13fc60726883781e4530a4437724273b560c8e0" }
+aether_lsp_utils = { git = "https://github.com/posit-dev/air", rev = "f959e32eee91" }
+aether_parser = { git = "https://github.com/posit-dev/air", package = "air_r_parser", rev = "f959e32eee91" }
+aether_syntax = { git = "https://github.com/posit-dev/air", package = "air_r_syntax", rev = "f959e32eee91" }
+# For https://github.com/ebkalderon/tower-lsp/pull/428
+tower-lsp = { branch = "bugfix/patches", git = "https://github.com/lionel-/tower-lsp" }

crates/ark/Cargo.toml

@@ -16,12 +16,16 @@ amalthea = { path = "../amalthea" }
 anyhow = "1.0.80"
 async-trait = "0.1.66"
 base64 = "0.21.0"
+biome_line_index.workspace = true
 bus = "2.3.0"
 cfg-if = "1.0.0"
 crossbeam = { version = "0.8.2", features = ["crossbeam-channel"] }
 ctor = "0.1.26"
 dap = { git = "https://github.com/sztomi/dap-rs", branch = "main" }
 dashmap = "5.4.0"
+aether_parser.workspace = true
+aether_syntax.workspace = true
+aether_lsp_utils.workspace = true
 ego-tree = "0.6.2"
 harp = { path = "../harp" }
 http = "0.2.9"
@@ -38,16 +42,16 @@ regex = "1.10.0"
 reqwest = { version = "0.12.5", default-features = false, features = ["json"] }
 reqwest-retry = "0.6.1"
 reqwest-middleware = "0.3.3"
-ropey = "1.6.0"
 rust-embed = "8.0.0"
 scraper = "0.15.0"
 serde = { version = "1.0.183", features = ["derive"] }
 serde_json = { version = "1.0.94", features = ["preserve_order"] }
 stdext = { path = "../stdext" }
+streaming-iterator = "0.1.9"
 tokio = { version = "1.26.0", features = ["full"] }
-tower-lsp = "0.19.0"
-tree-sitter = "0.23.0"
-tree-sitter-r = { git = "https://github.com/r-lib/tree-sitter-r", rev = "95aff097aa927a66bb357f715b58cde821be8867" }
+tower-lsp.workspace = true
+tree-sitter = "0.24.7"
+tree-sitter-r = { git = "https://github.com/r-lib/tree-sitter-r", rev = "daa26a2ff0d9546e9125c7d8bcec046027f02070" }
 uuid = "1.3.0"
 url = "2.4.1"
 walkdir = "2"

crates/ark/src/lsp/backend.rs

@@ -584,7 +584,7 @@ pub fn start_lsp(
 fn new_jsonrpc_error(message: String) -> jsonrpc::Error {
     jsonrpc::Error {
         code: jsonrpc::ErrorCode::ServerError(-1),
-        message,
+        message: message.into(),
         data: None,
     }
 }

crates/ark/src/lsp/code_action.rs

@@ -15,7 +15,7 @@ use url::Url;
 
 use crate::lsp::capabilities::Capabilities;
 use crate::lsp::code_action::roxygen::roxygen_documentation;
-use crate::lsp::documents::Document;
+use crate::lsp::document::Document;
 
 mod roxygen;
 

crates/ark/src/lsp/code_action/roxygen.rs

@@ -5,9 +5,8 @@ use crate::lsp::capabilities::Capabilities;
 use crate::lsp::code_action::code_action;
 use crate::lsp::code_action::code_action_workspace_text_edit;
 use crate::lsp::code_action::CodeActions;
-use crate::lsp::documents::Document;
-use crate::lsp::encoding::convert_point_to_position;
-use crate::lsp::traits::rope::RopeExt;
+use crate::lsp::document::Document;
+use crate::lsp::traits::node::NodeExt;
 use crate::treesitter::BinaryOperatorType;
 use crate::treesitter::NodeTypeExt;
 
@@ -64,8 +63,8 @@ pub(crate) fn roxygen_documentation(
 
     // Fairly simple detection of existing `#'` on the previous line (but starting at the
     // same `column` offset), which tells us not to provide this code action
-    if let Some(previous_line) = document.contents.get_line(position.row.saturating_sub(1)) {
-        if let Some(previous_line) = previous_line.get_byte_slice(position.column..) {
+    if let Some(previous_line) = document.get_line(position.row.saturating_sub(1)) {
+        if let Some(previous_line) = previous_line.get(position.column..) {
             let mut previous_line = previous_line.bytes();
 
             if previous_line
@@ -91,11 +90,7 @@ pub(crate) fn roxygen_documentation(
 
     for child in parameters.children_by_field_name("parameter", &mut cursor) {
         let parameter_name = child.child_by_field_name("name")?;
-        let parameter_name = document
-            .contents
-            .node_slice(&parameter_name)
-            .ok()?
-            .to_string();
+        let parameter_name = parameter_name.node_to_string(&document.contents).ok()?;
         parameter_names.push(parameter_name);
     }
 
@@ -105,8 +100,10 @@ pub(crate) fn roxygen_documentation(
     // We insert the documentation string at the start position of the function name.
     // This handles the indentation of the first documentation line, and makes new line
     // handling trivial (we just add a new line to every documentation line).
-    let position = convert_point_to_position(&document.contents, position);
-    let range = tower_lsp::lsp_types::Range::new(position, position);
+    let position = document
+        .lsp_position_from_tree_sitter_point(position)
+        .ok()?;
+    let range = lsp_types::Range::new(position, position);
     let edit = lsp_types::TextEdit::new(range, documentation);
     let edit =
         code_action_workspace_text_edit(uri.clone(), document.version, vec![edit], capabilities);
@@ -175,7 +172,7 @@ mod tests {
     use crate::lsp::capabilities::Capabilities;
     use crate::lsp::code_action::roxygen::roxygen_documentation;
     use crate::lsp::code_action::CodeActions;
-    use crate::lsp::documents::Document;
+    use crate::lsp::document::Document;
 
     fn point_range(point: Point, byte: usize) -> Range {
         Range {

crates/ark/src/lsp/completions/completion_context.rs

@@ -20,7 +20,7 @@ pub(crate) struct CompletionContext<'a> {
     pub(crate) state: &'a WorldState,
     pipe_root_cell: OnceCell<Option<PipeRoot>>,
     containing_call_cell: OnceCell<Option<Node<'a>>>,
-    function_context_cell: OnceCell<FunctionContext>,
+    function_context_cell: OnceCell<anyhow::Result<FunctionContext>>,
 }
 
 impl<'a> CompletionContext<'a> {
@@ -54,8 +54,10 @@ impl<'a> CompletionContext<'a> {
             .get_or_init(|| node_find_containing_call(self.document_context.node))
     }
 
-    pub fn function_context(&self) -> &FunctionContext {
+    pub fn function_context(&self) -> anyhow::Result<&FunctionContext> {
         self.function_context_cell
             .get_or_init(|| FunctionContext::new(&self.document_context))
+            .as_ref()
+            .map_err(|err| anyhow::anyhow!("{err:?}"))
     }
 }

crates/ark/src/lsp/completions/completion_item.rs

@@ -43,8 +43,7 @@ use crate::lsp::completions::function_context::FunctionRefUsage;
 use crate::lsp::completions::types::CompletionData;
 use crate::lsp::completions::types::PromiseStrategy;
 use crate::lsp::document_context::DocumentContext;
-use crate::lsp::encoding::convert_point_to_position;
-use crate::lsp::traits::rope::RopeExt;
+use crate::lsp::traits::node::NodeExt;
 use crate::treesitter::NodeType;
 use crate::treesitter::NodeTypeExt;
 
@@ -102,7 +101,7 @@ pub(super) fn completion_item_from_assignment(
     let lhs = node.child_by_field_name("lhs").into_result()?;
     let rhs = node.child_by_field_name("rhs").into_result()?;
 
-    let label = context.document.contents.node_slice(&lhs)?.to_string();
+    let label = lhs.node_as_str(&context.document.contents)?.to_string();
 
     // TODO: Resolve functions that exist in-document here.
     let mut item = completion_item(label.clone(), CompletionData::ScopeVariable {
@@ -125,11 +124,7 @@ pub(super) fn completion_item_from_assignment(
     // benefit from the logic in completion_item_from_function() :(
     if rhs.node_type() == NodeType::FunctionDefinition {
         if let Some(parameters) = rhs.child_by_field_name("parameters") {
-            let parameters = context
-                .document
-                .contents
-                .node_slice(&parameters)?
-                .to_string();
+            let parameters = parameters.node_as_str(&context.document.contents)?;
             item.detail = Some(join!(label, parameters));
         }
 
@@ -651,7 +646,9 @@ fn completion_item_from_dot_dot_dot(
 
     item.kind = Some(CompletionItemKind::FIELD);
 
-    let position = convert_point_to_position(&context.document.contents, context.point);
+    let position = context
+        .document
+        .lsp_position_from_tree_sitter_point(context.point)?;
 
     let range = Range {
         start: position,

crates/ark/src/lsp/completions/function_context.rs

@@ -5,14 +5,14 @@
 //
 //
 
+use stdext::result::ResultExt;
+use tower_lsp::lsp_types;
 use tower_lsp::lsp_types::Range;
 use tree_sitter::Node;
 
 use crate::lsp::document_context::DocumentContext;
-use crate::lsp::encoding::convert_point_to_position;
-use crate::lsp::encoding::convert_tree_sitter_range_to_lsp_range;
+use crate::lsp::traits::node::NodeExt;
 use crate::treesitter::node_find_parent_call;
-use crate::treesitter::node_text;
 use crate::treesitter::BinaryOperatorType;
 use crate::treesitter::NodeType;
 use crate::treesitter::NodeTypeExt;
@@ -54,25 +54,24 @@ pub(crate) enum ArgumentsStatus {
 }
 
 impl FunctionContext {
-    pub(crate) fn new(document_context: &DocumentContext) -> Self {
+    pub(crate) fn new(document_context: &DocumentContext) -> anyhow::Result<Self> {
         let completion_node = document_context.node;
 
         let Some(effective_function_node) = get_effective_function_node(completion_node) else {
             // We shouldn't ever attempt to instantiate a FunctionContext or
             // function-flavored CompletionItem in this degenerate case, but we
             // return a dummy FunctionContext just to be safe.
-            let node_end = convert_point_to_position(
-                &document_context.document.contents,
-                completion_node.range().end_point,
-            );
+            let node_end = document_context
+                .document
+                .lsp_position_from_tree_sitter_point(completion_node.range().end_point)?;
 
-            return Self {
+            return Ok(Self {
                 name: String::new(),
-                range: tower_lsp::lsp_types::Range::new(node_end, node_end),
+                range: lsp_types::Range::new(node_end, node_end),
                 usage: FunctionRefUsage::Call,
                 arguments_status: ArgumentsStatus::Absent,
                 cursor_is_at_end: true,
-            };
+            });
         };
 
         let usage = determine_function_usage(
@@ -93,7 +92,7 @@ impl FunctionContext {
             cursor.row == node_range.end_point.row && cursor.column == node_range.end_point.column;
 
         let name = match function_name_node {
-            Some(node) => node_text(&node, &document_context.document.contents).unwrap_or_default(),
+            Some(node) => node.node_to_string(&document_context.document.contents)?,
             None => String::new(),
         };
 
@@ -107,26 +106,26 @@ impl FunctionContext {
             "FunctionContext created with name: '{name}', usage: {usage:?}, arguments: {arguments_status:?}, cursor at end: {is_cursor_at_end}"
         );
 
-        Self {
+        Ok(Self {
             name,
             range: match function_name_node {
-                Some(node) => convert_tree_sitter_range_to_lsp_range(
-                    &document_context.document.contents,
-                    node.range(),
-                ),
+                Some(node) => document_context
+                    .document
+                    .lsp_range_from_tree_sitter_range(node.range())?,
                 None => {
                     // Create a zero-width range at the end of the effective_function_node
-                    let node_end = convert_point_to_position(
-                        &document_context.document.contents,
-                        effective_function_node.range().end_point,
-                    );
-                    tower_lsp::lsp_types::Range::new(node_end, node_end)
+                    let node_end = document_context
+                        .document
+                        .lsp_position_from_tree_sitter_point(
+                            effective_function_node.range().end_point,
+                        )?;
+                    lsp_types::Range::new(node_end, node_end)
                 },
             },
             usage,
             arguments_status,
             cursor_is_at_end: is_cursor_at_end,
-        }
+        })
     }
 }
 
@@ -165,7 +164,7 @@ static FUNCTIONS_EXPECTING_A_FUNCTION_REFERENCE: &[&str] = &[
     "str",
 ];
 
-fn is_inside_special_function(node: &Node, contents: &ropey::Rope) -> bool {
+fn is_inside_special_function(node: &Node, contents: &str) -> bool {
     let Some(call_node) = node_find_parent_call(node) else {
         return false;
     };
@@ -174,9 +173,12 @@ fn is_inside_special_function(node: &Node, contents: &ropey::Rope) -> bool {
         return false;
     };
 
-    let call_name = node_text(&call_name_node, contents).unwrap_or_default();
+    let call_name = call_name_node
+        .node_as_str(contents)
+        .log_err()
+        .unwrap_or_default();
 
-    FUNCTIONS_EXPECTING_A_FUNCTION_REFERENCE.contains(&call_name.as_str())
+    FUNCTIONS_EXPECTING_A_FUNCTION_REFERENCE.contains(&call_name)
 }
 
 /// Checks if the node is inside a help operator context like `?foo` or `method?foo`
@@ -226,7 +228,7 @@ fn determine_arguments_status(function_container_node: &Node) -> ArgumentsStatus
     }
 }
 
-fn determine_function_usage(node: &Node, contents: &ropey::Rope) -> FunctionRefUsage {
+fn determine_function_usage(node: &Node, contents: &str) -> FunctionRefUsage {
     if is_inside_special_function(node, contents) || is_inside_help_operator(node) {
         FunctionRefUsage::Value
     } else {

crates/ark/src/lsp/completions/provide.rs

@@ -12,7 +12,7 @@ use crate::lsp::completions::sources::composite;
 use crate::lsp::completions::sources::unique;
 use crate::lsp::document_context::DocumentContext;
 use crate::lsp::state::WorldState;
-use crate::treesitter::node_text;
+use crate::lsp::traits::node::NodeExt;
 use crate::treesitter::NodeTypeExt;
 
 // Entry point for completions.
@@ -23,7 +23,9 @@ pub(crate) fn provide_completions(
 ) -> anyhow::Result<Vec<CompletionItem>> {
     log::info!(
         "provide_completions() - Completion node text: '{node_text}', Node type: '{node_type:?}'",
-        node_text = node_text(&document_context.node, &document_context.document.contents)
+        node_text = document_context
+            .node
+            .node_as_str(&document_context.document.contents)
             .unwrap_or_default(),
         node_type = document_context.node.node_type()
     );

crates/ark/src/lsp/completions/sources/composite.rs

@@ -220,8 +220,8 @@ mod tests {
     use crate::lsp::completions::completion_context::CompletionContext;
     use crate::lsp::completions::sources::composite::get_completions;
     use crate::lsp::completions::sources::composite::is_identifier_like;
+    use crate::lsp::document::Document;
     use crate::lsp::document_context::DocumentContext;
-    use crate::lsp::documents::Document;
     use crate::lsp::state::WorldState;
     use crate::r_task;
     use crate::treesitter::NodeType;