sped up ext handling in language crate using constants and aho corasick; probably premature optimization, but it is faster

Author: bashandbone

.cargo/config.toml

@@ -0,0 +1,5 @@
+[unstable]
+codegen-backend = true
+
+[profile.dev]
+codegen-backend = "cranelift"

Cargo.lock

@@ -2,7 +2,6 @@
 # SPDX-FileContributor: Adam Poulemanos <[email protected]>
 #
 # SPDX-License-Identifier: MIT OR Apache-2.0
-cargo-features = ["codegen-backend"]
 
 # =========================================================
 #*               THREAD - Workspace
@@ -64,8 +63,8 @@ macro_rules_attribute = { version = "0.2.2" }
 async-trait = { version = "0.1.88" }
 
 bit-set = { version = "0.8.0" }
-ignore = { version = "0.4.22" }
-regex = { version = "1.10" }
+ignore = { version = "0.4.23" }
+regex = { version = "1.11.1" }
 thiserror = { version = "2.0.12" }
 
 # The center of it all

Cargo.toml

@@ -44,7 +44,7 @@ matching = ["dep:regex"]
 
 [dev-dependencies]
 tree-sitter-typescript = "0.23.2"
-thread-language = { workspace = true, features = ["builtin-parser"] }
+thread-language = { workspace = true, features = ["all-parsers"] }
 criterion = { version = "0.6.0", features = ["html_reports"] }
 
 [[bench]]

crates/ast-engine/Cargo.toml

@@ -121,7 +121,11 @@ impl<T, D: Doc + 'static> PinnedNodeData<D, T> {
     where
         F: FnOnce(&'static Root<D>) -> T,
     {
-        // TODO: explain why unsafe works here and what guarantee it needs
+        // SAFETY: We're creating a 'static reference to `pin`, which is safe because:
+        // 1. This struct owns `pin` and keeps it alive for the container's entire lifetime
+        // 2. Tree-sitter nodes are heap-allocated with stable pointers
+        // 3. We re-adopt nodes on each access to ensure they point to valid memory
+        // The 'static lifetime is a "lie" to the type system - nodes live as long as this container
         let reference = unsafe { &*(&raw const pin) as &'static Root<D> };
         let data = func(reference);
         Self { pin, data }
@@ -144,7 +148,9 @@ where
 }
 
 /// # Safety
-/// TODO: explain unsafe trait
+/// This trait is unsafe because implementors must ensure that `visit_nodes` calls
+/// the provided function on all nodes contained within the data structure.
+/// Failure to do so will result in stale node pointers that may reference freed memory.
 pub unsafe trait NodeData<D: Doc> {
     type Data;
     fn get_data(&self) -> &Self::Data;

crates/ast-engine/src/pinned.rs

@@ -363,6 +363,6 @@ if (true) {
 
     #[test]
     fn test_nested_matching_replace() {
-        // TODO impossible, we don't support nested replacement
+        // TODO we don't support nested replacement yet
     }
 }

crates/ast-engine/src/replacer/template.rs

@@ -8,7 +8,7 @@ description = "Language definitions and parsers for Thread"
 keywords = ["ast", "pattern", "codemod", "search", "rewrite", "languages"]
 license = "AGPL-3.0-or-later AND MIT"
 readme = "README.md"
-categories = ["ast", "pattern", "codemod", "search", "rewrite",]
+categories = ["ast", "pattern", "codemod", "search", "rewrite"]
 version = "0.1.0"
 authors = [
   "Herrington Darkholme <[email protected]>",
@@ -24,14 +24,17 @@ include.workspace = true
 cc = "1.2.30"
 
 [dependencies]
-thread-ast-engine = { workspace = true, features = ["parsing"] }
+thread-ast-engine = { workspace = true }
 thread-utils = { workspace = true, default-features = false, features = [
   "hashers",
 ] }
-aho-corasick.workspace = true
-ignore.workspace = true
-serde.workspace = true
-tree-sitter.workspace = true
+
+cfg-if = { workspace = true }
+
+aho-corasick = { workspace = true }
+ignore = { workspace = true }
+serde = { workspace = true }
+tree-sitter = { workspace = true }
 
 tree-sitter-bash = { version = "0.25.0", optional = true }
 tree-sitter-c = { version = "0.24.1", optional = true }
@@ -57,37 +60,41 @@ tree-sitter-typescript = { version = "0.23.2", optional = true }
 tree-sitter-yaml = { version = "0.7.1", optional = true }
 
 [features]
-default = ["builtin-parser"]
-matching = ["thread-ast-engine/matching"]
+default = ["all-parsers", "tree-sitter-parsing"]
+tree-sitter-parsing = ["thread-ast-engine/parsing"]
+matching = ["thread-ast-engine/matching", "tree-sitter-parsing"]
+
 
 # Individual language features
-bash = ["tree-sitter-bash"]
-c = ["tree-sitter-c"]
-cpp = ["tree-sitter-cpp"]
-csharp = ["tree-sitter-c-sharp"]
-css = ["tree-sitter-css"]
-elixir = ["tree-sitter-elixir"]
-go = ["tree-sitter-go"]
-haskell = ["tree-sitter-haskell"]
-html = ["tree-sitter-html"]
-html-embedded = ["html", "css", "matching", "typescript", "tsx", "javascript"]
-java = ["tree-sitter-java"]
-javascript = ["tree-sitter-javascript"]
-json = ["tree-sitter-json"]
-kotlin = ["tree-sitter-kotlin"]
-lua = ["tree-sitter-lua"]
-php = ["tree-sitter-php"]
-python = ["tree-sitter-python"]
-ruby = ["tree-sitter-ruby"]
-rust = ["tree-sitter-rust"]
-scala = ["tree-sitter-scala"]
-swift = ["tree-sitter-swift"]
-typescript = ["tree-sitter-typescript"]
-tsx = ["tree-sitter-typescript"]
-yaml = ["tree-sitter-yaml"]
+bash = ["tree-sitter-bash", "tree-sitter-parsing"]
+c = ["tree-sitter-c", "tree-sitter-parsing"]
+cpp = ["tree-sitter-cpp", "tree-sitter-parsing"]
+csharp = ["tree-sitter-c-sharp", "tree-sitter-parsing"]
+css = ["tree-sitter-css", "tree-sitter-parsing"]
+elixir = ["tree-sitter-elixir", "tree-sitter-parsing"]
+go = ["tree-sitter-go", "tree-sitter-parsing"]
+haskell = ["tree-sitter-haskell", "tree-sitter-parsing"]
+html = ["tree-sitter-html", "tree-sitter-parsing"]
+html-embedded = ["html", "css", "matching", "typescript", "tsx", "javascript", "tree-sitter-parsing"]
+java = ["tree-sitter-java", "tree-sitter-parsing"]
+javascript = ["tree-sitter-javascript", "tree-sitter-parsing"]
+json = ["tree-sitter-json", "tree-sitter-parsing"]
+kotlin = ["tree-sitter-kotlin", "tree-sitter-parsing"]
+lua = ["tree-sitter-lua", "tree-sitter-parsing"]
+php = ["tree-sitter-php", "tree-sitter-parsing"]
+python = ["tree-sitter-python", "tree-sitter-parsing"]
+ruby = ["tree-sitter-ruby", "tree-sitter-parsing"]
+rust = ["tree-sitter-rust", "tree-sitter-parsing"]
+scala = ["tree-sitter-scala", "tree-sitter-parsing"]
+swift = ["tree-sitter-swift", "tree-sitter-parsing"]
+typescript = ["tree-sitter-typescript", "tree-sitter-parsing"]
+tsx = ["tree-sitter-typescript", "tree-sitter-parsing"]
+yaml = ["tree-sitter-yaml", "tree-sitter-parsing"]
 
+#! Important: `all-parsers` feature enables all tree-sitter parsers
 # Language groups
-builtin-parser = [
+all-parsers = [
+  "tree-sitter-parsing",
   "bash",
   "c",
   "cpp",
@@ -113,15 +120,31 @@ builtin-parser = [
   "tsx",
   "yaml",
 ]
-napi-lang = [
-  "css",
-  "html",
-  "html-embedded",
-  "javascript",
-  "typescript",
-  "tsx",
-]
+
+css-napi = ["tree-sitter-css", "napi-environment"]
+html-napi = ["tree-sitter-html", "napi-environment"]
+javascript-napi = ["tree-sitter-javascript", "napi-environment"]
+typescript-napi = ["tree-sitter-typescript", "napi-environment"]
+tsx-napi = ["tree-sitter-typescript", "napi-environment"]
+
+napi-compatible = [
+  "css-napi",
+  "html-napi",
+  "javascript-napi",
+  "typescript-napi",
+  "tsx-napi",
+  ]
+# =============== Important NAPI Information ============
+# Thread does **not** currently support NAPI builds.
+# We've included this from our fork of `ast-grep-language`, in case someone wants to build from it.
+# We may eventually add the capability, but it's not currently planned.
+# Pull requests welcome.
+#! Important: `napi-environment` feature disables the tree-sitter parser
+#! Tree-sitter cannot build for NAPI-WASM
+#! (that's Node API -- wasm for nodejs environments)
+napi-environment = []
 profiling = []
+no-enabled-langs = []
 
 [dev-dependencies]
 criterion = { version = "0.6", features = ["html_reports"] }
@@ -130,3 +153,7 @@ thread-ast-engine = { workspace = true, features = ["matching", "parsing"] }
 [[bench]]
 name = "performance"
 harness = false
+
+[[bench]]
+name = "extension_matching"
+harness = false

crates/language/Cargo.toml

@@ -69,8 +69,8 @@ These languages accept `$` in identifiers and use standard pattern processing:
 
 #### Parser Groups
 
-- **`builtin-parser`** (default) - Includes all language parsers
-- **`napi-lang`** - Includes only NAPI-compatible (WASM for Node.js environments) parsers (CSS, HTML, JavaScript, TypeScript)
+- **`all-parsers`** (default) - Includes all language parsers
+- **`napi-environment`** - Includes only NAPI-compatible (WASM for Node.js environments) parsers (CSS, HTML, JavaScript, TypeScript)
 
 #### Individual Languages